X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2Ftex2lyx%2FParser.h;h=11af29b6667e312234195b8879aa26154488213a;hb=28f43b18963a5088529d4ac7c165fa95e62fd02b;hp=0a360fef974803bdf0a9f33a471b756f72a3ffaf;hpb=d3b2f943880ab1d3211785e4a3a5c1e8a207221e;p=lyx.git

diff --git a/src/tex2lyx/Parser.h b/src/tex2lyx/Parser.h
index 0a360fef97..11af29b666 100644
--- a/src/tex2lyx/Parser.h
+++ b/src/tex2lyx/Parser.h
@@ -4,7 +4,7 @@
  * This file is part of LyX, the document processor.
  * Licence details can be found in the file COPYING.
  *
- * \author André Pönitz
+ * \author André Pönitz
  *
  * Full author contact details are available in file CREDITS.
  */
@@ -12,10 +12,11 @@
 #ifndef PARSER_H
 #define PARSER_H
 
-#include <vector>
 #include <string>
 #include <utility>
+#include <vector>
 
+#include "support/docstream.h"
 
 namespace lyx {
 
@@ -46,9 +47,6 @@ enum CatCode {
 };
 
 
-CatCode catcode(unsigned char c);
-
-
 enum {
 	FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing
 	FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
@@ -75,18 +73,16 @@ enum {
 class Token {
 public:
 	///
-	Token() : cs_(), char_(0), cat_(catIgnore) {}
-	///
-	Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
+	Token() : cs_(), cat_(catIgnore) {}
 	///
-	Token(std::string const & cs, CatCode cat) : cs_(cs), char_(0), cat_(cat) {}
+	Token(docstring const & cs, CatCode cat) : cs_(to_utf8(cs)), cat_(cat) {}
 
 	///
 	std::string const & cs() const { return cs_; }
 	/// Returns the catcode of the token
 	CatCode cat() const { return cat_; }
 	///
-	char character() const { return char_; }
+	char character() const { return cs_.empty() ? 0 : cs_[0]; }
 	/// Returns the token as string
 	std::string asString() const;
 	/// Returns the token verbatim
@@ -96,8 +92,6 @@ private:
 	///
 	std::string cs_;
 	///
-	char char_;
-	///
 	CatCode cat_;
 };
 
@@ -119,9 +113,14 @@ class Parser {
 
 public:
 	///
-	Parser(std::istream & is);
+	Parser(idocstream & is);
 	///
 	Parser(std::string const & s);
+	///
+	~Parser();
+
+	/// change the encoding of the input stream
+	void setEncoding(std::string const & encoding);
 
 	///
 	int lineno() const { return lineno_; }
@@ -148,39 +147,46 @@ public:
 	std::string getArg(char left, char right);
 	/*!
 	 * \returns getFullArg('[', ']') including the brackets or the
-	 * empty string if no argument was found.
+	 * empty string if there is no such argument.
 	 */
 	std::string getFullOpt();
-	/// \returns getArg('[', ']') including the brackets
+	/*!
+	 * \returns getArg('[', ']') including the brackets or the
+	 * empty string if there is no such argument.
+	 */
 	std::string getOpt();
+	/*!
+	 * \returns getFullArg('(', ')') including the parentheses or the
+	 * empty string if there is no such argument.
+	 */
+	std::string getFullParentheseArg();
 	/*!
 	 * \returns the contents of the environment \p name.
 	 * <tt>\begin{name}</tt> must be parsed already, <tt>\end{name}</tt>
 	 * is parsed but not returned.
 	 */
-	std::string getFullParentheseOpt();
-	/// \returns getArg('(', ')') including the parentheses
 	std::string const verbatimEnvironment(std::string const & name);
-	/// Returns the character of the current token and increments the token position.
+	/*!
+	 * Returns the character of the current token and increments
+	 * the token position.
+	 */
 	char getChar();
 	///
 	void error(std::string const & msg);
-	/// Parses \p is into tokens
-	void tokenize(std::istream & is);
+	/// Parses one token from \p is 
+	void tokenize_one();
 	///
 	void push_back(Token const & t);
-	///
-	void pop_back();
 	/// The previous token.
 	Token const & prev_token() const;
 	/// The current token.
 	Token const & curr_token() const;
 	/// The next token.
-	Token const & next_token() const;
+	Token const & next_token();
 	/// Make the next token current and return that.
 	Token const & get_token();
 	/// \return whether the current token starts a new paragraph
-	bool isParagraph() const;
+	bool isParagraph();
 	/// skips spaces (and comments if \p skip_comments is true)
 	void skip_spaces(bool skip_comments = false);
 	/// puts back spaces (and comments if \p skip_comments is true)
@@ -188,7 +194,7 @@ public:
 	///
 	void lex(std::string const & s);
 	///
-	bool good() const;
+	bool good();
 	///
 	std::string verbatim_item();
 	///
@@ -207,6 +213,10 @@ private:
 	std::vector<Token> tokens_;
 	///
 	unsigned pos_;
+	///
+	idocstringstream * iss_;
+	///
+	idocstream & is_;
 };