4 * This file is part of LyX, the document processor.
5 * Licence details can be found in the file COPYING.
9 * Full author contact details are available in file CREDITS.
19 #include "support/docstream.h"
24 enum mode_type {UNDECIDED_MODE, TEXT_MODE, MATH_MODE, MATHTEXT_MODE, TABLE_MODE};
26 mode_type asMode(mode_type oldmode, std::string const & str);
29 // These are TeX's catcodes
31 catEscape, // 0 backslash
42 catLetter, // 11 a-zA-Z
43 catOther, // 12 none of the above
46 catInvalid // 15 <delete>
51 FLAG_BRACE_LAST = 1 << 1, // last closing brace ends the parsing
52 FLAG_RIGHT = 1 << 2, // next \\right ends the parsing process
53 FLAG_END = 1 << 3, // next \\end ends the parsing process
54 FLAG_BRACK_LAST = 1 << 4, // next closing bracket ends the parsing
55 FLAG_TEXTMODE = 1 << 5, // we are in a box
56 FLAG_ITEM = 1 << 6, // read a (possibly braced token)
57 FLAG_LEAVE = 1 << 7, // leave the loop at the end
58 FLAG_SIMPLE = 1 << 8, // next $ leaves the loop
59 FLAG_EQUATION = 1 << 9, // next \] leaves the loop
60 FLAG_SIMPLE2 = 1 << 10, // next \) leaves the loop
61 FLAG_OPTION = 1 << 11, // read [...] style option
62 FLAG_BRACED = 1 << 12, // read {...} style argument
63 FLAG_CELL = 1 << 13, // read table cell
64 FLAG_TABBING = 1 << 14 // We are inside a tabbing environment
70 // Helper class for parsing
76 Token() : cs_(), cat_(catIgnore) {}
78 Token(docstring const & cs, CatCode cat) : cs_(to_utf8(cs)), cat_(cat) {}
80 /// Returns the token as string
81 std::string const & cs() const { return cs_; }
82 /// Returns the catcode of the token
83 CatCode cat() const { return cat_; }
84 /** Get the character of tokens that were constructed from a single
85 * character input or a two character input and cat_ == catEscape.
86 * FIXME: The intended usage is not clear. The Token class in
87 * ../mathed/MathParser.cpp (which is the anchestor of this
88 * class) uses a separate char member for this method. I
89 * believe that the intended usage is to not cover tokens with
90 * catEscape, e.g. \code
91 * return (cs_.empty() || cat_ == catEscape) ? 0 : cs_[0];
93 * All usages of this method should be checked. gb 2011-01-05
95 char character() const { return cs_.empty() ? 0 : cs_[0]; }
96 /// Returns the token verbatim
97 std::string asInput() const;
98 /// Is the token an alphanumerical character?
99 bool isAlnumASCII() const;
108 std::ostream & operator<<(std::ostream & os, Token const & t);
111 extern void debugToken(std::ostream & os, Token const & t, unsigned int flags);
116 * Actual parser class
118 * The parser parses every character of the inputstream into a token
119 * and classifies the token.
120 * The following transformations are done:
121 * - Consecutive spaces are combined into one single token with CatCode catSpace
122 * - Consecutive newlines are combined into one single token with CatCode catNewline
123 * - Comments and %\n combinations are parsed into one token with CatCode catComment
128 Parser(Parser const & p);
129 Parser & operator=(Parser const & p);
132 Parser(idocstream & is);
134 Parser(std::string const & s);
138 /// change the latex encoding of the input stream
139 void setEncoding(std::string const & encoding);
140 /// get the current latex encoding of the input stream
141 std::string getEncoding() const { return encoding_latex_; }
144 int lineno() const { return lineno_; }
147 /// store current position
149 /// restore previous position
151 /// dump contents to screen
154 /// Does an optional argument follow after the current token?
157 typedef std::pair<bool, std::string> Arg;
159 * Get an argument enclosed by \p left and \p right.
160 * \returns wether an argument was found in \p Arg.first and the
161 * argument in \p Arg.second. \see getArg().
163 Arg getFullArg(char left, char right);
165 * Get an argument enclosed by \p left and \p right.
166 * \returns the argument (without \p left and \p right) or the empty
167 * string if the next non-space token is not \p left. Use
168 * getFullArg() if you need to know wether there was an empty
169 * argument or no argument at all.
171 std::string getArg(char left, char right);
173 * Like getOpt(), but distinguishes between a missing argument ""
174 * and an empty argument "[]".
176 std::string getFullOpt(bool keepws = false);
178 * \returns getArg('[', ']') including the brackets or the
179 * empty string if there is no such argument.
180 * No whitespace is eaten if \p keepws is true and no optional
181 * argument exists. This is important if an optional argument is
182 * parsed that would go after a command in ERT: In this case the
183 * whitespace is needed to separate the ERT from the subsequent
184 * word. Without it, the ERT and the next word would be concatenated
185 * during .tex export, thus creating an invalid command.
187 std::string getOpt(bool keepws = false);
189 * \returns getFullArg('(', ')') including the parentheses or the
190 * empty string if there is no such argument.
192 std::string getFullParentheseArg();
194 * \returns the contents of the environment \p name.
195 * <tt>\begin{name}</tt> must be parsed already, <tt>\end{name}</tt>
196 * is parsed but not returned.
198 std::string const verbatimEnvironment(std::string const & name);
200 * The same as verbatimEnvironment(std::string const & name) but
201 * \begin and \end commands inside the name environment are not parsed.
202 * This function is designed to parse verbatim environments.
204 std::string const plainEnvironment(std::string const & name);
206 * Basically the same as plainEnvironment(std::string const & name) but
207 * instead of \begin and \end commands the parsing is started/stopped
208 * at given characters.
209 * This function is designed to parse verbatim commands.
211 std::string const plainCommand(char left, char right, std::string const & name);
213 * Returns the character of the current token and increments
214 * the token position.
218 void error(std::string const & msg);
219 /// Parses one token from \p is
222 void push_back(Token const & t);
223 /// The previous token.
224 Token const prev_token() const;
225 /// The current token.
226 Token const curr_token() const;
228 Token const next_token();
229 /// The next but one token.
230 Token const next_next_token();
231 /// Make the next token current and return that.
232 Token const get_token();
233 /// \return whether the current token starts a new paragraph
235 /// skips spaces (and comments if \p skip_comments is true)
236 /// \return whether whitespace was skipped (not comments)
237 bool skip_spaces(bool skip_comments = false);
238 /// puts back spaces (and comments if \p skip_comments is true)
239 void unskip_spaces(bool skip_comments = false);
241 void lex(std::string const & s);
245 std::string verbatim_item();
247 std::string verbatimOption();
248 /// resets the parser to initial state
251 void setCatCode(char c, CatCode cat);
253 CatCode getCatCode(char c) const;
259 std::vector<Token> tokens_;
263 std::vector<unsigned> positions_;
265 idocstringstream * iss_;
268 /// latex name of the current encoding
269 std::string encoding_latex_;