4 * This file is part of LyX, the document processor.
5 * Licence details can be found in the file COPYING.
9 * Full author contact details are available in file CREDITS.
19 #include "support/docstream.h"
24 enum mode_type {UNDECIDED_MODE, TEXT_MODE, MATH_MODE, MATHTEXT_MODE, TABLE_MODE};
26 mode_type asMode(mode_type oldmode, std::string const & str);
29 // These are TeX's catcodes
31 catEscape, // 0 backslash
42 catLetter, // 11 a-zA-Z
43 catOther, // 12 none of the above
46 catInvalid // 15 <delete>
57 FLAG_BRACE_LAST = 1 << 1, // last closing brace ends the parsing
58 FLAG_RIGHT = 1 << 2, // next \\right ends the parsing process
59 FLAG_END = 1 << 3, // next \\end ends the parsing process
60 FLAG_BRACK_LAST = 1 << 4, // next closing bracket ends the parsing
61 FLAG_TEXTMODE = 1 << 5, // we are in a box
62 FLAG_ITEM = 1 << 6, // read a (possibly braced) token
63 FLAG_LEAVE = 1 << 7, // leave the loop at the end
64 FLAG_SIMPLE = 1 << 8, // next $ leaves the loop
65 FLAG_EQUATION = 1 << 9, // next \] leaves the loop
66 FLAG_SIMPLE2 = 1 << 10, // next \) leaves the loop
67 FLAG_OPTION = 1 << 11, // read [...] style option
68 FLAG_BRACED = 1 << 12, // read {...} style argument
69 FLAG_CELL = 1 << 13, // read table cell
70 FLAG_TABBING = 1 << 14, // We are inside a tabbing environment
71 FLAG_RDELIM = 1 << 15, // next right delimiter ends the parsing
77 // Helper class for parsing
83 Token() : cs_(), cat_(catIgnore) {}
85 Token(docstring const & cs, CatCode cat) : cs_(to_utf8(cs)), cat_(cat) {}
87 /// Returns the token as string
88 std::string const & cs() const { return cs_; }
89 /// Returns the catcode of the token
90 CatCode cat() const { return cat_; }
91 /** Get the character of tokens that were constructed from a single
92 * character input or a two character input and cat_ == catEscape.
93 * FIXME: The intended usage is not clear. The Token class in
94 * ../mathed/MathParser.cpp (which is the anchestor of this
95 * class) uses a separate char member for this method. I
96 * believe that the intended usage is to not cover tokens with
97 * catEscape or catComment, e.g. \code
98 * return (cs_.empty() || cat_ == catEscape || cat_ == catComment) ? 0 : cs_[0];
100 * All usages of this method should be checked. gb 2011-01-05
102 char character() const { return cs_.empty() ? 0 : cs_[0]; }
103 /// Returns the token verbatim
104 std::string asInput() const;
105 /// Is the token an alphanumerical character?
106 bool isAlnumASCII() const;
115 std::ostream & operator<<(std::ostream & os, Token const & t);
118 extern void debugToken(std::ostream & os, Token const & t, unsigned int flags);
121 /// A docstream version that supports putback even when not buffered
122 class iparserdocstream
125 typedef idocstream::int_type int_type;
127 iparserdocstream(idocstream & is) : is_(is) {}
129 /// Like std::istream::operator bool()
130 /// Do not convert is_ implicitly to bool, since that is forbidden in C++11.
131 explicit operator bool() const { return s_.empty() ? !is_.fail() : true; }
133 /// change the encoding of the input stream to \p e (iconv name)
134 void setEncoding(std::string const & e);
136 // add to the list of characters to read before actually reading
138 void putback(char_type c);
140 // add to the list of characters to read before actually reading
142 void putback(const docstring &s);
144 /// Like std::istream::get()
145 iparserdocstream & get(char_type &c);
147 /// Like std::istream::good()
148 bool good() const { return s_.empty() ? is_.good() : true; }
150 /// Like std::istream::peek()
151 int_type peek() const { return s_.empty() ? is_.peek() : s_[0]; }
155 /// characters to read before actually reading the stream
161 * Actual parser class
163 * The parser parses every character of the inputstream into a token
164 * and classifies the token.
165 * The following transformations are done:
166 * - Consecutive spaces are combined into one single token with CatCode catSpace
167 * - Consecutive newlines are combined into one single token with CatCode catNewline
168 * - Comments and %\n combinations are parsed into one token with CatCode catComment
173 Parser(Parser const & p);
174 Parser & operator=(Parser const & p);
177 Parser(idocstream & is, std::string const & fixedenc);
179 Parser(std::string const & s);
183 /** forget already parsed next tokens and put the
184 * corresponding characters into the input stream for
185 * re-reading. Useful when changing catcodes. */
188 /// change the encoding of the input stream according to \p encoding
189 /// (latex name) and package \p package
190 bool setEncoding(std::string const & encoding, int package);
191 /// change the encoding of the input stream to \p encoding (iconv name)
192 bool setEncoding(std::string const & encoding);
193 /// get the current iconv encoding of the input stream
194 std::string getEncoding() const { return encoding_iconv_; }
197 CatCode catcode(char_type c) const;
199 void setCatcode(char c, CatCode cat);
200 /// set parser to normal or verbatim mode
201 void setCatcodes(cat_type t);
204 int lineno() const { return lineno_; }
207 /// store current position
209 /// restore previous position
211 /// forget last saved position
213 /// dump contents to screen
216 /// Does an optional argument follow after the current token?
217 bool hasOpt(std::string const & l = "[");
219 typedef std::pair<bool, std::string> Arg;
221 * Get an argument enclosed by \p left and \p right.
222 * If \p allow_escaping is true, a right delimiter escaped by a
223 * backslash does not count as delimiter, but is included in the
225 * \returns whether an argument was found in \p Arg.first and the
226 * argument in \p Arg.second. \see getArg().
228 Arg getFullArg(char left, char right, bool allow_escaping = true);
230 * Get an argument enclosed by \p left and \p right.
231 * If \p allow_escaping is true, a right delimiter escaped by a
232 * backslash does not count as delimiter, but is included in the
234 * \returns the argument (without \p left and \p right) or the empty
235 * string if the next non-space token is not \p left. Use
236 * getFullArg() if you need to know whether there was an empty
237 * argument or no argument at all.
239 std::string getArg(char left, char right, bool allow_escaping = true);
241 * Like getOpt(), but distinguishes between a missing argument ""
242 * and an empty argument "[]".
244 std::string getFullOpt(bool keepws = false,
245 char left = '[', char right = ']');
247 * \returns getArg('[', ']') including the brackets or the
248 * empty string if there is no such argument.
249 * No whitespace is eaten if \p keepws is true and no optional
250 * argument exists. This is important if an optional argument is
251 * parsed that would go after a command in ERT: In this case the
252 * whitespace is needed to separate the ERT from the subsequent
253 * word. Without it, the ERT and the next word would be concatenated
254 * during .tex export, thus creating an invalid command.
256 std::string getOpt(bool keepws = false);
258 * \returns getFullArg('(', ')') including the parentheses or the
259 * empty string if there is no such argument.
261 std::string getFullParentheseArg();
262 /// Check if we have a list preamble
263 bool hasListPreamble(std::string const & itemcmd);
265 * \returns the contents of the environment \p name.
266 * <tt>\begin{name}</tt> must be parsed already, <tt>\end{name}</tt>
267 * is parsed but not returned. This parses nested environments properly.
269 std::string const ertEnvironment(std::string const & name);
271 * The same as ertEnvironment(std::string const & name) but
272 * \begin and \end commands inside the name environment are not parsed.
273 * This function is designed to parse verbatim environments.
275 std::string const plainEnvironment(std::string const & name);
277 * Basically the same as plainEnvironment(std::string const & name) but
278 * instead of \begin and \end commands the parsing is started/stopped
279 * at given characters.
280 * This function is designed to parse verbatim commands.
282 std::string const plainCommand(char left, char right, std::string const & name);
284 * Returns everything before the main command argument.
285 * This is where the LaTeXParam value of a layout is output.
287 std::string const getCommandLatexParam();
289 * Basically the same as plainEnvironment() but the parsing is
290 * stopped at string \p end_string. Contrary to the other
291 * methods, this uses proper catcode setting. This function is
292 * designed to parse verbatim environments and command. The
293 * intention is to eventually replace all of its siblings. the
294 * member \p first of the result tells whether the arg was
295 * found and the member \p second is the value. If \p
296 * allow_linebreak is false, then the parsing is limited to one line
298 Arg verbatimStuff(std::string const & end_string,
299 bool allow_linebreak = true);
301 * \returns the contents of the environment \p name.
302 * <tt>\begin{name}</tt> must be parsed already,
303 * <tt>\end{name}</tt> is parsed but not returned. The string
304 * is parsed with proper verbatim catcodes and one newline is
305 * removed from head and tail of the string if applicable.
307 std::string const verbatimEnvironment(std::string const & end_string);
309 std::string verbatim_item();
311 std::string verbatimOption();
313 void error(std::string const & msg) const;
314 /// The previous token.
315 Token const prev_token() const;
316 /// The current token.
317 Token const curr_token() const;
318 /// The next token. Caution: If this is called, an encoding change is
319 /// only possible again after get_token() has been called.
320 Token const next_token();
321 /// The next but one token. Caution: If this is called, an encoding
322 /// change is only possible again after get_token() has been called
324 Token const next_next_token();
325 /// Make the next token current and return that.
326 Token const get_token();
327 /// \return whether the current token starts a new paragraph
329 /// skips spaces (and comments if \p skip_comments is true)
330 /// \return whether whitespace was skipped (not comments)
331 bool skip_spaces(bool skip_comments = false);
332 /// puts back spaces (and comments if \p skip_comments is true)
333 void unskip_spaces(bool skip_comments = false);
334 /// Is any further input pending()? This is not like
335 /// std::istream::good(), which returns true if all available input
336 /// was read, and the next attempt to read would return EOF.
338 /// resets the parser to initial state
342 /// Setup catcode table
344 /// Parses one token from \p is
347 void push_back(Token const & t);
351 std::vector<Token> tokens_;
355 std::vector<unsigned> positions_;
357 idocstringstream * iss_;
359 iparserdocstream is_;
360 /// iconv name of the current encoding
361 std::string encoding_iconv_;
363 CatCode theCatcode_[256];
365 cat_type theCatcodesType_;