#include <config.h>
-#include "Encoding.h"
#include "Parser.h"
-#include "support/foreach.h"
+
+#include "Encoding.h"
#include "support/lstrings.h"
#include "support/textutils.h"
return c;
}
-}
+} // namespace
//
// Token
}
-void iparserdocstream::putback(docstring s)
+void iparserdocstream::putback(docstring const & s)
{
s_ = s + s_;
}
Parser::Parser(idocstream & is, std::string const & fixedenc)
- : lineno_(0), pos_(0), iss_(0), is_(is),
+ : lineno_(0), pos_(0), iss_(nullptr), is_(is),
encoding_iconv_(fixedenc.empty() ? "UTF-8" : fixedenc),
theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES),
fixed_enc_(!fixedenc.empty())
{
if (fixed_enc_)
is_.setEncoding(fixedenc);
+ catInit();
}
// An idocstringstream can not change the encoding
fixed_enc_(true)
{
+ catInit();
}
}
-bool Parser::setEncoding(std::string const & e, int const & p)
+bool Parser::setEncoding(std::string const & e, int p)
{
// We may (and need to) use unsafe encodings here: Since the text is
// converted to unicode while reading from is_, we never see text in
return dummy;
// If tokenize_one() has not been called after the last get_token() we
// need to tokenize two more tokens.
- if (pos_ >= tokens_.size()) {
+ if (pos_ >= tokens_.size())
+ tokenize_one();
+ if (pos_ + 1 >= tokens_.size())
tokenize_one();
- if (pos_ + 1 >= tokens_.size())
- tokenize_one();
- }
return pos_ + 1 < tokens_.size() ? tokens_[pos_ + 1] : dummy;
}
if (pos_ >= tokens_.size())
return dummy;
}
- // cerr << "looking at token " << tokens_[pos_]
+ // cerr << "looking at token " << tokens_[pos_]
// << " pos: " << pos_ << '\n';
return tokens_[pos_++];
}
bool Parser::isParagraph()
{
- // A new paragraph in TeX ist started
+ // A new paragraph in TeX is started
// - either by a newline, following any amount of whitespace
// characters (including zero), and another newline
// - or the token \par
}
-bool Parser::good()
+bool Parser::good() const
{
if (pos_ < tokens_.size())
return true;
}
-bool Parser::hasOpt()
+bool Parser::hasOpt(string const & l)
{
// An optional argument can occur in any of the following forms:
// - \foo[bar]
putback();
break;
}
- bool const retval = (next_token().asInput() == "[");
+ bool const retval = (next_token().asInput() == l);
pos_ = oldpos;
return retval;
}
if (! good())
return make_pair(false, string());
+ int group_level = 0;
string result;
Token t = get_token();
} else {
while (good()) {
t = get_token();
+ // honor grouping
+ if (left != '{' && t.cat() == catBegin) {
+ ++group_level;
+ continue;
+ }
+ if (left != '{' && t.cat() == catEnd) {
+ --group_level;
+ continue;
+ }
// Ignore comments
if (t.cat() == catComment) {
if (!t.cs().empty())
continue;
}
if (allow_escaping) {
- if (t.cat() != catEscape && t.character() == right)
+ if (t.cat() != catEscape && t.character() == right
+ && group_level == 0)
break;
} else {
if (t.character() == right) {
if (t.cat() == catEscape)
result += '\\';
- break;
+ if (group_level == 0)
+ break;
}
}
result += t.asInput();
}
-string Parser::getFullOpt(bool keepws)
+string Parser::getFullOpt(bool keepws, char left, char right)
{
- Arg arg = getFullArg('[', ']');
+ Arg arg = getFullArg(left, right);
if (arg.first)
- return '[' + arg.second + ']';
+ return left + arg.second + right;
if (keepws)
unskip_spaces(true);
return string();
}
+bool Parser::hasListPreamble(string const & itemcmd)
+{
+ // remember current position
+ unsigned int oldpos = pos_;
+ // jump over arguments
+ if (hasOpt())
+ getOpt();
+ if (hasOpt("{"))
+ getArg('{', '}');
+ // and swallow spaces and comments
+ skip_spaces(true);
+ // we have a list preamble if the next thing
+ // that follows is not the \item command
+ bool res = next_token().cs() != itemcmd;
+ // back to orig position
+ pos_ = oldpos;
+ return res;
+}
+
+
string const Parser::ertEnvironment(string const & name)
{
if (!good())
}
+string const Parser::getCommandLatexParam()
+{
+ if (!good())
+ return string();
+ string res;
+ size_t offset = 0;
+ while (true) {
+ if (pos_ + offset >= tokens_.size())
+ tokenize_one();
+ if (pos_ + offset >= tokens_.size())
+ break;
+ Token t = tokens_[pos_ + offset];
+ if (t.cat() == catBegin)
+ break;
+ res += t.asInput();
+ ++offset;
+ }
+ return res;
+}
+
+
Parser::Arg Parser::verbatimStuff(string const & end_string, bool const allow_linebreak)
{
if (!good())
return Arg(false, string());
}
if (match_index) {
- oss << end_string.substr(0, match_index)
+ oss << end_string.substr(0, match_index)
<< t.asInput();
match_index = 0;
} else
putback();
res += '{' + verbatim_item() + '}';
} else
- res += t.cs();
+ res += t.asInput();
}
}
return res;
if (next_token().cat() == catBegin) {
Token t = get_token(); // skip brace
string res;
- for (Token t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
+ for (t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
if (t.cat() == catBegin) {
putback();
res += '{' + verbatim_item() + '}';
}
case catIgnore: {
- cerr << "ignoring a char: " << c << "\n";
+ cerr << "ignoring a char: " << static_cast<uint32_t>(c) << "\n";
break;
}
}
-void Parser::error(string const & msg)
+void Parser::error(string const & msg) const
{
cerr << "Line ~" << lineno_ << ": parse error: " << msg << endl;
dump();