#include "Encoding.h"
#include "Parser.h"
-#include "support/foreach.h"
#include "support/lstrings.h"
#include "support/textutils.h"
//
-Parser::Parser(idocstream & is)
- : lineno_(0), pos_(0), iss_(0), is_(is), encoding_iconv_("UTF-8"),
- theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES)
+Parser::Parser(idocstream & is, std::string const & fixedenc)
+ : lineno_(0), pos_(0), iss_(0), is_(is),
+ encoding_iconv_(fixedenc.empty() ? "UTF-8" : fixedenc),
+ theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES),
+ fixed_enc_(!fixedenc.empty())
{
+ if (fixed_enc_)
+ is_.setEncoding(fixedenc);
+ catInit();
}
: lineno_(0), pos_(0),
iss_(new idocstringstream(from_utf8(s))), is_(*iss_),
encoding_iconv_("UTF-8"),
- theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES)
+ theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES),
+ // An idocstringstream can not change the encoding
+ fixed_enc_(true)
{
+ catInit();
}
{
//cerr << "setting encoding to " << e << std::endl;
encoding_iconv_ = e;
- is_.setEncoding(e);
+ // If the encoding is fixed, we must not change the stream encoding
+ // (because the whole input uses that encoding, e.g. if it comes from
+ // the clipboard). We still need to track the original encoding in
+ // encoding_iconv_, so that the generated output is correct.
+ if (!fixed_enc_)
+ is_.setEncoding(e);
return true;
}
return dummy;
// If tokenize_one() has not been called after the last get_token() we
// need to tokenize two more tokens.
- if (pos_ >= tokens_.size()) {
+ if (pos_ >= tokens_.size())
+ tokenize_one();
+ if (pos_ + 1 >= tokens_.size())
tokenize_one();
- if (pos_ + 1 >= tokens_.size())
- tokenize_one();
- }
return pos_ + 1 < tokens_.size() ? tokens_[pos_ + 1] : dummy;
}
putback();
res += '{' + verbatim_item() + '}';
} else
- res += t.cs();
+ res += t.asInput();
}
}
return res;