-#include "parser.h"
+/**
+ * \file texparser.C
+ * This file is part of LyX, the document processor.
+ * Licence details can be found in the file COPYING.
+ *
+ * \author André Pönitz
+ *
+ * Full author contact details are available in file CREDITS.
+ */
+
+#include <config.h>
+
+#include "texparser.h"
+
+#include <iostream>
+#include <sstream>
using std::cerr;
using std::endl;
using std::fill;
-using std::ios;
using std::istream;
using std::istringstream;
using std::ostream;
using std::string;
-//
+namespace {
+
+CatCode theCatcode[256];
+
+void skipSpaceTokens(istream & is, char c)
+{
+ // skip trailing spaces
+ while (catcode(c) == catSpace || catcode(c) == catNewline)
+ if (!is.get(c))
+ break;
+ //cerr << "putting back: " << c << "\n";
+ is.putback(c);
+}
+
+
+void catInit()
+{
+ fill(theCatcode, theCatcode + 256, catOther);
+ fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
+ fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
+
+ theCatcode[int('\\')] = catEscape;
+ theCatcode[int('{')] = catBegin;
+ theCatcode[int('}')] = catEnd;
+ theCatcode[int('$')] = catMath;
+ theCatcode[int('&')] = catAlign;
+ theCatcode[10] = catNewline;
+ theCatcode[int('#')] = catParameter;
+ theCatcode[int('^')] = catSuper;
+ theCatcode[int('_')] = catSub;
+ theCatcode[0x7f] = catIgnore;
+ theCatcode[int(' ')] = catSpace;
+ theCatcode[int('\t')] = catSpace;
+ theCatcode[13] = catIgnore;
+ theCatcode[int('~')] = catActive;
+ theCatcode[int('%')] = catComment;
+
+ // This is wrong!
+ theCatcode[int('@')] = catLetter;
+}
+
+}
+
+
+//
// catcodes
//
}
-CatCode theCatcode[256];
-
-
CatCode catcode(unsigned char c)
{
return theCatcode[c];
}
-void catInit()
-{
- fill(theCatcode, theCatcode + 256, catOther);
- fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
- fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
-
- theCatcode['\\'] = catEscape;
- theCatcode['{'] = catBegin;
- theCatcode['}'] = catEnd;
- theCatcode['$'] = cat;
- theCatcode['&'] = catAlign;
- theCatcode['\n'] = catNewline;
- theCatcode['#'] = catParameter;
- theCatcode['^'] = catSuper;
- theCatcode['_'] = catSub;
- theCatcode['\7f'] = catIgnore;
- theCatcode[' '] = catSpace;
- theCatcode['\t'] = catSpace;
- theCatcode['\r'] = catNewline;
- theCatcode['~'] = catActive;
- theCatcode['%'] = catComment;
-}
-
//
// Token
ostream & operator<<(ostream & os, Token const & t)
{
if (t.cs().size())
- os << '\\' << t.cs();
+ os << '\\' << t.cs() << ' ';
+ else if (t.cat() == catLetter)
+ os << t.character();
+ else if (t.cat() == catNewline)
+ os << "[\\n," << t.cat() << "]\n";
else
os << '[' << t.character() << ',' << t.cat() << ']';
return os;
}
+string Token::asString() const
+{
+ return cs_.size() ? cs_ : string(1, char_);
+}
+
+
+string Token::asInput() const
+{
+ return char_ ? string(1, char_) : '\\' + cs_ + ' ';
+}
+
+
//
// Parser
//
}
+Parser::Parser(string const & s)
+ : lineno_(0), pos_(0)
+{
+ istringstream is(s);
+ tokenize(is);
+}
+
+
void Parser::push_back(Token const & t)
{
tokens_.push_back(t);
}
-Token const & Parser::prevToken() const
+Token const & Parser::prev_token() const
{
static const Token dummy;
return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
}
-Token const & Parser::nextToken() const
+Token const & Parser::next_token() const
{
static const Token dummy;
return good() ? tokens_[pos_] : dummy;
}
-Token const & Parser::getToken()
+Token const & Parser::get_token()
{
static const Token dummy;
//cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
}
-void Parser::skipSpaces()
+void Parser::skip_spaces()
{
- while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
- getToken();
+ while (1) {
+ if (next_token().cat() == catSpace || next_token().cat() == catNewline)
+ get_token();
+ else if (next_token().cat() == catComment)
+ while (next_token().cat() != catNewline)
+ get_token();
+ else
+ break;
+ }
}
string Parser::getArg(char left, char right)
{
- skipSpaces();
+ skip_spaces();
string result;
char c = getChar();
}
-void Parser::skipSpaceTokens(istream & is, char c)
+string Parser::getOpt()
{
- // skip trailing spaces
- while (catcode(c) == catSpace || catcode(c) == catNewline)
- if (!is.get(c))
- break;
- //cerr << "putting back: " << c << "\n";
- is.putback(c);
+ string const res = getArg('[', ']');
+ return res.size() ? '[' + res + ']' : string();
}
void Parser::tokenize(istream & is)
-{
- // eat everything up to the next \end_inset or end of stream
- // and store it in s for further tokenization
- string s;
- char c;
- while (is.get(c)) {
- s += c;
- if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
- s = s.substr(0, s.size() - 10);
- break;
- }
- }
- // Remove the space after \end_inset
- if (is.get(c) && c != ' ')
- is.unget();
-
- // tokenize buffer
- tokenize(s);
-}
-
-
-void Parser::tokenize(string const & buffer)
{
static bool init_done = false;
init_done = true;
}
- istringstream is(buffer.c_str(), ios::in | ios::binary);
-
char c;
while (is.get(c)) {
//cerr << "reading c: " << c << "\n";
case catNewline: {
++lineno_;
is.get(c);
- if (catcode(c) == catNewline)
+ if (catcode(c) == catNewline) {
+ //do {
+ is.get(c);
+ //} while (catcode(c) == catNewline);
push_back(Token("par"));
- else {
+ } else {
push_back(Token('\n', catNewline));
- is.putback(c);
}
+ is.putback(c);
break;
}
-/*
case catComment: {
+ push_back(Token(c, catComment));
while (is.get(c) && catcode(c) != catNewline)
- ;
+ push_back(Token(c, catLetter));
+ push_back(Token(c, catNewline));
++lineno_;
+ is.get(c);
+ if (catcode(c) == catNewline) {
+ push_back(Token("par"));
+ ++lineno_;
+ } else {
+ is.putback(c);
+ }
break;
}
-*/
case catEscape: {
is.get(c);
}
case catIgnore: {
- cerr << "ignoring a char: " << int(c) << "\n";
+ if (c != 13)
+ cerr << "ignoring a char: " << int(c) << "\n";
break;
}
push_back(Token(c, catcode(c)));
}
}
-
-#ifdef FILEDEBUG
- dump();
-#endif
}
string Parser::verbatimOption()
{
string res;
- if (nextToken().character() == '[') {
- Token t = getToken();
- for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
+ if (next_token().character() == '[') {
+ Token t = get_token();
+ for (Token t = get_token(); t.character() != ']' && good(); t = get_token()) {
if (t.cat() == catBegin) {
putback();
- res += '{' + verbatimItem() + '}';
+ res += '{' + verbatim_item() + '}';
} else
res += t.asString();
}
}
-string Parser::verbatimItem()
+string Parser::verbatim_item()
{
- string res;
- if (nextToken().cat() == catBegin) {
- Token t = getToken();
- for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
+ if (!good())
+ error("stream bad");
+ skip_spaces();
+ if (next_token().cat() == catBegin) {
+ Token t = get_token(); // skip brace
+ string res;
+ for (Token t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
if (t.cat() == catBegin) {
putback();
- res += '{' + verbatimItem() + '}';
+ res += '{' + verbatim_item() + '}';
}
else
- res += t.asString();
+ res += t.asInput();
}
+ return res;
}
- return res;
+ return get_token().asInput();
+}
+
+
+void Parser::setCatCode(char c, CatCode cat)
+{
+ theCatcode[(unsigned char)c] = cat;
+}
+
+
+CatCode Parser::getCatCode(char c) const
+{
+ return theCatcode[(unsigned char)c];
}
projects / lyx.git / blobdiff