3 #include "lyxlex_pimpl.h"
6 #include "support/lyxalgo.h"
7 #include "support/filetools.h"
8 #include "support/lstrings.h"
17 using std::lower_bound;
21 // used by lower_bound, sort and sorted
23 int operator()(keyword_item const & a, keyword_item const & b) const {
24 // we use the ascii version, because in turkish, 'i'
25 // is not the lowercase version of 'I', and thus
26 // turkish locale breaks parsing of tags.
27 return compare_ascii_no_case(a.tag, b.tag) < 0;
30 // } // end of anon namespace
33 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
34 : is(&fb__), table(tab), no_items(num),
35 status(0), lineno(0), commentChar('#')
41 string const LyXLex::Pimpl::getString() const
47 void LyXLex::Pimpl::printError(string const & message) const
49 string const tmpmsg = subst(message, "$$Token", getString());
50 lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
51 << " of file " << MakeDisplayPath(name) << ']' << endl;
55 void LyXLex::Pimpl::printTable(ostream & os)
57 os << "\nNumber of tags: " << no_items << '\n';
58 for (int i= 0; i < no_items; ++i)
60 << "]: tag: `" << table[i].tag
61 << "' code:" << table[i].code << '\n';
66 void LyXLex::Pimpl::verifyTable()
68 // Check if the table is sorted and if not, sort it.
70 && !lyx::sorted(table, table + no_items, compare_tags())) {
71 lyxerr << "The table passed to LyXLex is not sorted!\n"
72 << "Tell the developers to fix it!" << endl;
73 // We sort it anyway to avoid problems.
74 lyxerr << "\nUnsorted:\n";
77 sort(table, table + no_items, compare_tags());
78 lyxerr << "\nSorted:\n";
84 void LyXLex::Pimpl::pushTable(keyword_item * tab, int num)
86 pushed_table tmppu(table, no_items);
96 void LyXLex::Pimpl::popTable()
99 lyxerr << "LyXLex error: nothing to pop!" << endl;
103 pushed_table tmp = pushed.top();
105 table = tmp.table_elem;
106 no_items = tmp.table_siz;
110 bool LyXLex::Pimpl::setFile(string const & filename)
112 // The check only outputs a debug message, because it triggers
113 // a bug in compaq cxx 6.2, where is_open() returns 'true' for a
114 // fresh new filebuf. (JMarc)
115 if (fb__.is_open() || is.tellg() > 0)
116 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
117 "file or stream already set." << endl;
118 fb__.open(filename.c_str(), ios::in);
122 return fb__.is_open() && is.good();
126 void LyXLex::Pimpl::setStream(istream & i)
128 if (fb__.is_open() || is.tellg() > 0)
129 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setStream: "
130 "file or stream already set." << endl;
135 void LyXLex::Pimpl::setCommentChar(char c)
141 bool LyXLex::Pimpl::next(bool esc /* = false */)
143 if (!pushTok.empty()) {
144 // There can have been a whole line pushed so
145 // we extract the first word and leaves the rest
147 if (pushTok.find(' ') != string::npos) {
149 pushTok = split(pushTok, tmp, ' ');
150 tmp.copy(buff, string::npos);
151 buff[tmp.length()] = '\0';
154 pushTok.copy(buff, string::npos);
155 buff[pushTok.length()] = '\0';
161 unsigned char c = 0; // getc() returns an int
164 while (is && !status) {
167 if (c == commentChar) {
168 // Read rest of line (fast :-)
169 // That is not fast... (Lgb)
171 is.getline(buff, sizeof(buff));
172 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
173 << buff << '\'' << endl;
175 // unfortunately ignore is buggy (Lgb)
176 is.ignore(100, '\n');
189 } while (c != '\"' && c != '\n' && is &&
190 i != (LEX_MAX_BUFF - 2));
192 if (i == (LEX_MAX_BUFF - 2)) {
193 printError("Line too long");
194 c = '\"'; // Pretend we got a "
199 printError("Missing quote");
210 continue; /* Skip ','s */
212 // using relational operators with chars other
213 // than == and != is not safe. And if it is done
214 // the type _have_ to be unsigned. It usually a
215 // lot better to use the functions from cctype
222 } while (c > ' ' && c != ',' && is
223 && (i != LEX_MAX_BUFF - 1));
224 if (i == LEX_MAX_BUFF - 1) {
225 printError("Line too long");
231 if (c == '\r' && is) {
232 // The Windows support has lead to the
233 // possibility of "\r\n" at the end of
234 // a line. This will stop LyX choking
235 // when it expected to find a '\n'
244 if (status) return true;
246 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
250 unsigned char c = 0; // getc() returns an int
254 while (is && !status) {
259 if (c == ',') continue;
266 // escape the next char
273 } while (c > ' ' && c != ',' && is
274 && (i != LEX_MAX_BUFF - 1));
275 if (i == LEX_MAX_BUFF - 1) {
276 printError("Line too long");
283 if (c == commentChar) {
284 // Read rest of line (fast :-)
285 // That is still not fast... (Lgb)
287 is.getline(buff, sizeof(buff));
288 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
289 << buff << '\'' << endl;
291 // but ignore is also still buggy (Lgb)
292 // This is fast (Lgb)
293 is.ignore(100, '\n');
302 bool escaped = false;
307 if (c == '\r') continue;
309 // escape the next char
312 if (c == '\"' || c == '\\')
319 if (!escaped && c == '\"') break;
320 } while (c != '\n' && is &&
321 i != (LEX_MAX_BUFF - 2));
323 if (i == (LEX_MAX_BUFF - 2)) {
324 printError("Line too long");
325 c = '\"'; // Pretend we got a "
330 printError("Missing quote");
344 // escape the next char
352 } while (c > ' ' && c != ',' && is
353 && (i != LEX_MAX_BUFF - 1));
354 if (i == LEX_MAX_BUFF - 1) {
355 printError("Line too long");
365 if (status) return true;
367 status = is.eof() ? LEX_FEOF : LEX_UNDEF;
374 int LyXLex::Pimpl::search_kw(char const * const tag) const
376 keyword_item search_tag = { tag, 0 };
378 lower_bound(table, table + no_items,
379 search_tag, compare_tags());
380 // use the compare_ascii_no_case instead of compare_no_case,
381 // because in turkish, 'i' is not the lowercase version of 'I',
382 // and thus turkish locale breaks parsing of tags.
383 if (res != table + no_items
384 && !compare_ascii_no_case(res->tag, tag))
390 int LyXLex::Pimpl::lex()
392 //NOTE: possible bug.
393 if (next() && status == LEX_TOKEN)
394 return search_kw(buff);
400 bool LyXLex::Pimpl::eatLine()
403 unsigned char c = '\0';
405 while (is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
408 //lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
409 // << c << '\'' << endl;
413 if (i == (LEX_MAX_BUFF - 1) && c != '\n') {
414 printError("Line too long");
415 c = '\n'; // Pretend we had an end of line
416 --lineno; // but don't increase line counter (netto effect)
417 ++i; // and preserve last character read.
421 buff[--i] = '\0'; // i can never be 0 here, so no danger
431 bool LyXLex::Pimpl::nextToken()
433 if (!pushTok.empty()) {
434 // There can have been a whole line pushed so
435 // we extract the first word and leaves the rest
437 if (pushTok.find(' ') != string::npos) {
439 pushTok = split(pushTok, tmp, ' ');
440 tmp.copy(buff, string::npos);
441 buff[tmp.length()] = '\0';
444 pushTok.copy(buff, string::npos);
445 buff[pushTok.length()] = '\0';
452 while (is && !status) {
457 if (c >= ' ' && is) {
459 if (c == '\\') { // first char == '\\'
464 } while (c > ' ' && c != '\\' && is
465 && i != (LEX_MAX_BUFF - 1));
471 } while (c >= ' ' && c != '\\' && is
472 && i != (LEX_MAX_BUFF - 1));
475 if (i == (LEX_MAX_BUFF - 1)) {
476 printError("Line too long");
479 if (c == '\\') is.putback(c); // put it back
488 if (status) return true;
490 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
496 void LyXLex::Pimpl::pushToken(string const & pt)