src/lyxlex_pimpl.C

   1 #include <config.h>
   2
   3 #include "lyxlex_pimpl.h"
   4 #include "debug.h"
   5
   6 #include "support/lyxalgo.h"
   7 #include "support/filetools.h"
   8 #include "support/lstrings.h"
   9
  10 #include <algorithm>
  11
  12 using std::sort;
  13 using std::ostream;
  14 using std::ios;
  15 using std::istream;
  16 using std::endl;
  17 using std::lower_bound;
  18
  19 // namespace {
  20 struct compare_tags {
  21         // used by lower_bound, sort and sorted
  22         inline
  23         int operator()(keyword_item const & a, keyword_item const & b) const {
  24                 // we use the ascii version, because in turkish, 'i'
  25                 // is not the lowercase version of 'I', and thus
  26                 // turkish locale breaks parsing of tags.
  27                 return compare_ascii_no_case(a.tag, b.tag) < 0;
  28         }
  29 };
  30 // } // end of anon namespace
  31
  32
  33 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
  34         : is(&fb__), table(tab), no_items(num),
  35           status(0), lineno(0), commentChar('#')
  36 {
  37         verifyTable();
  38 }
  39
  40
  41 string const LyXLex::Pimpl::getString() const
  42 {
  43         return string(buff);
  44 }
  45
  46
  47 void LyXLex::Pimpl::printError(string const & message) const
  48 {
  49         string const tmpmsg = subst(message, "$$Token", getString());
  50         lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
  51                << " of file " << MakeDisplayPath(name) << ']' << endl;
  52 }
  53
  54
  55 void LyXLex::Pimpl::printTable(ostream & os)
  56 {
  57         os << "\nNumber of tags: " << no_items << '\n';
  58         for (int i= 0; i < no_items; ++i)
  59                 os << "table[" << i
  60                    << "]:  tag: `" << table[i].tag
  61                    << "'  code:" << table[i].code << '\n';
  62         os.flush();
  63 }
  64
  65
  66 void LyXLex::Pimpl::verifyTable()
  67 {
  68         // Check if the table is sorted and if not, sort it.
  69         if (table
  70             && !lyx::sorted(table, table + no_items, compare_tags())) {
  71                 lyxerr << "The table passed to LyXLex is not sorted!\n"
  72                        << "Tell the developers to fix it!" << endl;
  73                 // We sort it anyway to avoid problems.
  74                 lyxerr << "\nUnsorted:\n";
  75                 printTable(lyxerr);
  76
  77                 sort(table, table + no_items, compare_tags());
  78                 lyxerr << "\nSorted:\n";
  79                 printTable(lyxerr);
  80         }
  81 }
  82
  83
  84 void LyXLex::Pimpl::pushTable(keyword_item * tab, int num)
  85 {
  86         pushed_table tmppu(table, no_items);
  87         pushed.push(tmppu);
  88
  89         table = tab;
  90         no_items = num;
  91
  92         verifyTable();
  93 }
  94
  95
  96 void LyXLex::Pimpl::popTable()
  97 {
  98         if (pushed.empty()) {
  99                 lyxerr << "LyXLex error: nothing to pop!" << endl;
 100                 return;
 101         }
 102
 103         pushed_table tmp = pushed.top();
 104         pushed.pop();
 105         table = tmp.table_elem;
 106         no_items = tmp.table_siz;
 107 }
 108
 109
 110 bool LyXLex::Pimpl::setFile(string const & filename)
 111 {
 112         // The check only outputs a debug message, because it triggers
 113         // a bug in compaq cxx 6.2, where is_open() returns 'true' for a
 114         // fresh new filebuf.  (JMarc)
 115         if (fb__.is_open() || is.tellg() > 0)
 116                 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
 117                         "file or stream already set." << endl;
 118         fb__.open(filename.c_str(), ios::in);
 119         is.rdbuf(&fb__);
 120         name = filename;
 121         lineno = 0;
 122         return fb__.is_open() && is.good();
 123 }
 124
 125
 126 void LyXLex::Pimpl::setStream(istream & i)
 127 {
 128         if (fb__.is_open() || is.tellg() > 0)
 129                 lyxerr[Debug::LYXLEX]  << "Error in LyXLex::setStream: "
 130                         "file or stream already set." << endl;
 131         is.rdbuf(i.rdbuf());
 132         lineno = 0;
 133 }
 134
 135 void LyXLex::Pimpl::setCommentChar(char c)
 136 {
 137         commentChar = c;
 138 }
 139
 140
 141 bool LyXLex::Pimpl::next(bool esc /* = false */)
 142 {
 143         if (!pushTok.empty()) {
 144                 // There can have been a whole line pushed so
 145                 // we extract the first word and leaves the rest
 146                 // in pushTok. (Lgb)
 147                 if (pushTok.find(' ') != string::npos) {
 148                         string tmp;
 149                         pushTok = split(pushTok, tmp, ' ');
 150                         tmp.copy(buff, string::npos);
 151                         buff[tmp.length()] = '\0';
 152                         return true;
 153                 } else {
 154                         pushTok.copy(buff, string::npos);
 155                         buff[pushTok.length()] = '\0';
 156                         pushTok.erase();
 157                         return true;
 158                 }
 159         }
 160         if (!esc) {
 161                 unsigned char c = 0; // getc() returns an int
 162                 char cc = 0;
 163                 status = 0;
 164                 while (is && !status) {
 165                         is.get(cc);
 166                         c = cc;
 167                         if (c == commentChar) {
 168                                 // Read rest of line (fast :-)
 169                                 // That is not fast... (Lgb)
 170 #if 1
 171                                 is.getline(buff, sizeof(buff));
 172                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 173                                                       << buff << '\'' << endl;
 174 #else
 175                                 // unfortunately ignore is buggy (Lgb)
 176                                 is.ignore(100, '\n');
 177 #endif
 178                                 ++lineno;
 179                                 continue;
 180                         }
 181
 182                         if (c == '\"') {
 183                                 int i = -1;
 184                                 do {
 185                                         is.get(cc);
 186                                         c = cc;
 187                                         if (c != '\r')
 188                                                 buff[++i] = c;
 189                                 } while (c != '\"' && c != '\n' && is &&
 190                                          i != (LEX_MAX_BUFF - 2));
 191
 192                                 if (i == (LEX_MAX_BUFF - 2)) {
 193                                         printError("Line too long");
 194                                         c = '\"'; // Pretend we got a "
 195                                         ++i;
 196                                 }
 197
 198                                 if (c != '\"') {
 199                                         printError("Missing quote");
 200                                         if (c == '\n')
 201                                                 ++lineno;
 202                                 }
 203
 204                                 buff[i] = '\0';
 205                                 status = LEX_DATA;
 206                                 break;
 207                         }
 208
 209                         if (c == ',')
 210                                 continue;              /* Skip ','s */
 211
 212                                 // using relational operators with chars other
 213                                 // than == and != is not safe. And if it is done
 214                                 // the type _have_ to be unsigned. It usually a
 215                                 // lot better to use the functions from cctype
 216                         if (c > ' ' && is)  {
 217                                 int i = 0;
 218                                 do {
 219                                         buff[i++] = c;
 220                                         is.get(cc);
 221                                         c = cc;
 222                                 } while (c > ' ' && c != ',' && is
 223                                          && (i != LEX_MAX_BUFF - 1));
 224                                 if (i == LEX_MAX_BUFF - 1) {
 225                                         printError("Line too long");
 226                                 }
 227                                 buff[i] = '\0';
 228                                 status = LEX_TOKEN;
 229                         }
 230
 231                         if (c == '\r' && is) {
 232                                 // The Windows support has lead to the
 233                                 // possibility of "\r\n" at the end of
 234                                 // a line.  This will stop LyX choking
 235                                 // when it expected to find a '\n'
 236                                 is.get(cc);
 237                                 c = cc;
 238                         }
 239
 240                         if (c == '\n')
 241                                 ++lineno;
 242
 243                 }
 244                 if (status) return true;
 245
 246                 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 247                 buff[0] = '\0';
 248                 return false;
 249         } else {
 250                 unsigned char c = 0; // getc() returns an int
 251                 char cc = 0;
 252
 253                 status = 0;
 254                 while (is && !status) {
 255                         is.get(cc);
 256                         c = cc;
 257
 258                         // skip ','s
 259                         if (c == ',') continue;
 260
 261                         if (c == '\\') {
 262                                 // escape
 263                                 int i = 0;
 264                                 do {
 265                                         if (c == '\\') {
 266                                                 // escape the next char
 267                                                 is.get(cc);
 268                                                 c = cc;
 269                                         }
 270                                         buff[i++] = c;
 271                                         is.get(cc);
 272                                         c = cc;
 273                                 } while (c > ' ' && c != ',' && is
 274                                          && (i != LEX_MAX_BUFF - 1));
 275                                 if (i == LEX_MAX_BUFF - 1) {
 276                                         printError("Line too long");
 277                                 }
 278                                 buff[i] = '\0';
 279                                 status = LEX_TOKEN;
 280                                 continue;
 281                         }
 282
 283                         if (c == commentChar) {
 284                                 // Read rest of line (fast :-)
 285                                 // That is still not fast... (Lgb)
 286 #if 1
 287                                 is.getline(buff, sizeof(buff));
 288                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 289                                                       << buff << '\'' << endl;
 290 #else
 291                                 // but ignore is also still buggy (Lgb)
 292                                 // This is fast (Lgb)
 293                                 is.ignore(100, '\n');
 294 #endif
 295                                 ++lineno;
 296                                 continue;
 297                         }
 298
 299                         // string
 300                         if (c == '\"') {
 301                                 int i = -1;
 302                                 bool escaped = false;
 303                                 do {
 304                                         escaped = false;
 305                                         is.get(cc);
 306                                         c = cc;
 307                                         if (c == '\r') continue;
 308                                         if (c == '\\') {
 309                                                 // escape the next char
 310                                                 is.get(cc);
 311                                                 c = cc;
 312                                                 if (c == '\"' || c == '\\')
 313                                                         escaped = true;
 314                                                 else
 315                                                         buff[++i] = '\\';
 316                                         }
 317                                         buff[++i] = c;
 318
 319                                         if (!escaped && c == '\"') break;
 320                                 } while (c != '\n' && is &&
 321                                          i != (LEX_MAX_BUFF - 2));
 322
 323                                 if (i == (LEX_MAX_BUFF - 2)) {
 324                                         printError("Line too long");
 325                                         c = '\"'; // Pretend we got a "
 326                                         ++i;
 327                                 }
 328
 329                                 if (c != '\"') {
 330                                         printError("Missing quote");
 331                                         if (c == '\n')
 332                                                 ++lineno;
 333                                 }
 334
 335                                 buff[i] = '\0';
 336                                 status = LEX_DATA;
 337                                 break;
 338                         }
 339
 340                         if (c > ' ' && is) {
 341                                 int i = 0;
 342                                 do {
 343                                         if (c == '\\') {
 344                                                 // escape the next char
 345                                                 is.get(cc);
 346                                                 c = cc;
 347                                                 //escaped = true;
 348                                         }
 349                                         buff[i++] = c;
 350                                         is.get(cc);
 351                                         c = cc;
 352                                 } while (c > ' ' && c != ',' && is
 353                                          && (i != LEX_MAX_BUFF - 1));
 354                                 if (i == LEX_MAX_BUFF - 1) {
 355                                         printError("Line too long");
 356                                 }
 357                                 buff[i] = '\0';
 358                                 status = LEX_TOKEN;
 359                         }
 360                         // new line
 361                         if (c == '\n')
 362                                 ++lineno;
 363                 }
 364
 365                 if (status) return true;
 366
 367                 status = is.eof() ? LEX_FEOF : LEX_UNDEF;
 368                 buff[0] = '\0';
 369                 return false;
 370         }
 371 }
 372
 373
 374 int LyXLex::Pimpl::search_kw(char const * const tag) const
 375 {
 376         keyword_item search_tag = { tag, 0 };
 377         keyword_item * res =
 378                 lower_bound(table, table + no_items,
 379                             search_tag, compare_tags());
 380         // use the compare_ascii_no_case instead of compare_no_case,
 381         // because in turkish, 'i' is not the lowercase version of 'I',
 382         // and thus turkish locale breaks parsing of tags.
 383         if (res != table + no_items
 384             && !compare_ascii_no_case(res->tag, tag))
 385                 return res->code;
 386         return LEX_UNDEF;
 387 }
 388
 389
 390 int LyXLex::Pimpl::lex()
 391 {
 392         //NOTE: possible bug.
 393         if (next() && status == LEX_TOKEN)
 394                 return search_kw(buff);
 395         else
 396                 return status;
 397 }
 398
 399
 400 bool LyXLex::Pimpl::eatLine()
 401 {
 402         int i = 0;
 403         unsigned char c = '\0';
 404         char cc = 0;
 405         while (is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
 406                 is.get(cc);
 407                 c = cc;
 408                 //lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
 409                 //                    << c << '\'' << endl;
 410                 if (c != '\r')
 411                         buff[i++] = c;
 412         }
 413         if (i == (LEX_MAX_BUFF - 1) && c != '\n') {
 414                 printError("Line too long");
 415                 c = '\n'; // Pretend we had an end of line
 416                 --lineno; // but don't increase line counter (netto effect)
 417                 ++i; // and preserve last character read.
 418         }
 419         if (c == '\n') {
 420                 ++lineno;
 421                 buff[--i] = '\0'; // i can never be 0 here, so no danger
 422                 status = LEX_DATA;
 423                 return true;
 424         } else {
 425                 buff[i] = '\0';
 426                 return false;
 427         }
 428 }
 429
 430
 431 bool LyXLex::Pimpl::nextToken()
 432 {
 433         if (!pushTok.empty()) {
 434                 // There can have been a whole line pushed so
 435                 // we extract the first word and leaves the rest
 436                 // in pushTok. (Lgb)
 437                 if (pushTok.find(' ') != string::npos) {
 438                         string tmp;
 439                         pushTok = split(pushTok, tmp, ' ');
 440                         tmp.copy(buff, string::npos);
 441                         buff[tmp.length()] = '\0';
 442                         return true;
 443                 } else {
 444                         pushTok.copy(buff, string::npos);
 445                         buff[pushTok.length()] = '\0';
 446                         pushTok.erase();
 447                         return true;
 448                 }
 449         }
 450
 451         status = 0;
 452         while (is && !status) {
 453                 unsigned char c = 0;
 454                 char cc = 0;
 455                 is.get(cc);
 456                 c = cc;
 457                 if (c >= ' ' && is) {
 458                         int i = 0;
 459                         if (c == '\\') { // first char == '\\'
 460                                 do {
 461                                         buff[i++] = c;
 462                                         is.get(cc);
 463                                         c = cc;
 464                                 } while (c > ' ' && c != '\\' && is
 465                                          && i != (LEX_MAX_BUFF - 1));
 466                         } else {
 467                                 do {
 468                                         buff[i++] = c;
 469                                         is.get(cc);
 470                                         c = cc;
 471                                 } while (c >= ' ' && c != '\\' && is
 472                                          && i != (LEX_MAX_BUFF - 1));
 473                         }
 474
 475                         if (i == (LEX_MAX_BUFF - 1)) {
 476                                 printError("Line too long");
 477                         }
 478
 479                         if (c == '\\') is.putback(c); // put it back
 480                         buff[i] = '\0';
 481                         status = LEX_TOKEN;
 482                 }
 483
 484                 if (c == '\n')
 485                         ++lineno;
 486
 487         }
 488         if (status)  return true;
 489
 490         status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 491         buff[0] = '\0';
 492         return false;
 493 }
 494
 495
 496 void LyXLex::Pimpl::pushToken(string const & pt)
 497 {
 498         pushTok = pt;
 499 }