src/lyxlex_pimpl.C

   1 #include <config.h>
   2
   3 #ifdef __GNUG__
   4 #pragma implementation
   5 #endif
   6 #include <algorithm>
   7
   8 #include "lyxlex_pimpl.h"
   9 #include "support/lyxalgo.h"
  10 #include "support/filetools.h"
  11 #include "debug.h"
  12 #if 1 // to get NEW_INSETS
  13 #include "lyxparagraph.h"
  14 #endif
  15
  16 using std::sort;
  17 using std::ostream;
  18 using std::ios;
  19 using std::istream;
  20 using std::endl;
  21 using std::lower_bound;
  22
  23 // namespace {
  24 struct compare_tags {
  25         // used by lower_bound, sort and sorted
  26         inline
  27         int operator()(keyword_item const & a, keyword_item const & b) const {
  28                 return compare_no_case(a.tag, b.tag) < 0;
  29         }
  30 };
  31 // } // end of anon namespace
  32
  33
  34 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
  35         : is(&fb__), table(tab), no_items(num),
  36           status(0), lineno(0), commentChar('#')
  37 {
  38         verifyTable();
  39 }
  40
  41
  42 string const LyXLex::Pimpl::GetString() const
  43 {
  44         return string(buff);
  45 }
  46
  47
  48 void LyXLex::Pimpl::printError(string const & message) const
  49 {
  50         string tmpmsg = subst(message, "$$Token", GetString());
  51         lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
  52                << " of file " << MakeDisplayPath(name) << ']' << endl;
  53 }
  54
  55
  56 void LyXLex::Pimpl::printTable(ostream & os)
  57 {
  58         os << "\nNumber of tags: " << no_items << '\n';
  59         for (int i= 0; i < no_items; ++i)
  60                 os << "table[" << i
  61                    << "]:  tag: `" << table[i].tag
  62                    << "'  code:" << table[i].code << '\n';
  63         os.flush();
  64 }
  65
  66
  67 void LyXLex::Pimpl::verifyTable()
  68 {
  69         // Check if the table is sorted and if not, sort it.
  70         if (table
  71             && !sorted(table, table + no_items, compare_tags())) {
  72                 lyxerr << "The table passed to LyXLex is not sorted!\n"
  73                        << "Tell the developers to fix it!" << endl;
  74                 // We sort it anyway to avoid problems.
  75                 lyxerr << "\nUnsorted:\n";
  76                 printTable(lyxerr);
  77
  78                 sort(table, table + no_items, compare_tags());
  79                 lyxerr << "\nSorted:\n";
  80                 printTable(lyxerr);
  81         }
  82 }
  83
  84
  85 void LyXLex::Pimpl::pushTable(keyword_item * tab, int num)
  86 {
  87         pushed_table tmppu(table, no_items);
  88         pushed.push(tmppu);
  89
  90         table = tab;
  91         no_items = num;
  92
  93         verifyTable();
  94 }
  95
  96
  97 void LyXLex::Pimpl::popTable()
  98 {
  99         if (pushed.empty()) {
 100                 lyxerr << "LyXLex error: nothing to pop!" << endl;
 101                 return;
 102         }
 103
 104         pushed_table tmp = pushed.top();
 105         pushed.pop();
 106         table = tmp.table_elem;
 107         no_items = tmp.table_siz;
 108 }
 109
 110
 111 bool LyXLex::Pimpl::setFile(string const & filename)
 112 {
 113         // The check only outputs a debug message, because it triggers
 114         // a bug in compaq cxx 6.2, where is_open() returns 'true' for a
 115         // fresh new filebuf.  (JMarc)
 116         if (fb__.is_open() || is.tellg() > 0)
 117                 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
 118                         "file or stream already set." << endl;
 119         fb__.open(filename.c_str(), ios::in);
 120         is.rdbuf(&fb__);
 121         name = filename;
 122         lineno = 0;
 123         return fb__.is_open() && is.good();
 124 }
 125
 126
 127 void LyXLex::Pimpl::setStream(istream & i)
 128 {
 129         if (fb__.is_open() || is.tellg() > 0)
 130                 lyxerr[Debug::LYXLEX]  << "Error in LyXLex::setStream: "
 131                         "file or stream already set." << endl;
 132         is.rdbuf(i.rdbuf());
 133         lineno = 0;
 134 }
 135
 136 void LyXLex::Pimpl::setCommentChar(char c)
 137 {
 138         commentChar = c;
 139 }
 140
 141
 142 bool LyXLex::Pimpl::next(bool esc /* = false */)
 143 {
 144         if (!pushTok.empty()) {
 145 #ifndef NEW_INSETS
 146                 pushTok.copy(buff, string::npos);
 147                 buff[pushTok.length()] = '\0';
 148                 pushTok.erase();
 149                 return true;
 150 #else
 151                 // There can have been a whole line pushed so
 152                 // we extract the first word and leaves the rest
 153                 // in pushTok. (Lgb)
 154                 if (pushTok.find(' ') != string::npos) {
 155                         string tmp;
 156                         pushTok = split(pushTok, tmp, ' ');
 157                         tmp.copy(buff, string::npos);
 158                         buff[tmp.length()] = '\0';
 159                         return true;
 160                 } else {
 161                         pushTok.copy(buff, string::npos);
 162                         buff[pushTok.length()] = '\0';
 163                         pushTok.erase();
 164                         return true;
 165                 }
 166 #endif
 167         }
 168         if (!esc) {
 169                 unsigned char c = 0; // getc() returns an int
 170                 char cc = 0;
 171                 status = 0;
 172                 while (is && !status) {
 173                         is.get(cc);
 174                         c = cc;
 175                         if (c == commentChar) {
 176                                 // Read rest of line (fast :-)
 177                                 // That is not fast... (Lgb)
 178 #if 1
 179                                 is.getline(buff, sizeof(buff));
 180                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 181                                                       << buff << "'" << endl;
 182 #else
 183                                 // unfortunately is ignore buggy (Lgb)
 184                                 is.ignore(100, '\n');
 185 #endif
 186                                 ++lineno;
 187                                 continue;
 188                         }
 189
 190                         if (c == '\"') {
 191                                 int i = -1;
 192                                 do {
 193                                         is.get(cc);
 194                                         c = cc;
 195                                         if (c != '\r')
 196                                                 buff[++i] = c;
 197                                 } while (c != '\"' && c != '\n' && is &&
 198                                          i != (LEX_MAX_BUFF - 2));
 199
 200                                 if (i == (LEX_MAX_BUFF - 2)) {
 201                                         printError("Line too long");
 202                                         c = '\"'; // Pretend we got a "
 203                                         ++i;
 204                                 }
 205
 206                                 if (c != '\"') {
 207                                         printError("Missing quote");
 208                                         if (c == '\n')
 209                                                 ++lineno;
 210                                 }
 211
 212                                 buff[i] = '\0';
 213                                 status = LEX_DATA;
 214                                 break;
 215                         }
 216
 217                         if (c == ',')
 218                                 continue;              /* Skip ','s */
 219
 220                                 // using relational operators with chars other
 221                                 // than == and != is not safe. And if it is done
 222                                 // the type _have_ to be unsigned. It usually a
 223                                 // lot better to use the functions from cctype
 224                         if (c > ' ' && is)  {
 225                                 int i = 0;
 226                                 do {
 227                                         buff[i++] = c;
 228                                         is.get(cc);
 229                                         c = cc;
 230                                 } while (c > ' ' && c != ',' && is
 231                                          && (i != LEX_MAX_BUFF - 1) );
 232                                 if (i == LEX_MAX_BUFF - 1) {
 233                                         printError("Line too long");
 234                                 }
 235                                 buff[i] = '\0';
 236                                 status = LEX_TOKEN;
 237                         }
 238
 239                         if (c == '\r' && is) {
 240                                 // The Windows support has lead to the
 241                                 // possibility of "\r\n" at the end of
 242                                 // a line.  This will stop LyX choking
 243                                 // when it expected to find a '\n'
 244                                 is.get(cc);
 245                                 c = cc;
 246                         }
 247
 248                         if (c == '\n')
 249                                 ++lineno;
 250
 251                 }
 252                 if (status) return true;
 253
 254                 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 255                 buff[0] = '\0';
 256                 return false;
 257         } else {
 258                 unsigned char c = 0; // getc() returns an int
 259                 char cc = 0;
 260
 261                 status = 0;
 262                 while (is && !status) {
 263                         is.get(cc);
 264                         c = cc;
 265
 266                         // skip ','s
 267                         if (c == ',') continue;
 268
 269                         if (c == '\\') {
 270                                 // escape
 271                                 int i = 0;
 272                                 do {
 273                                         if (c == '\\') {
 274                                                 // escape the next char
 275                                                 is.get(cc);
 276                                                 c = cc;
 277                                         }
 278                                         buff[i++] = c;
 279                                         is.get(cc);
 280                                         c = cc;
 281                                 } while (c > ' ' && c != ',' && is
 282                                          && (i != LEX_MAX_BUFF - 1) );
 283                                 if (i == LEX_MAX_BUFF - 1) {
 284                                         printError("Line too long");
 285                                 }
 286                                 buff[i] = '\0';
 287                                 status = LEX_TOKEN;
 288                                 continue;
 289                         }
 290
 291                         if (c == commentChar) {
 292                                 // Read rest of line (fast :-)
 293                                 // That is still not fast... (Lgb)
 294 #if 1
 295                                 is.getline(buff, sizeof(buff));
 296                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 297                                                       << buff << "'" << endl;
 298 #else
 299                                 // but ignore is also still buggy (Lgb)
 300                                 // This is fast (Lgb)
 301                                 is.ignore(100, '\n');
 302 #endif
 303                                 ++lineno;
 304                                 continue;
 305                         }
 306
 307                         // string
 308                         if (c == '\"') {
 309                                 int i = -1;
 310                                 bool escaped = false;
 311                                 do {
 312                                         escaped = false;
 313                                         is.get(cc);
 314                                         c = cc;
 315                                         if (c == '\r') continue;
 316                                         if (c == '\\') {
 317                                                 // escape the next char
 318                                                 is.get(cc);
 319                                                 c = cc;
 320                                                 escaped = true;
 321                                         }
 322                                         buff[++i] = c;
 323
 324                                         if (!escaped && c == '\"') break;
 325                                 } while (c != '\n' && is &&
 326                                          i != (LEX_MAX_BUFF - 2));
 327
 328                                 if (i == (LEX_MAX_BUFF - 2)) {
 329                                         printError("Line too long");
 330                                         c = '\"'; // Pretend we got a "
 331                                         ++i;
 332                                 }
 333
 334                                 if (c != '\"') {
 335                                         printError("Missing quote");
 336                                         if (c == '\n')
 337                                                 ++lineno;
 338                                 }
 339
 340                                 buff[i] = '\0';
 341                                 status = LEX_DATA;
 342                                 break;
 343                         }
 344
 345                         if (c > ' ' && is) {
 346                                 int i = 0;
 347                                 do {
 348                                         if (c == '\\') {
 349                                                 // escape the next char
 350                                                 is.get(cc);
 351                                                 c = cc;
 352                                                 //escaped = true;
 353                                         }
 354                                         buff[i++] = c;
 355                                         is.get(cc);
 356                                         c = cc;
 357                                 } while (c > ' ' && c != ',' && is
 358                                          && (i != LEX_MAX_BUFF-1) );
 359                                 if (i == LEX_MAX_BUFF-1) {
 360                                         printError("Line too long");
 361                                 }
 362                                 buff[i] = '\0';
 363                                 status = LEX_TOKEN;
 364                         }
 365                         // new line
 366                         if (c == '\n')
 367                                 ++lineno;
 368                 }
 369
 370                 if (status) return true;
 371
 372                 status = is.eof() ? LEX_FEOF : LEX_UNDEF;
 373                 buff[0] = '\0';
 374                 return false;
 375         }
 376 }
 377
 378
 379 int LyXLex::Pimpl::search_kw(char const * const tag) const
 380 {
 381         keyword_item search_tag = { tag, 0 };
 382         keyword_item * res =
 383                 lower_bound(table, table + no_items,
 384                             search_tag, compare_tags());
 385         if (res != table + no_items
 386             && !compare_no_case(res->tag, tag))
 387                 return res->code;
 388         return LEX_UNDEF;
 389 }
 390
 391
 392 int LyXLex::Pimpl::lex()
 393 {
 394         //NOTE: possible bug.
 395         if (next() && status == LEX_TOKEN)
 396                 return search_kw(buff);
 397         else
 398                 return status;
 399 }
 400
 401
 402 bool LyXLex::Pimpl::EatLine()
 403 {
 404 #ifndef NEW_INSETS
 405         // This is not handling the pushed token
 406         if (!pushTok.empty()) {
 407                 pushTok.copy(buff, string::npos);
 408                 buff[pushTok.length()] = '\0';
 409                 pushTok.erase();
 410                 return true;
 411         }
 412 #endif
 413         int i = 0;
 414         unsigned char c = '\0';
 415         char cc = 0;
 416         while(is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
 417                 is.get(cc);
 418                 c = cc;
 419                 lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
 420                                       << c << "'" << endl;
 421                 if (c != '\r')
 422                         buff[i++] = c;
 423         }
 424         if (i == (LEX_MAX_BUFF - 1) && c != '\n') {
 425                 printError("Line too long");
 426                 c = '\n'; // Pretend we had an end of line
 427                 --lineno; // but don't increase line counter (netto effect)
 428                 ++i; // and preserve last character read.
 429         }
 430         if (c == '\n') {
 431                 ++lineno;
 432                 buff[--i] = '\0'; // i can never be 0 here, so no danger
 433                 status = LEX_DATA;
 434                 return true;
 435         } else {
 436                 buff[i] = '\0';
 437                 return false;
 438         }
 439 }
 440
 441
 442 bool LyXLex::Pimpl::nextToken()
 443 {
 444         if (!pushTok.empty()) {
 445 #ifndef NEW_INSETS
 446                 pushTok.copy(buff, string::npos);
 447                 buff[pushTok.length()] = '\0';
 448                 pushTok.erase();
 449                 return true;
 450 #else
 451                 // There can have been a whole line pushed so
 452                 // we extract the first word and leaves the rest
 453                 // in pushTok. (Lgb)
 454                 if (pushTok.find(' ') != string::npos) {
 455                         string tmp;
 456                         pushTok = split(pushTok, tmp, ' ');
 457                         tmp.copy(buff, string::npos);
 458                         buff[tmp.length()] = '\0';
 459                         return true;
 460                 } else {
 461                         pushTok.copy(buff, string::npos);
 462                         buff[pushTok.length()] = '\0';
 463                         pushTok.erase();
 464                         return true;
 465                 }
 466 #endif
 467         }
 468
 469         status = 0;
 470         while (is && !status) {
 471                 unsigned char c = 0;
 472                 char cc = 0;
 473                 is.get(cc);
 474                 c = cc;
 475                 if (c >= ' ' && is) {
 476                         int i = 0;
 477                         if (c == '\\') { // first char == '\\'
 478                                 do {
 479                                         buff[i++] = c;
 480                                         is.get(cc);
 481                                         c = cc;
 482                                 } while (c > ' ' && c != '\\' && is
 483                                          && i != (LEX_MAX_BUFF-1));
 484                         } else {
 485                                 do {
 486                                         buff[i++] = c;
 487                                         is.get(cc);
 488                                         c = cc;
 489                                 } while (c >= ' ' && c != '\\' && is
 490                                          && i != (LEX_MAX_BUFF-1));
 491                         }
 492
 493                         if (i == (LEX_MAX_BUFF - 1)) {
 494                                 printError("Line too long");
 495                         }
 496
 497                         if (c == '\\') is.putback(c); // put it back
 498                         buff[i] = '\0';
 499                         status = LEX_TOKEN;
 500                 }
 501
 502                 if (c == '\n')
 503                         ++lineno;
 504
 505         }
 506         if (status)  return true;
 507
 508         status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 509         buff[0] = '\0';
 510         return false;
 511 }
 512
 513
 514 void LyXLex::Pimpl::pushToken(string const & pt)
 515 {
 516         pushTok = pt;
 517 }