src/lyxlex_pimpl.C

   1 #include <config.h>
   2
   3 #ifdef __GNUG__
   4 #pragma implementation
   5 #endif
   6 #include <algorithm>
   7
   8 #include "lyxlex_pimpl.h"
   9 #include "support/lyxalgo.h"
  10 #include "support/filetools.h"
  11 #include "debug.h"
  12
  13 using std::sort;
  14 using std::ostream;
  15 using std::ios;
  16 using std::istream;
  17 using std::endl;
  18 using std::lower_bound;
  19
  20 // namespace {
  21 struct compare_tags {
  22         // used by lower_bound, sort and sorted
  23         inline
  24         int operator()(keyword_item const & a, keyword_item const & b) const {
  25                 // we use the ascii version, because in turkish, 'i'
  26                 // is not the lowercase version of 'I', and thus
  27                 // turkish locale breaks parsing of tags.
  28                 return compare_ascii_no_case(a.tag, b.tag) < 0;
  29         }
  30 };
  31 // } // end of anon namespace
  32
  33
  34 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
  35         : is(&fb__), table(tab), no_items(num),
  36           status(0), lineno(0), commentChar('#')
  37 {
  38         verifyTable();
  39 }
  40
  41
  42 string const LyXLex::Pimpl::GetString() const
  43 {
  44         return string(buff);
  45 }
  46
  47
  48 void LyXLex::Pimpl::printError(string const & message) const
  49 {
  50         string const tmpmsg = subst(message, "$$Token", GetString());
  51         lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
  52                << " of file " << MakeDisplayPath(name) << ']' << endl;
  53 }
  54
  55
  56 void LyXLex::Pimpl::printTable(ostream & os)
  57 {
  58         os << "\nNumber of tags: " << no_items << '\n';
  59         for (int i= 0; i < no_items; ++i)
  60                 os << "table[" << i
  61                    << "]:  tag: `" << table[i].tag
  62                    << "'  code:" << table[i].code << '\n';
  63         os.flush();
  64 }
  65
  66
  67 void LyXLex::Pimpl::verifyTable()
  68 {
  69         // Check if the table is sorted and if not, sort it.
  70         if (table
  71             && !lyx::sorted(table, table + no_items, compare_tags())) {
  72                 lyxerr << "The table passed to LyXLex is not sorted!\n"
  73                        << "Tell the developers to fix it!" << endl;
  74                 // We sort it anyway to avoid problems.
  75                 lyxerr << "\nUnsorted:\n";
  76                 printTable(lyxerr);
  77
  78                 sort(table, table + no_items, compare_tags());
  79                 lyxerr << "\nSorted:\n";
  80                 printTable(lyxerr);
  81         }
  82 }
  83
  84
  85 void LyXLex::Pimpl::pushTable(keyword_item * tab, int num)
  86 {
  87         pushed_table tmppu(table, no_items);
  88         pushed.push(tmppu);
  89
  90         table = tab;
  91         no_items = num;
  92
  93         verifyTable();
  94 }
  95
  96
  97 void LyXLex::Pimpl::popTable()
  98 {
  99         if (pushed.empty()) {
 100                 lyxerr << "LyXLex error: nothing to pop!" << endl;
 101                 return;
 102         }
 103
 104         pushed_table tmp = pushed.top();
 105         pushed.pop();
 106         table = tmp.table_elem;
 107         no_items = tmp.table_siz;
 108 }
 109
 110
 111 bool LyXLex::Pimpl::setFile(string const & filename)
 112 {
 113         // The check only outputs a debug message, because it triggers
 114         // a bug in compaq cxx 6.2, where is_open() returns 'true' for a
 115         // fresh new filebuf.  (JMarc)
 116         if (fb__.is_open() || is.tellg() > 0)
 117                 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
 118                         "file or stream already set." << endl;
 119         fb__.open(filename.c_str(), ios::in);
 120         is.rdbuf(&fb__);
 121         name = filename;
 122         lineno = 0;
 123         return fb__.is_open() && is.good();
 124 }
 125
 126
 127 void LyXLex::Pimpl::setStream(istream & i)
 128 {
 129         if (fb__.is_open() || is.tellg() > 0)
 130                 lyxerr[Debug::LYXLEX]  << "Error in LyXLex::setStream: "
 131                         "file or stream already set." << endl;
 132         is.rdbuf(i.rdbuf());
 133         lineno = 0;
 134 }
 135
 136 void LyXLex::Pimpl::setCommentChar(char c)
 137 {
 138         commentChar = c;
 139 }
 140
 141
 142 bool LyXLex::Pimpl::next(bool esc /* = false */)
 143 {
 144         if (!pushTok.empty()) {
 145                 // There can have been a whole line pushed so
 146                 // we extract the first word and leaves the rest
 147                 // in pushTok. (Lgb)
 148                 if (pushTok.find(' ') != string::npos) {
 149                         string tmp;
 150                         pushTok = split(pushTok, tmp, ' ');
 151                         tmp.copy(buff, string::npos);
 152                         buff[tmp.length()] = '\0';
 153                         return true;
 154                 } else {
 155                         pushTok.copy(buff, string::npos);
 156                         buff[pushTok.length()] = '\0';
 157                         pushTok.erase();
 158                         return true;
 159                 }
 160         }
 161         if (!esc) {
 162                 unsigned char c = 0; // getc() returns an int
 163                 char cc = 0;
 164                 status = 0;
 165                 while (is && !status) {
 166                         is.get(cc);
 167                         c = cc;
 168                         if (c == commentChar) {
 169                                 // Read rest of line (fast :-)
 170                                 // That is not fast... (Lgb)
 171 #if 1
 172                                 is.getline(buff, sizeof(buff));
 173                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 174                                                       << buff << "'" << endl;
 175 #else
 176                                 // unfortunately ignore is buggy (Lgb)
 177                                 is.ignore(100, '\n');
 178 #endif
 179                                 ++lineno;
 180                                 continue;
 181                         }
 182
 183                         if (c == '\"') {
 184                                 int i = -1;
 185                                 do {
 186                                         is.get(cc);
 187                                         c = cc;
 188                                         if (c != '\r')
 189                                                 buff[++i] = c;
 190                                 } while (c != '\"' && c != '\n' && is &&
 191                                          i != (LEX_MAX_BUFF - 2));
 192
 193                                 if (i == (LEX_MAX_BUFF - 2)) {
 194                                         printError("Line too long");
 195                                         c = '\"'; // Pretend we got a "
 196                                         ++i;
 197                                 }
 198
 199                                 if (c != '\"') {
 200                                         printError("Missing quote");
 201                                         if (c == '\n')
 202                                                 ++lineno;
 203                                 }
 204
 205                                 buff[i] = '\0';
 206                                 status = LEX_DATA;
 207                                 break;
 208                         }
 209
 210                         if (c == ',')
 211                                 continue;              /* Skip ','s */
 212
 213                                 // using relational operators with chars other
 214                                 // than == and != is not safe. And if it is done
 215                                 // the type _have_ to be unsigned. It usually a
 216                                 // lot better to use the functions from cctype
 217                         if (c > ' ' && is)  {
 218                                 int i = 0;
 219                                 do {
 220                                         buff[i++] = c;
 221                                         is.get(cc);
 222                                         c = cc;
 223                                 } while (c > ' ' && c != ',' && is
 224                                          && (i != LEX_MAX_BUFF - 1) );
 225                                 if (i == LEX_MAX_BUFF - 1) {
 226                                         printError("Line too long");
 227                                 }
 228                                 buff[i] = '\0';
 229                                 status = LEX_TOKEN;
 230                         }
 231
 232                         if (c == '\r' && is) {
 233                                 // The Windows support has lead to the
 234                                 // possibility of "\r\n" at the end of
 235                                 // a line.  This will stop LyX choking
 236                                 // when it expected to find a '\n'
 237                                 is.get(cc);
 238                                 c = cc;
 239                         }
 240
 241                         if (c == '\n')
 242                                 ++lineno;
 243
 244                 }
 245                 if (status) return true;
 246
 247                 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 248                 buff[0] = '\0';
 249                 return false;
 250         } else {
 251                 unsigned char c = 0; // getc() returns an int
 252                 char cc = 0;
 253
 254                 status = 0;
 255                 while (is && !status) {
 256                         is.get(cc);
 257                         c = cc;
 258
 259                         // skip ','s
 260                         if (c == ',') continue;
 261
 262                         if (c == '\\') {
 263                                 // escape
 264                                 int i = 0;
 265                                 do {
 266                                         if (c == '\\') {
 267                                                 // escape the next char
 268                                                 is.get(cc);
 269                                                 c = cc;
 270                                         }
 271                                         buff[i++] = c;
 272                                         is.get(cc);
 273                                         c = cc;
 274                                 } while (c > ' ' && c != ',' && is
 275                                          && (i != LEX_MAX_BUFF - 1) );
 276                                 if (i == LEX_MAX_BUFF - 1) {
 277                                         printError("Line too long");
 278                                 }
 279                                 buff[i] = '\0';
 280                                 status = LEX_TOKEN;
 281                                 continue;
 282                         }
 283
 284                         if (c == commentChar) {
 285                                 // Read rest of line (fast :-)
 286                                 // That is still not fast... (Lgb)
 287 #if 1
 288                                 is.getline(buff, sizeof(buff));
 289                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 290                                                       << buff << "'" << endl;
 291 #else
 292                                 // but ignore is also still buggy (Lgb)
 293                                 // This is fast (Lgb)
 294                                 is.ignore(100, '\n');
 295 #endif
 296                                 ++lineno;
 297                                 continue;
 298                         }
 299
 300                         // string
 301                         if (c == '\"') {
 302                                 int i = -1;
 303                                 bool escaped = false;
 304                                 do {
 305                                         escaped = false;
 306                                         is.get(cc);
 307                                         c = cc;
 308                                         if (c == '\r') continue;
 309                                         if (c == '\\') {
 310                                                 // escape the next char
 311                                                 is.get(cc);
 312                                                 c = cc;
 313                                                 escaped = true;
 314                                         }
 315                                         buff[++i] = c;
 316
 317                                         if (!escaped && c == '\"') break;
 318                                 } while (c != '\n' && is &&
 319                                          i != (LEX_MAX_BUFF - 2));
 320
 321                                 if (i == (LEX_MAX_BUFF - 2)) {
 322                                         printError("Line too long");
 323                                         c = '\"'; // Pretend we got a "
 324                                         ++i;
 325                                 }
 326
 327                                 if (c != '\"') {
 328                                         printError("Missing quote");
 329                                         if (c == '\n')
 330                                                 ++lineno;
 331                                 }
 332
 333                                 buff[i] = '\0';
 334                                 status = LEX_DATA;
 335                                 break;
 336                         }
 337
 338                         if (c > ' ' && is) {
 339                                 int i = 0;
 340                                 do {
 341                                         if (c == '\\') {
 342                                                 // escape the next char
 343                                                 is.get(cc);
 344                                                 c = cc;
 345                                                 //escaped = true;
 346                                         }
 347                                         buff[i++] = c;
 348                                         is.get(cc);
 349                                         c = cc;
 350                                 } while (c > ' ' && c != ',' && is
 351                                          && (i != LEX_MAX_BUFF-1) );
 352                                 if (i == LEX_MAX_BUFF-1) {
 353                                         printError("Line too long");
 354                                 }
 355                                 buff[i] = '\0';
 356                                 status = LEX_TOKEN;
 357                         }
 358                         // new line
 359                         if (c == '\n')
 360                                 ++lineno;
 361                 }
 362
 363                 if (status) return true;
 364
 365                 status = is.eof() ? LEX_FEOF : LEX_UNDEF;
 366                 buff[0] = '\0';
 367                 return false;
 368         }
 369 }
 370
 371
 372 int LyXLex::Pimpl::search_kw(char const * const tag) const
 373 {
 374         keyword_item search_tag = { tag, 0 };
 375         keyword_item * res =
 376                 lower_bound(table, table + no_items,
 377                             search_tag, compare_tags());
 378         if (res != table + no_items
 379             && !compare_no_case(res->tag, tag))
 380                 return res->code;
 381         return LEX_UNDEF;
 382 }
 383
 384
 385 int LyXLex::Pimpl::lex()
 386 {
 387         //NOTE: possible bug.
 388         if (next() && status == LEX_TOKEN)
 389                 return search_kw(buff);
 390         else
 391                 return status;
 392 }
 393
 394
 395 bool LyXLex::Pimpl::EatLine()
 396 {
 397         int i = 0;
 398         unsigned char c = '\0';
 399         char cc = 0;
 400         while(is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
 401                 is.get(cc);
 402                 c = cc;
 403                 //lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
 404                 //                    << c << "'" << endl;
 405                 if (c != '\r')
 406                         buff[i++] = c;
 407         }
 408         if (i == (LEX_MAX_BUFF - 1) && c != '\n') {
 409                 printError("Line too long");
 410                 c = '\n'; // Pretend we had an end of line
 411                 --lineno; // but don't increase line counter (netto effect)
 412                 ++i; // and preserve last character read.
 413         }
 414         if (c == '\n') {
 415                 ++lineno;
 416                 buff[--i] = '\0'; // i can never be 0 here, so no danger
 417                 status = LEX_DATA;
 418                 return true;
 419         } else {
 420                 buff[i] = '\0';
 421                 return false;
 422         }
 423 }
 424
 425
 426 bool LyXLex::Pimpl::nextToken()
 427 {
 428         if (!pushTok.empty()) {
 429                 // There can have been a whole line pushed so
 430                 // we extract the first word and leaves the rest
 431                 // in pushTok. (Lgb)
 432                 if (pushTok.find(' ') != string::npos) {
 433                         string tmp;
 434                         pushTok = split(pushTok, tmp, ' ');
 435                         tmp.copy(buff, string::npos);
 436                         buff[tmp.length()] = '\0';
 437                         return true;
 438                 } else {
 439                         pushTok.copy(buff, string::npos);
 440                         buff[pushTok.length()] = '\0';
 441                         pushTok.erase();
 442                         return true;
 443                 }
 444         }
 445
 446         status = 0;
 447         while (is && !status) {
 448                 unsigned char c = 0;
 449                 char cc = 0;
 450                 is.get(cc);
 451                 c = cc;
 452                 if (c >= ' ' && is) {
 453                         int i = 0;
 454                         if (c == '\\') { // first char == '\\'
 455                                 do {
 456                                         buff[i++] = c;
 457                                         is.get(cc);
 458                                         c = cc;
 459                                 } while (c > ' ' && c != '\\' && is
 460                                          && i != (LEX_MAX_BUFF-1));
 461                         } else {
 462                                 do {
 463                                         buff[i++] = c;
 464                                         is.get(cc);
 465                                         c = cc;
 466                                 } while (c >= ' ' && c != '\\' && is
 467                                          && i != (LEX_MAX_BUFF-1));
 468                         }
 469
 470                         if (i == (LEX_MAX_BUFF - 1)) {
 471                                 printError("Line too long");
 472                         }
 473
 474                         if (c == '\\') is.putback(c); // put it back
 475                         buff[i] = '\0';
 476                         status = LEX_TOKEN;
 477                 }
 478
 479                 if (c == '\n')
 480                         ++lineno;
 481
 482         }
 483         if (status)  return true;
 484
 485         status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 486         buff[0] = '\0';
 487         return false;
 488 }
 489
 490
 491 void LyXLex::Pimpl::pushToken(string const & pt)
 492 {
 493         pushTok = pt;
 494 }