src/lyxlex_pimpl.C

   1 #include <config.h>
   2
   3 #ifdef __GNUG__
   4 #pragma implementation
   5 #endif
   6 #include <algorithm>
   7
   8 #include "lyxlex_pimpl.h"
   9 #include "support/lyxalgo.h"
  10 #include "support/filetools.h"
  11 #include "debug.h"
  12
  13 using std::sort;
  14 using std::ostream;
  15 using std::ios;
  16 using std::istream;
  17 using std::endl;
  18 using std::lower_bound;
  19
  20 // namespace {
  21 struct compare_tags {
  22         // used by lower_bound, sort and sorted
  23         inline
  24         int operator()(keyword_item const & a, keyword_item const & b) const {
  25                 return compare_no_case(a.tag, b.tag) < 0;
  26         }
  27 };
  28 // } // end of anon namespace
  29
  30
  31 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
  32         : is(&fb__), table(tab), no_items(num),
  33           status(0), lineno(0), commentChar('#')
  34 {
  35         verifyTable();
  36 }
  37
  38
  39 string const LyXLex::Pimpl::GetString() const
  40 {
  41         return string(buff);
  42 }
  43
  44
  45 void LyXLex::Pimpl::printError(string const & message) const
  46 {
  47         string tmpmsg = subst(message, "$$Token", GetString());
  48         lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
  49                << " of file " << MakeDisplayPath(name) << ']' << endl;
  50 }
  51
  52
  53 void LyXLex::Pimpl::printTable(ostream & os)
  54 {
  55         os << "\nNumber of tags: " << no_items << '\n';
  56         for (int i= 0; i < no_items; ++i)
  57                 os << "table[" << i
  58                    << "]:  tag: `" << table[i].tag
  59                    << "'  code:" << table[i].code << '\n';
  60         os.flush();
  61 }
  62
  63
  64 void LyXLex::Pimpl::verifyTable()
  65 {
  66         // Check if the table is sorted and if not, sort it.
  67         if (table
  68             && !lyx::sorted(table, table + no_items, compare_tags())) {
  69                 lyxerr << "The table passed to LyXLex is not sorted!\n"
  70                        << "Tell the developers to fix it!" << endl;
  71                 // We sort it anyway to avoid problems.
  72                 lyxerr << "\nUnsorted:\n";
  73                 printTable(lyxerr);
  74
  75                 sort(table, table + no_items, compare_tags());
  76                 lyxerr << "\nSorted:\n";
  77                 printTable(lyxerr);
  78         }
  79 }
  80
  81
  82 void LyXLex::Pimpl::pushTable(keyword_item * tab, int num)
  83 {
  84         pushed_table tmppu(table, no_items);
  85         pushed.push(tmppu);
  86
  87         table = tab;
  88         no_items = num;
  89
  90         verifyTable();
  91 }
  92
  93
  94 void LyXLex::Pimpl::popTable()
  95 {
  96         if (pushed.empty()) {
  97                 lyxerr << "LyXLex error: nothing to pop!" << endl;
  98                 return;
  99         }
 100
 101         pushed_table tmp = pushed.top();
 102         pushed.pop();
 103         table = tmp.table_elem;
 104         no_items = tmp.table_siz;
 105 }
 106
 107
 108 bool LyXLex::Pimpl::setFile(string const & filename)
 109 {
 110         // The check only outputs a debug message, because it triggers
 111         // a bug in compaq cxx 6.2, where is_open() returns 'true' for a
 112         // fresh new filebuf.  (JMarc)
 113         if (fb__.is_open() || is.tellg() > 0)
 114                 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
 115                         "file or stream already set." << endl;
 116         fb__.open(filename.c_str(), ios::in);
 117         is.rdbuf(&fb__);
 118         name = filename;
 119         lineno = 0;
 120         return fb__.is_open() && is.good();
 121 }
 122
 123
 124 void LyXLex::Pimpl::setStream(istream & i)
 125 {
 126         if (fb__.is_open() || is.tellg() > 0)
 127                 lyxerr[Debug::LYXLEX]  << "Error in LyXLex::setStream: "
 128                         "file or stream already set." << endl;
 129         is.rdbuf(i.rdbuf());
 130         lineno = 0;
 131 }
 132
 133 void LyXLex::Pimpl::setCommentChar(char c)
 134 {
 135         commentChar = c;
 136 }
 137
 138
 139 bool LyXLex::Pimpl::next(bool esc /* = false */)
 140 {
 141         if (!pushTok.empty()) {
 142                 // There can have been a whole line pushed so
 143                 // we extract the first word and leaves the rest
 144                 // in pushTok. (Lgb)
 145                 if (pushTok.find(' ') != string::npos) {
 146                         string tmp;
 147                         pushTok = split(pushTok, tmp, ' ');
 148                         tmp.copy(buff, string::npos);
 149                         buff[tmp.length()] = '\0';
 150                         return true;
 151                 } else {
 152                         pushTok.copy(buff, string::npos);
 153                         buff[pushTok.length()] = '\0';
 154                         pushTok.erase();
 155                         return true;
 156                 }
 157         }
 158         if (!esc) {
 159                 unsigned char c = 0; // getc() returns an int
 160                 char cc = 0;
 161                 status = 0;
 162                 while (is && !status) {
 163                         is.get(cc);
 164                         c = cc;
 165                         if (c == commentChar) {
 166                                 // Read rest of line (fast :-)
 167                                 // That is not fast... (Lgb)
 168 #if 1
 169                                 is.getline(buff, sizeof(buff));
 170                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 171                                                       << buff << "'" << endl;
 172 #else
 173                                 // unfortunately is ignore buggy (Lgb)
 174                                 is.ignore(100, '\n');
 175 #endif
 176                                 ++lineno;
 177                                 continue;
 178                         }
 179
 180                         if (c == '\"') {
 181                                 int i = -1;
 182                                 do {
 183                                         is.get(cc);
 184                                         c = cc;
 185                                         if (c != '\r')
 186                                                 buff[++i] = c;
 187                                 } while (c != '\"' && c != '\n' && is &&
 188                                          i != (LEX_MAX_BUFF - 2));
 189
 190                                 if (i == (LEX_MAX_BUFF - 2)) {
 191                                         printError("Line too long");
 192                                         c = '\"'; // Pretend we got a "
 193                                         ++i;
 194                                 }
 195
 196                                 if (c != '\"') {
 197                                         printError("Missing quote");
 198                                         if (c == '\n')
 199                                                 ++lineno;
 200                                 }
 201
 202                                 buff[i] = '\0';
 203                                 status = LEX_DATA;
 204                                 break;
 205                         }
 206
 207                         if (c == ',')
 208                                 continue;              /* Skip ','s */
 209
 210                                 // using relational operators with chars other
 211                                 // than == and != is not safe. And if it is done
 212                                 // the type _have_ to be unsigned. It usually a
 213                                 // lot better to use the functions from cctype
 214                         if (c > ' ' && is)  {
 215                                 int i = 0;
 216                                 do {
 217                                         buff[i++] = c;
 218                                         is.get(cc);
 219                                         c = cc;
 220                                 } while (c > ' ' && c != ',' && is
 221                                          && (i != LEX_MAX_BUFF - 1) );
 222                                 if (i == LEX_MAX_BUFF - 1) {
 223                                         printError("Line too long");
 224                                 }
 225                                 buff[i] = '\0';
 226                                 status = LEX_TOKEN;
 227                         }
 228
 229                         if (c == '\r' && is) {
 230                                 // The Windows support has lead to the
 231                                 // possibility of "\r\n" at the end of
 232                                 // a line.  This will stop LyX choking
 233                                 // when it expected to find a '\n'
 234                                 is.get(cc);
 235                                 c = cc;
 236                         }
 237
 238                         if (c == '\n')
 239                                 ++lineno;
 240
 241                 }
 242                 if (status) return true;
 243
 244                 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 245                 buff[0] = '\0';
 246                 return false;
 247         } else {
 248                 unsigned char c = 0; // getc() returns an int
 249                 char cc = 0;
 250
 251                 status = 0;
 252                 while (is && !status) {
 253                         is.get(cc);
 254                         c = cc;
 255
 256                         // skip ','s
 257                         if (c == ',') continue;
 258
 259                         if (c == '\\') {
 260                                 // escape
 261                                 int i = 0;
 262                                 do {
 263                                         if (c == '\\') {
 264                                                 // escape the next char
 265                                                 is.get(cc);
 266                                                 c = cc;
 267                                         }
 268                                         buff[i++] = c;
 269                                         is.get(cc);
 270                                         c = cc;
 271                                 } while (c > ' ' && c != ',' && is
 272                                          && (i != LEX_MAX_BUFF - 1) );
 273                                 if (i == LEX_MAX_BUFF - 1) {
 274                                         printError("Line too long");
 275                                 }
 276                                 buff[i] = '\0';
 277                                 status = LEX_TOKEN;
 278                                 continue;
 279                         }
 280
 281                         if (c == commentChar) {
 282                                 // Read rest of line (fast :-)
 283                                 // That is still not fast... (Lgb)
 284 #if 1
 285                                 is.getline(buff, sizeof(buff));
 286                                 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 287                                                       << buff << "'" << endl;
 288 #else
 289                                 // but ignore is also still buggy (Lgb)
 290                                 // This is fast (Lgb)
 291                                 is.ignore(100, '\n');
 292 #endif
 293                                 ++lineno;
 294                                 continue;
 295                         }
 296
 297                         // string
 298                         if (c == '\"') {
 299                                 int i = -1;
 300                                 bool escaped = false;
 301                                 do {
 302                                         escaped = false;
 303                                         is.get(cc);
 304                                         c = cc;
 305                                         if (c == '\r') continue;
 306                                         if (c == '\\') {
 307                                                 // escape the next char
 308                                                 is.get(cc);
 309                                                 c = cc;
 310                                                 escaped = true;
 311                                         }
 312                                         buff[++i] = c;
 313
 314                                         if (!escaped && c == '\"') break;
 315                                 } while (c != '\n' && is &&
 316                                          i != (LEX_MAX_BUFF - 2));
 317
 318                                 if (i == (LEX_MAX_BUFF - 2)) {
 319                                         printError("Line too long");
 320                                         c = '\"'; // Pretend we got a "
 321                                         ++i;
 322                                 }
 323
 324                                 if (c != '\"') {
 325                                         printError("Missing quote");
 326                                         if (c == '\n')
 327                                                 ++lineno;
 328                                 }
 329
 330                                 buff[i] = '\0';
 331                                 status = LEX_DATA;
 332                                 break;
 333                         }
 334
 335                         if (c > ' ' && is) {
 336                                 int i = 0;
 337                                 do {
 338                                         if (c == '\\') {
 339                                                 // escape the next char
 340                                                 is.get(cc);
 341                                                 c = cc;
 342                                                 //escaped = true;
 343                                         }
 344                                         buff[i++] = c;
 345                                         is.get(cc);
 346                                         c = cc;
 347                                 } while (c > ' ' && c != ',' && is
 348                                          && (i != LEX_MAX_BUFF-1) );
 349                                 if (i == LEX_MAX_BUFF-1) {
 350                                         printError("Line too long");
 351                                 }
 352                                 buff[i] = '\0';
 353                                 status = LEX_TOKEN;
 354                         }
 355                         // new line
 356                         if (c == '\n')
 357                                 ++lineno;
 358                 }
 359
 360                 if (status) return true;
 361
 362                 status = is.eof() ? LEX_FEOF : LEX_UNDEF;
 363                 buff[0] = '\0';
 364                 return false;
 365         }
 366 }
 367
 368
 369 int LyXLex::Pimpl::search_kw(char const * const tag) const
 370 {
 371         keyword_item search_tag = { tag, 0 };
 372         keyword_item * res =
 373                 lower_bound(table, table + no_items,
 374                             search_tag, compare_tags());
 375         if (res != table + no_items
 376             && !compare_no_case(res->tag, tag))
 377                 return res->code;
 378         return LEX_UNDEF;
 379 }
 380
 381
 382 int LyXLex::Pimpl::lex()
 383 {
 384         //NOTE: possible bug.
 385         if (next() && status == LEX_TOKEN)
 386                 return search_kw(buff);
 387         else
 388                 return status;
 389 }
 390
 391
 392 bool LyXLex::Pimpl::EatLine()
 393 {
 394         int i = 0;
 395         unsigned char c = '\0';
 396         char cc = 0;
 397         while(is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
 398                 is.get(cc);
 399                 c = cc;
 400                 lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
 401                                       << c << "'" << endl;
 402                 if (c != '\r')
 403                         buff[i++] = c;
 404         }
 405         if (i == (LEX_MAX_BUFF - 1) && c != '\n') {
 406                 printError("Line too long");
 407                 c = '\n'; // Pretend we had an end of line
 408                 --lineno; // but don't increase line counter (netto effect)
 409                 ++i; // and preserve last character read.
 410         }
 411         if (c == '\n') {
 412                 ++lineno;
 413                 buff[--i] = '\0'; // i can never be 0 here, so no danger
 414                 status = LEX_DATA;
 415                 return true;
 416         } else {
 417                 buff[i] = '\0';
 418                 return false;
 419         }
 420 }
 421
 422
 423 bool LyXLex::Pimpl::nextToken()
 424 {
 425         if (!pushTok.empty()) {
 426                 // There can have been a whole line pushed so
 427                 // we extract the first word and leaves the rest
 428                 // in pushTok. (Lgb)
 429                 if (pushTok.find(' ') != string::npos) {
 430                         string tmp;
 431                         pushTok = split(pushTok, tmp, ' ');
 432                         tmp.copy(buff, string::npos);
 433                         buff[tmp.length()] = '\0';
 434                         return true;
 435                 } else {
 436                         pushTok.copy(buff, string::npos);
 437                         buff[pushTok.length()] = '\0';
 438                         pushTok.erase();
 439                         return true;
 440                 }
 441         }
 442
 443         status = 0;
 444         while (is && !status) {
 445                 unsigned char c = 0;
 446                 char cc = 0;
 447                 is.get(cc);
 448                 c = cc;
 449                 if (c >= ' ' && is) {
 450                         int i = 0;
 451                         if (c == '\\') { // first char == '\\'
 452                                 do {
 453                                         buff[i++] = c;
 454                                         is.get(cc);
 455                                         c = cc;
 456                                 } while (c > ' ' && c != '\\' && is
 457                                          && i != (LEX_MAX_BUFF-1));
 458                         } else {
 459                                 do {
 460                                         buff[i++] = c;
 461                                         is.get(cc);
 462                                         c = cc;
 463                                 } while (c >= ' ' && c != '\\' && is
 464                                          && i != (LEX_MAX_BUFF-1));
 465                         }
 466
 467                         if (i == (LEX_MAX_BUFF - 1)) {
 468                                 printError("Line too long");
 469                         }
 470
 471                         if (c == '\\') is.putback(c); // put it back
 472                         buff[i] = '\0';
 473                         status = LEX_TOKEN;
 474                 }
 475
 476                 if (c == '\n')
 477                         ++lineno;
 478
 479         }
 480         if (status)  return true;
 481
 482         status = is.eof() ? LEX_FEOF: LEX_UNDEF;
 483         buff[0] = '\0';
 484         return false;
 485 }
 486
 487
 488 void LyXLex::Pimpl::pushToken(string const & pt)
 489 {
 490         pushTok = pt;
 491 }