src/Encoding.cpp

   1 /**
   2  * \file Encoding.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  * \author Jean-Marc Lasgouttes
   8  * \author Dekel Tsur
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "Encoding.h"
  16
  17 #include "Lexer.h"
  18
  19 #include "support/debug.h"
  20 #include "support/gettext.h"
  21 #include "support/lstrings.h"
  22 #include "support/textutils.h"
  23 #include "support/unicode.h"
  24
  25 #include <boost/cstdint.hpp>
  26
  27 #include <sstream>
  28
  29 using namespace std;
  30 using namespace lyx::support;
  31
  32 namespace lyx {
  33
  34 int const Encoding::any = -1;
  35
  36 Encodings encodings;
  37
  38 Encodings::MathCommandSet Encodings::mathcmd;
  39 Encodings::TextCommandSet Encodings::textcmd;
  40 Encodings::MathSymbolSet  Encodings::mathsym;
  41
  42 namespace {
  43
  44 char_type arabic_table[172][4] = {
  45         {0xfe80, 0xfe80, 0xfe80, 0xfe80}, // 0x0621 = hamza
  46         {0xfe81, 0xfe82, 0xfe81, 0xfe82}, // 0x0622 = ligature madda on alef
  47         {0xfe83, 0xfe84, 0xfe83, 0xfe84}, // 0x0623 = ligature hamza on alef
  48         {0xfe85, 0xfe86, 0xfe85, 0xfe86}, // 0x0624 = ligature hamza on waw
  49         {0xfe87, 0xfe88, 0xfe87, 0xfe88}, // 0x0625 = ligature hamza under alef
  50         {0xfe89, 0xfe8a, 0xfe8b, 0xfe8c}, // 0x0626 = ligature hamza on ya
  51         {0xfe8d, 0xfe8e, 0xfe8d, 0xfe8e}, // 0x0627 = alef
  52         {0xfe8f, 0xfe90, 0xfe91, 0xfe92}, // 0x0628 = baa
  53         {0xfe93, 0xfe94, 0xfe93, 0xfe94}, // 0x0629 = taa marbuta
  54         {0xfe95, 0xfe96, 0xfe97, 0xfe98}, // 0x062a = taa
  55         {0xfe99, 0xfe9a, 0xfe9b, 0xfe9c}, // 0x062b = thaa
  56         {0xfe9d, 0xfe9e, 0xfe9f, 0xfea0}, // 0x062c = jeem
  57         {0xfea1, 0xfea2, 0xfea3, 0xfea4}, // 0x062d = haa
  58         {0xfea5, 0xfea6, 0xfea7, 0xfea8}, // 0x062e = khaa
  59         {0xfea9, 0xfeaa, 0xfea9, 0xfeaa}, // 0x062f = dal
  60
  61         {0xfeab, 0xfeac, 0xfeab, 0xfeac}, // 0x0630 = thal
  62         {0xfead, 0xfeae, 0xfead, 0xfeae}, // 0x0631 = ra
  63         {0xfeaf, 0xfeb0, 0xfeaf, 0xfeb0}, // 0x0632 = zain
  64         {0xfeb1, 0xfeb2, 0xfeb3, 0xfeb4}, // 0x0633 = seen
  65         {0xfeb5, 0xfeb6, 0xfeb7, 0xfeb8}, // 0x0634 = sheen
  66         {0xfeb9, 0xfeba, 0xfebb, 0xfebc}, // 0x0635 = sad
  67         {0xfebd, 0xfebe, 0xfebf, 0xfec0}, // 0x0636 = dad
  68         {0xfec1, 0xfec2, 0xfec3, 0xfec4}, // 0x0637 = tah
  69         {0xfec5, 0xfec6, 0xfec7, 0xfec8}, // 0x0638 = zah
  70         {0xfec9, 0xfeca, 0xfecb, 0xfecc}, // 0x0639 = ain
  71         {0xfecd, 0xfece, 0xfecf, 0xfed0}, // 0x063a = ghain
  72         {0, 0, 0, 0}, // 0x063b
  73         {0, 0, 0, 0}, // 0x063c
  74         {0, 0, 0, 0}, // 0x063d
  75         {0, 0, 0, 0}, // 0x063e
  76         {0, 0, 0, 0}, // 0x063f
  77
  78         {0, 0, 0, 0}, // 0x0640
  79         {0xfed1, 0xfed2, 0xfed3, 0xfed4}, // 0x0641 = fa
  80         {0xfed5, 0xfed6, 0xfed7, 0xfed8}, // 0x0642 = qaf
  81         {0xfed9, 0xfeda, 0xfedb, 0xfedc}, // 0x0643 = kaf
  82         {0xfedd, 0xfede, 0xfedf, 0xfee0}, // 0x0644 = lam
  83         {0xfee1, 0xfee2, 0xfee3, 0xfee4}, // 0x0645 = meem
  84         {0xfee5, 0xfee6, 0xfee7, 0xfee8}, // 0x0646 = noon
  85         {0xfee9, 0xfeea, 0xfeeb, 0xfeec}, // 0x0647 = ha
  86         {0xfeed, 0xfeee, 0xfeed, 0xfeee}, // 0x0648 = waw
  87         {0xfeef, 0xfef0, 0xfeef, 0xfef0}, // 0x0649 = alef maksura
  88         {0xfef1, 0xfef2, 0xfef3, 0xfef4}, // 0x064a = ya
  89         {0x065b, 0x065b, 0x065b, 0x065b}, // 0x064b = fathatan
  90         {0x065c, 0x065c, 0x065c, 0x065c}, // 0x064c = dammatan
  91         {0x064d, 0x064d, 0x064d, 0x064d}, // 0x064d = kasratan
  92         {0x064e, 0x064e, 0x064e, 0x064e}, // 0x064e = fatha
  93         {0x064f, 0x064f, 0x064f, 0x064f}, // 0x064f = damma
  94
  95         {0x0650, 0x0650, 0x0650, 0x0650}, // 0x0650 = kasra
  96         {0x0651, 0x0651, 0x0651, 0x0651}, // 0x0651 = shadda
  97         {0x0652, 0x0652, 0x0652, 0x0652}, // 0x0652 = sukun
  98
  99         {0, 0, 0, 0}, // 0x0653
 100         {0, 0, 0, 0}, // 0x0654
 101         {0, 0, 0, 0}, // 0x0655
 102         {0, 0, 0, 0}, // 0x0656
 103         {0, 0, 0, 0}, // 0x0657
 104         {0, 0, 0, 0}, // 0x0658
 105         {0, 0, 0, 0}, // 0x0659
 106         {0, 0, 0, 0}, // 0x065a
 107         {0, 0, 0, 0}, // 0x065b
 108         {0, 0, 0, 0}, // 0x065c
 109         {0, 0, 0, 0}, // 0x065d
 110         {0, 0, 0, 0}, // 0x065e
 111         {0, 0, 0, 0}, // 0x065f
 112         {0, 0, 0, 0}, // 0x0660
 113         {0, 0, 0, 0}, // 0x0661
 114         {0, 0, 0, 0}, // 0x0662
 115         {0, 0, 0, 0}, // 0x0663
 116         {0, 0, 0, 0}, // 0x0664
 117         {0, 0, 0, 0}, // 0x0665
 118         {0, 0, 0, 0}, // 0x0666
 119         {0, 0, 0, 0}, // 0x0667
 120         {0, 0, 0, 0}, // 0x0668
 121         {0, 0, 0, 0}, // 0x0669
 122         {0, 0, 0, 0}, // 0x066a
 123         {0, 0, 0, 0}, // 0x066b
 124         {0, 0, 0, 0}, // 0x066c
 125         {0, 0, 0, 0}, // 0x066d
 126         {0, 0, 0, 0}, // 0x066e
 127         {0, 0, 0, 0}, // 0x066f
 128         {0, 0, 0, 0}, // 0x0670
 129         {0, 0, 0, 0}, // 0x0671
 130         {0, 0, 0, 0}, // 0x0672
 131         {0, 0, 0, 0}, // 0x0673
 132         {0, 0, 0, 0}, // 0x0674
 133         {0, 0, 0, 0}, // 0x0675
 134         {0, 0, 0, 0}, // 0x0676
 135         {0, 0, 0, 0}, // 0x0677
 136         {0, 0, 0, 0}, // 0x0678
 137         {0, 0, 0, 0}, // 0x0679
 138         {0, 0, 0, 0}, // 0x067a
 139         {0, 0, 0, 0}, // 0x067b
 140         {0, 0, 0, 0}, // 0x067c
 141         {0, 0, 0, 0}, // 0x067d
 142         {0xfb56, 0xfb57, 0xfb58, 0xfb59}, // 0x067e = peh
 143         {0, 0, 0, 0}, // 0x067f
 144         {0, 0, 0, 0}, // 0x0680
 145         {0, 0, 0, 0}, // 0x0681
 146         {0, 0, 0, 0}, // 0x0682
 147         {0, 0, 0, 0}, // 0x0683
 148         {0, 0, 0, 0}, // 0x0684
 149         {0, 0, 0, 0}, // 0x0685
 150         {0xfb7a, 0xfb7b, 0xfb7c, 0xfb7d}, // 0x0686 = tcheh
 151         {0, 0, 0, 0}, // 0x0687
 152         {0, 0, 0, 0}, // 0x0688
 153         {0, 0, 0, 0}, // 0x0689
 154         {0, 0, 0, 0}, // 0x068a
 155         {0, 0, 0, 0}, // 0x068b
 156         {0, 0, 0, 0}, // 0x068c
 157         {0, 0, 0, 0}, // 0x068d
 158         {0, 0, 0, 0}, // 0x068e
 159         {0, 0, 0, 0}, // 0x068f
 160         {0, 0, 0, 0}, // 0x0690
 161         {0, 0, 0, 0}, // 0x0691
 162         {0, 0, 0, 0}, // 0x0692
 163         {0, 0, 0, 0}, // 0x0693
 164         {0, 0, 0, 0}, // 0x0694
 165         {0, 0, 0, 0}, // 0x0695
 166         {0, 0, 0, 0}, // 0x0696
 167         {0, 0, 0, 0}, // 0x0697
 168         {0xfb8a, 0xfb8b, 0xfb8a, 0xfb8b}, // 0x0698 = jeh
 169         {0, 0, 0, 0}, // 0x0699
 170         {0, 0, 0, 0}, // 0x069a
 171         {0, 0, 0, 0}, // 0x069b
 172         {0, 0, 0, 0}, // 0x069c
 173         {0, 0, 0, 0}, // 0x069d
 174         {0, 0, 0, 0}, // 0x069e
 175         {0, 0, 0, 0}, // 0x069f
 176         {0, 0, 0, 0}, // 0x06a0
 177         {0, 0, 0, 0}, // 0x06a1
 178         {0, 0, 0, 0}, // 0x06a2
 179         {0, 0, 0, 0}, // 0x06a3
 180         {0, 0, 0, 0}, // 0x06a4
 181         {0, 0, 0, 0}, // 0x06a5
 182         {0, 0, 0, 0}, // 0x06a6
 183         {0, 0, 0, 0}, // 0x06a7
 184         {0, 0, 0, 0}, // 0x06a8
 185         {0xfb8e, 0xfb8f, 0xfb90, 0xfb91}, // 0x06a9 = farsi kaf
 186         {0, 0, 0, 0}, // 0x06aa
 187         {0, 0, 0, 0}, // 0x06ab
 188         {0, 0, 0, 0}, // 0x06ac
 189         {0, 0, 0, 0}, // 0x06ad
 190         {0, 0, 0, 0}, // 0x06ae
 191         {0xfb92, 0xfb93, 0xfb94, 0xfb95}, // 0x06af = gaf
 192         {0, 0, 0, 0}, // 0x06b0
 193         {0, 0, 0, 0}, // 0x06b1
 194         {0, 0, 0, 0}, // 0x06b2
 195         {0, 0, 0, 0}, // 0x06b3
 196         {0, 0, 0, 0}, // 0x06b4
 197         {0, 0, 0, 0}, // 0x06b5
 198         {0, 0, 0, 0}, // 0x06b6
 199         {0, 0, 0, 0}, // 0x06b7
 200         {0, 0, 0, 0}, // 0x06b8
 201         {0, 0, 0, 0}, // 0x06b9
 202         {0, 0, 0, 0}, // 0x06ba
 203         {0, 0, 0, 0}, // 0x06bb
 204         {0, 0, 0, 0}, // 0x06bc
 205         {0, 0, 0, 0}, // 0x06bd
 206         {0, 0, 0, 0}, // 0x06be
 207         {0, 0, 0, 0}, // 0x06bf
 208         {0, 0, 0, 0}, // 0x06c0
 209         {0, 0, 0, 0}, // 0x06c1
 210         {0, 0, 0, 0}, // 0x06c2
 211         {0, 0, 0, 0}, // 0x06c3
 212         {0, 0, 0, 0}, // 0x06c4
 213         {0, 0, 0, 0}, // 0x06c5
 214         {0, 0, 0, 0}, // 0x06c6
 215         {0, 0, 0, 0}, // 0x06c7
 216         {0, 0, 0, 0}, // 0x06c8
 217         {0, 0, 0, 0}, // 0x06c9
 218         {0, 0, 0, 0}, // 0x06ca
 219         {0, 0, 0, 0}, // 0x06cb
 220         {0xfbfc, 0xfbfd, 0xfbfe, 0xfbff} // 0x06cc = farsi yeh
 221 };
 222
 223
 224 char_type const arabic_start = 0x0621;
 225 char_type const arabic_end = 0x06cc;
 226
 227
 228 typedef map<char_type, CharInfo> CharInfoMap;
 229 CharInfoMap unicodesymbols;
 230
 231 typedef set<char_type> CharSet;
 232 typedef map<string, CharSet> CharSetMap;
 233 CharSet forced;
 234 CharSetMap forcedselected;
 235
 236 typedef set<char_type> MathAlphaSet;
 237 MathAlphaSet mathalpha;
 238
 239
 240 /// The highest code point in UCS4 encoding (1<<20 + 1<<16)
 241 char_type const max_ucs4 = 0x110000;
 242
 243 } // namespace anon
 244
 245
 246 EncodingException::EncodingException(char_type c)
 247         : failed_char(c), par_id(0), pos(0)
 248 {
 249 }
 250
 251
 252 const char * EncodingException::what() const throw()
 253 {
 254         return "Could not find LaTeX command for a character";
 255 }
 256
 257
 258 Encoding::Encoding(string const & n, string const & l, string const & g,
 259                    string const & i, bool f, bool u, Encoding::Package p)
 260         : name_(n), latexName_(l), guiName_(g), iconvName_(i), fixedwidth_(f),
 261           unsafe_(u), forced_(&forcedselected[n]), package_(p)
 262 {
 263         if (n == "ascii") {
 264                 // ASCII can encode 128 code points and nothing else
 265                 start_encodable_ = 128;
 266                 complete_ = true;
 267         } else if (i == "UTF-8") {
 268                 // UTF8 can encode all UCS4 code points
 269                 start_encodable_ = max_ucs4;
 270                 complete_ = true;
 271         } else {
 272                 start_encodable_ = 0;
 273                 complete_ = false;
 274         }
 275 }
 276
 277
 278 void Encoding::init() const
 279 {
 280         if (complete_)
 281                 return;
 282
 283         start_encodable_ = 0;
 284         // temporarily switch off lyxerr, since we will generate iconv errors
 285         lyxerr.disable();
 286         if (fixedwidth_) {
 287                 // We do not need to check all UCS4 code points, it is enough
 288                 // if we check all 256 code points of this encoding.
 289                 for (unsigned short j = 0; j < 256; ++j) {
 290                         char const c = char(j);
 291                         vector<char_type> const ucs4 = eightbit_to_ucs4(&c, 1, iconvName_);
 292                         if (ucs4.size() != 1)
 293                                 continue;
 294                         char_type const uc = ucs4[0];
 295                         CharInfoMap::const_iterator const it = unicodesymbols.find(uc);
 296                         if (it == unicodesymbols.end())
 297                                 encodable_.insert(uc);
 298                         else if (!it->second.force()) {
 299                                 if (forced_->empty() || forced_->find(uc) == forced_->end())
 300                                         encodable_.insert(uc);
 301                         }
 302                 }
 303         } else {
 304                 // We do not know how many code points this encoding has, and
 305                 // they do not have a direct representation as a single byte,
 306                 // therefore we need to check all UCS4 code points.
 307                 // This is expensive!
 308                 for (char_type c = 0; c < max_ucs4; ++c) {
 309                         vector<char> const eightbit = ucs4_to_eightbit(&c, 1, iconvName_);
 310                         if (!eightbit.empty()) {
 311                                 CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 312                                 if (it == unicodesymbols.end())
 313                                         encodable_.insert(c);
 314                                 else if (!it->second.force()) {
 315                                         if (forced_->empty() || forced_->find(c) == forced_->end())
 316                                                 encodable_.insert(c);
 317                                 }
 318                         }
 319                 }
 320         }
 321         lyxerr.enable();
 322         CharSet::iterator it = encodable_.find(start_encodable_);
 323         while (it != encodable_.end()) {
 324                 encodable_.erase(it);
 325                 ++start_encodable_;
 326                 it = encodable_.find(start_encodable_);
 327         }
 328         complete_ = true;
 329 }
 330
 331
 332 bool Encoding::isForced(char_type c) const
 333 {
 334         if (!forced.empty() && forced.find(c) != forced.end())
 335                 return true;
 336         return !forced_->empty() && forced_->find(c) != forced_->end();
 337 }
 338
 339
 340 bool Encoding::encodable(char_type c) const
 341 {
 342         // assure the used encoding is properly initialized
 343         init();
 344
 345         if (iconvName_ == "UTF-8" && package_ == none)
 346                 return true;
 347         if (c < start_encodable_ && !isForced(c))
 348                 return true;
 349         if (encodable_.find(c) != encodable_.end())
 350                 return true;
 351         return false;
 352 }
 353
 354
 355 pair<docstring, bool> Encoding::latexChar(char_type c) const
 356 {
 357         if (encodable(c))
 358                 return make_pair(docstring(1, c), false);
 359
 360         // c cannot (or should not) be encoded in this encoding
 361         CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 362         if (it == unicodesymbols.end())
 363                 throw EncodingException(c);
 364         // at least one of mathcommand and textcommand is nonempty
 365         if (it->second.textcommand.empty())
 366                 return make_pair(
 367                         "\\ensuremath{" + it->second.mathcommand + '}', false);
 368         return make_pair(it->second.textcommand, !it->second.textnotermination());
 369 }
 370
 371
 372 pair<docstring, docstring> Encoding::latexString(docstring const input, bool dryrun) const
 373 {
 374         docstring result;
 375         docstring uncodable;
 376         bool terminate = false;
 377         for (size_t n = 0; n < input.size(); ++n) {
 378                 try {
 379                         char_type const c = input[n];
 380                         pair<docstring, bool> latex_char = latexChar(c);
 381                         docstring const latex = latex_char.first;
 382                         if (terminate && !prefixIs(latex, '\\')
 383                             && !prefixIs(latex, '{')
 384                             && !prefixIs(latex, '}')) {
 385                                         // Prevent eating of a following
 386                                         // space or command corruption by
 387                                         // following characters
 388                                         if (latex == " ")
 389                                                 result += "{}";
 390                                         else
 391                                                 result += " ";
 392                                 }
 393                         result += latex;
 394                         terminate = latex_char.second;
 395                 } catch (EncodingException & /* e */) {
 396                         LYXERR0("Uncodable character in latexString!");
 397                         if (dryrun) {
 398                                 result += "<" + _("LyX Warning: ")
 399                                            + _("uncodable character") + " '";
 400                                 result += docstring(1, input[n]);
 401                                 result += "'>";
 402                         } else
 403                                 uncodable += input[n];
 404                 }
 405         }
 406         return make_pair(result, uncodable);
 407 }
 408
 409
 410 vector<char_type> Encoding::symbolsList() const
 411 {
 412         // assure the used encoding is properly initialized
 413         init();
 414
 415         // first all encodable characters
 416         vector<char_type> symbols(encodable_.begin(), encodable_.end());
 417         // add those below start_encodable_
 418         for (char_type c = 0; c < start_encodable_; ++c)
 419                 symbols.push_back(c);
 420         // now the ones from the unicodesymbols file
 421         CharInfoMap::const_iterator const end = unicodesymbols.end();
 422         CharInfoMap::const_iterator it = unicodesymbols.begin();
 423         for (; it != end; ++it)
 424                 symbols.push_back(it->first);
 425         return symbols;
 426 }
 427
 428
 429 bool Encodings::latexMathChar(char_type c, bool mathmode,
 430                         Encoding const * encoding, docstring & command,
 431                         bool & needsTermination)
 432 {
 433         command = empty_docstring();
 434         if (encoding)
 435                 if (encoding->encodable(c))
 436                         command = docstring(1, c);
 437         needsTermination = false;
 438
 439         CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 440         if (it == unicodesymbols.end()) {
 441                 if (!encoding || command.empty())
 442                         throw EncodingException(c);
 443                 if (mathmode)
 444                         addMathSym(c);
 445                 return false;
 446         }
 447         // at least one of mathcommand and textcommand is nonempty
 448         bool use_math = (mathmode && !it->second.mathcommand.empty()) ||
 449                         (!mathmode && it->second.textcommand.empty());
 450         if (use_math) {
 451                 command = it->second.mathcommand;
 452                 needsTermination = !it->second.mathnotermination();
 453                 addMathCmd(c);
 454         } else {
 455                 if (!encoding || command.empty()) {
 456                         command = it->second.textcommand;
 457                         needsTermination = !it->second.textnotermination();
 458                         addTextCmd(c);
 459                 }
 460                 if (mathmode)
 461                         addMathSym(c);
 462         }
 463         return use_math;
 464 }
 465
 466
 467 char_type Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
 468                 bool & combining, bool & needsTermination, set<string> * req)
 469 {
 470         CharInfoMap::const_iterator const end = unicodesymbols.end();
 471         CharInfoMap::const_iterator it = unicodesymbols.begin();
 472         for (combining = false; it != end; ++it) {
 473                 docstring const math = it->second.mathcommand;
 474                 docstring const text = it->second.textcommand;
 475                 if ((cmdtype & MATH_CMD) && math == cmd) {
 476                         combining = it->second.combining();
 477                         needsTermination = !it->second.mathnotermination();
 478                         if (req && it->second.mathfeature() &&
 479                             !it->second.mathpreamble.empty())
 480                                 req->insert(it->second.mathpreamble);
 481                         return it->first;
 482                 }
 483                 if ((cmdtype & TEXT_CMD) && text == cmd) {
 484                         combining = it->second.combining();
 485                         needsTermination = !it->second.textnotermination();
 486                         if (req && it->second.textfeature() &&
 487                             !it->second.textpreamble.empty())
 488                                 req->insert(it->second.textpreamble);
 489                         return it->first;
 490                 }
 491         }
 492         needsTermination = false;
 493         return 0;
 494 }
 495
 496
 497 docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
 498                 bool & needsTermination, docstring & rem, set<string> * req)
 499 {
 500         needsTermination = false;
 501         rem = empty_docstring();
 502         bool const mathmode = cmdtype & MATH_CMD;
 503         bool const textmode = cmdtype & TEXT_CMD;
 504         docstring symbols;
 505         size_t const cmdend = cmd.size();
 506         size_t prefix = 0;
 507         CharInfoMap::const_iterator const uniend = unicodesymbols.end();
 508         for (size_t i = 0, j = 0; j < cmdend; ++j) {
 509                 // Also get the char after a backslash
 510                 if (j + 1 < cmdend && cmd[j] == '\\') {
 511                         ++j;
 512                         prefix = 1;
 513                         // Detect things like \=*{e} as well
 514                         if (j + 3 < cmdend && cmd[j+1] == '*' &&
 515                             cmd[j+2] == '{') {
 516                                 ++j;
 517                                 prefix = 2;
 518                         }
 519                 }
 520                 // position of the last character before a possible macro
 521                 // argument
 522                 size_t m = j;
 523                 // If a macro argument follows, get it, too
 524                 // Do it here only for single character commands. Other
 525                 // combining commands need this too, but they are handled in
 526                 // the loop below for performance reasons.
 527                 if (j + 1 < cmdend && cmd[j + 1] == '{') {
 528                         size_t k = j + 1;
 529                         int count = 1;
 530                         while (k < cmdend && count && k != docstring::npos) {
 531                                 k = cmd.find_first_of(from_ascii("{}"), k + 1);
 532                                 if (cmd[k] == '{')
 533                                         ++count;
 534                                 else
 535                                         --count;
 536                         }
 537                         if (k != docstring::npos)
 538                                 j = k;
 539                 } else if (m + 1 < cmdend && isAlphaASCII(cmd[m])) {
 540                         while (m + 2 < cmdend && isAlphaASCII(cmd[m+1]))
 541                                 m++;
 542                 }
 543                 // Start with this substring and try augmenting it when it is
 544                 // the prefix of some command in the unicodesymbols file
 545                 docstring subcmd = cmd.substr(i, j - i + 1);
 546
 547                 CharInfoMap::const_iterator it = unicodesymbols.begin();
 548                 // First part of subcmd which might be a combining character
 549                 docstring combcmd = (m == j) ? docstring() : cmd.substr(i, m - i + 1);
 550                 // The combining character of combcmd if it exists
 551                 CharInfoMap::const_iterator combining = uniend;
 552                 size_t unicmd_size = 0;
 553                 char_type c = 0;
 554                 for (; it != uniend; ++it) {
 555                         docstring const math = mathmode ? it->second.mathcommand
 556                                                         : docstring();
 557                         docstring const text = textmode ? it->second.textcommand
 558                                                         : docstring();
 559                         if (!combcmd.empty() && it->second.combining() &&
 560                             (math == combcmd || text == combcmd))
 561                                 combining = it;
 562                         size_t cur_size = max(math.size(), text.size());
 563                         // The current math or text unicode command cannot
 564                         // match, or we already matched a longer one
 565                         if (cur_size < subcmd.size() || cur_size <= unicmd_size)
 566                                 continue;
 567
 568                         docstring tmp = subcmd;
 569                         size_t k = j;
 570                         while (prefixIs(math, tmp) || prefixIs(text, tmp)) {
 571                                 ++k;
 572                                 if (k >= cmdend || cur_size <= tmp.size())
 573                                         break;
 574                                 tmp += cmd[k];
 575                         }
 576                         // No match
 577                         if (k == j)
 578                                 continue;
 579
 580                         // The last added char caused a mismatch, because
 581                         // we didn't exhaust the chars in cmd and didn't
 582                         // exceed the maximum size of the current unicmd
 583                         if (k < cmdend && cur_size > tmp.size())
 584                                 tmp.resize(tmp.size() - 1);
 585
 586                         // If this is an exact match, we found a (longer)
 587                         // matching entry in the unicodesymbols file.
 588                         if (math != tmp && text != tmp)
 589                                 continue;
 590                         // If we found a combining command, we need to append
 591                         // the macro argument if this has not been done above.
 592                         if (tmp == combcmd && combining != uniend &&
 593                             k < cmdend && cmd[k] == '{') {
 594                                 size_t l = k;
 595                                 int count = 1;
 596                                 while (l < cmdend && count && l != docstring::npos) {
 597                                         l = cmd.find_first_of(from_ascii("{}"), l + 1);
 598                                         if (cmd[l] == '{')
 599                                                 ++count;
 600                                         else
 601                                                 --count;
 602                                 }
 603                                 if (l != docstring::npos) {
 604                                         j = l;
 605                                         subcmd = cmd.substr(i, j - i + 1);
 606                                 }
 607                         }
 608                         // If the entry doesn't start with '\', we take note
 609                         // of the match and continue (this is not a ultimate
 610                         // acceptance, as some other entry may match a longer
 611                         // portion of the cmd string). However, if the entry
 612                         // does start with '\', we accept the match only if
 613                         // this is a valid macro, i.e., either it is a single
 614                         // (nonletter) char macro, or nothing else follows,
 615                         // or what follows is a nonletter char, or the last
 616                         // character is a }.
 617                         else if (tmp[0] != '\\'
 618                                    || (tmp.size() == prefix + 1 &&
 619                                        !isAlphaASCII(tmp[1]) &&
 620                                        (prefix == 1 || !isAlphaASCII(tmp[2])))
 621                                    || k == cmdend
 622                                    || !isAlphaASCII(cmd[k])
 623                                    || tmp[tmp.size() - 1] == '}'
 624                                  ) {
 625                                 c = it->first;
 626                                 j = k - 1;
 627                                 i = j + 1;
 628                                 unicmd_size = cur_size;
 629                                 if (math == tmp)
 630                                         needsTermination = !it->second.mathnotermination();
 631                                 else
 632                                         needsTermination = !it->second.textnotermination();
 633                                 if (req) {
 634                                         if (math == tmp && it->second.mathfeature() &&
 635                                             !it->second.mathpreamble.empty())
 636                                                 req->insert(it->second.mathpreamble);
 637                                         if (text == tmp && it->second.textfeature() &&
 638                                             !it->second.textpreamble.empty())
 639                                                 req->insert(it->second.textpreamble);
 640                                 }
 641                         }
 642                 }
 643                 if (unicmd_size)
 644                         symbols += c;
 645                 else if (combining != uniend &&
 646                          prefixIs(subcmd, combcmd + '{')) {
 647                         // We know that subcmd starts with combcmd and
 648                         // contains an argument in braces.
 649                         docstring const arg = subcmd.substr(
 650                                 combcmd.length() + 1,
 651                                 subcmd.length() - combcmd.length() - 2);
 652                         // If arg is a single character we can construct a
 653                         // combining sequence.
 654                         char_type a;
 655                         bool argcomb = false;
 656                         if (arg.size() == 1 && isAlnumASCII(arg[0]))
 657                                 a = arg[0];
 658                         else {
 659                                 // Use the version of fromLaTeXCommand() that
 660                                 // parses only one command, since we cannot
 661                                 // use more than one character.
 662                                 bool dummy = false;
 663                                 set<string> r;
 664                                 a = fromLaTeXCommand(arg, cmdtype, argcomb,
 665                                                      dummy, &r);
 666                                 if (a && req && !argcomb)
 667                                         req->insert(r.begin(), r.end());
 668                         }
 669                         if (a && !argcomb) {
 670                                 // In unicode the combining character comes
 671                                 // after its base
 672                                 symbols += a;
 673                                 symbols += combining->first;
 674                                 i = j + 1;
 675                                 unicmd_size = 2;
 676                         }
 677                 }
 678                 if (j + 1 == cmdend && !unicmd_size) {
 679                         // No luck. Return what remains
 680                         rem = cmd.substr(i);
 681                         if (needsTermination && !rem.empty()) {
 682                                 if (rem.substr(0, 2) == "{}") {
 683                                         rem = rem.substr(2);
 684                                         needsTermination = false;
 685                                 } else if (rem[0] == ' ') {
 686                                         needsTermination = false;
 687                                         // LaTeX would swallow all spaces
 688                                         rem = ltrim(rem);
 689                                 }
 690                         }
 691                 }
 692         }
 693         return symbols;
 694 }
 695
 696
 697 bool Encodings::isHebrewComposeChar(char_type c)
 698 {
 699         return c <= 0x05c2 && c >= 0x05b0 && c != 0x05be && c != 0x05c0;
 700 }
 701
 702
 703 // Special Arabic letters are ones that do not get connected from left
 704 // they are hamza, alef_madda, alef_hamza, waw_hamza, alef_hamza_under,
 705 // alef, tah_marbota, dal, thal, rah, zai, wow, alef_maksoura
 706
 707 bool Encodings::isArabicSpecialChar(char_type c)
 708 {
 709         return (c >= 0x0621 && c <= 0x0625) || (c >= 0x0630 && c <= 0x0632)
 710                 || c == 0x0627 || c == 0x0629 || c == 0x062f || c == 0x0648
 711                 || c == 0x0649 || c == 0x0698;
 712 }
 713
 714
 715 bool Encodings::isArabicComposeChar(char_type c)
 716 {
 717         return c >= 0x064b && c <= 0x0652;
 718 }
 719
 720
 721 bool Encodings::isArabicChar(char_type c)
 722 {
 723         return c >= arabic_start && c <= arabic_end
 724                 && arabic_table[c-arabic_start][0];
 725 }
 726
 727
 728 CharInfo const & Encodings::unicodeCharInfo(char_type c)
 729 {
 730         static CharInfo empty;
 731         CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 732         return it != unicodesymbols.end() ? it->second : empty;
 733 }
 734
 735
 736 char_type Encodings::transformChar(char_type c, Encodings::LetterForm form)
 737 {
 738         return isArabicChar(c) ? arabic_table[c-arabic_start][form] : c;
 739 }
 740
 741
 742 bool Encodings::isCombiningChar(char_type c)
 743 {
 744         CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 745         if (it != unicodesymbols.end())
 746                 return it->second.combining();
 747         return false;
 748 }
 749
 750
 751 string const Encodings::TIPAShortcut(char_type c)
 752 {
 753         CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 754         if (it != unicodesymbols.end())
 755                 return it->second.tipashortcut;
 756         return string();
 757 }
 758
 759
 760 bool Encodings::isKnownScriptChar(char_type const c, string & preamble)
 761 {
 762         CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 763
 764         if (it == unicodesymbols.end())
 765                 return false;
 766
 767         if (it->second.textpreamble != "textgreek" && it->second.textpreamble != "textcyr")
 768                 return false;
 769
 770         if (preamble.empty()) {
 771                 preamble = it->second.textpreamble;
 772                 return true;
 773         }
 774         return it->second.textpreamble == preamble;
 775 }
 776
 777
 778 bool Encodings::isMathAlpha(char_type c)
 779 {
 780         return mathalpha.count(c);
 781 }
 782
 783
 784 Encoding const *
 785 Encodings::fromLyXName(string const & name, bool allowUnsafe) const
 786 {
 787         EncodingList::const_iterator const it = encodinglist.find(name);
 788         if (!allowUnsafe && it->second.unsafe())
 789                 return 0;
 790         return it != encodinglist.end() ? &it->second : 0;
 791 }
 792
 793
 794 Encoding const *
 795 Encodings::fromLaTeXName(string const & n, int const & p, bool allowUnsafe) const
 796 {
 797         string name = n;
 798         // FIXME: if we have to test for too many of these synonyms,
 799         // we should instead extend the format of lib/encodings
 800         if (n == "ansinew")
 801                 name = "cp1252";
 802
 803         // We don't use find_if because it makes copies of the pairs in
 804         // the map.
 805         // This linear search is OK since we don't have many encodings.
 806         // Users could even optimize it by putting the encodings they use
 807         // most at the top of lib/encodings.
 808         EncodingList::const_iterator const end = encodinglist.end();
 809         for (EncodingList::const_iterator it = encodinglist.begin(); it != end; ++it)
 810                 if ((it->second.latexName() == name) && (it->second.package() & p)
 811                                 && (!it->second.unsafe() || allowUnsafe))
 812                         return &it->second;
 813         return 0;
 814 }
 815
 816
 817 Encoding const *
 818 Encodings::fromIconvName(string const & n, int const & p, bool allowUnsafe) const
 819 {
 820         EncodingList::const_iterator const end = encodinglist.end();
 821         for (EncodingList::const_iterator it = encodinglist.begin(); it != end; ++it)
 822                 if ((it->second.iconvName() == n) && (it->second.package() & p)
 823                                 && (!it->second.unsafe() || allowUnsafe))
 824                         return &it->second;
 825         return 0;
 826 }
 827
 828
 829 Encodings::Encodings()
 830 {}
 831
 832
 833 void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
 834 {
 835         // We must read the symbolsfile first, because the Encoding
 836         // constructor depends on it.
 837         CharSetMap forcednotselected;
 838         Lexer symbolslex;
 839         symbolslex.setFile(symbolsfile);
 840         bool getNextToken = true;
 841         while (symbolslex.isOK()) {
 842                 char_type symbol;
 843                 CharInfo info;
 844                 string flags;
 845
 846                 if (getNextToken) {
 847                         if (!symbolslex.next(true))
 848                                 break;
 849                 } else
 850                         getNextToken = true;
 851
 852                 istringstream is(symbolslex.getString());
 853                 // reading symbol directly does not work if
 854                 // char_type == wchar_t.
 855                 boost::uint32_t tmp;
 856                 if(!(is >> hex >> tmp))
 857                         break;
 858                 symbol = tmp;
 859
 860                 if (!symbolslex.next(true))
 861                         break;
 862                 info.textcommand = symbolslex.getDocString();
 863                 if (!symbolslex.next(true))
 864                         break;
 865                 info.textpreamble = symbolslex.getString();
 866                 if (!symbolslex.next(true))
 867                         break;
 868                 flags = symbolslex.getString();
 869
 870                 info.flags = 0;
 871                 if (suffixIs(info.textcommand, '}'))
 872                         info.flags |= CharInfoTextNoTermination;
 873                 if (suffixIs(info.mathcommand, '}'))
 874                         info.flags |= CharInfoMathNoTermination;
 875                 while (!flags.empty()) {
 876                         string flag;
 877                         flags = split(flags, flag, ',');
 878                         if (flag == "combining") {
 879                                 info.flags |= CharInfoCombining;
 880                         } else if (flag == "force") {
 881                                 info.flags |= CharInfoForce;
 882                                 forced.insert(symbol);
 883                         } else if (prefixIs(flag, "force=")) {
 884                                 vector<string> encodings =
 885                                         getVectorFromString(flag.substr(6), ";");
 886                                 for (size_t i = 0; i < encodings.size(); ++i)
 887                                         forcedselected[encodings[i]].insert(symbol);
 888                                 info.flags |= CharInfoForceSelected;
 889                         } else if (prefixIs(flag, "force!=")) {
 890                                 vector<string> encodings =
 891                                         getVectorFromString(flag.substr(7), ";");
 892                                 for (size_t i = 0; i < encodings.size(); ++i)
 893                                         forcednotselected[encodings[i]].insert(symbol);
 894                                 info.flags |= CharInfoForceSelected;
 895                         } else if (flag == "mathalpha") {
 896                                 mathalpha.insert(symbol);
 897                         } else if (flag == "notermination=text") {
 898                                 info.flags |= CharInfoTextNoTermination;
 899                         } else if (flag == "notermination=math") {
 900                                 info.flags |= CharInfoMathNoTermination;
 901                         } else if (flag == "notermination=both") {
 902                                 info.flags |= CharInfoTextNoTermination;
 903                                 info.flags |= CharInfoMathNoTermination;
 904                         } else if (flag == "notermination=none") {
 905                                 info.flags &= ~CharInfoTextNoTermination;
 906                                 info.flags &= ~CharInfoMathNoTermination;
 907                         } else if (contains(flag, "tipashortcut=")) {
 908                                 info.tipashortcut = split(flag, '=');
 909                         } else {
 910                                 lyxerr << "Ignoring unknown flag `" << flag
 911                                        << "' for symbol `0x"
 912                                        << hex << symbol << dec
 913                                        << "'." << endl;
 914                         }
 915                 }
 916                 // mathcommand and mathpreamble have been added for 1.6.0.
 917                 // make them optional so that old files still work.
 918                 int const lineno = symbolslex.lineNumber();
 919                 bool breakout = false;
 920                 if (symbolslex.next(true)) {
 921                         if (symbolslex.lineNumber() != lineno) {
 922                                 // line in old format without mathcommand and mathpreamble
 923                                 getNextToken = false;
 924                         } else {
 925                                 info.mathcommand = symbolslex.getDocString();
 926                                 if (symbolslex.next(true)) {
 927                                         if (symbolslex.lineNumber() != lineno) {
 928                                                 // line in new format with mathcommand only
 929                                                 getNextToken = false;
 930                                         } else {
 931                                                 // line in new format with mathcommand and mathpreamble
 932                                                 info.mathpreamble = symbolslex.getString();
 933                                         }
 934                                 } else
 935                                         breakout = true;
 936                         }
 937                 } else {
 938                         breakout = true;
 939                 }
 940
 941                 // backward compatibility
 942                 if (info.mathpreamble == "esintoramsmath")
 943                         info.mathpreamble = "esint|amsmath";
 944
 945                 if (!info.textpreamble.empty())
 946                         if (info.textpreamble[0] != '\\')
 947                                 info.flags |= CharInfoTextFeature;
 948                 if (!info.mathpreamble.empty())
 949                         if (info.mathpreamble[0] != '\\')
 950                                 info.flags |= CharInfoMathFeature;
 951
 952                 LYXERR(Debug::INFO, "Read unicode symbol " << symbol << " '"
 953                         << to_utf8(info.textcommand) << "' '" << info.textpreamble
 954                         << " '" << info.textfeature() << ' ' << info.textnotermination()
 955                         << ' ' << to_utf8(info.mathcommand) << "' '" << info.mathpreamble
 956                         << "' " << info.mathfeature() << ' ' << info.mathnotermination()
 957                         << ' ' << info.combining() << ' ' << info.force()
 958                         << ' ' << info.forceselected());
 959
 960                 // we assume that at least one command is nonempty when using unicodesymbols
 961                 if (!info.textcommand.empty() || !info.mathcommand.empty())
 962                         unicodesymbols[symbol] = info;
 963
 964                 if (breakout)
 965                         break;
 966         }
 967
 968         // Now read the encodings
 969         enum {
 970                 et_encoding = 1,
 971                 et_end
 972         };
 973
 974         LexerKeyword encodingtags[] = {
 975                 { "encoding", et_encoding },
 976                 { "end", et_end }
 977         };
 978
 979         Lexer lex(encodingtags);
 980         lex.setFile(encfile);
 981         lex.setContext("Encodings::read");
 982         while (lex.isOK()) {
 983                 switch (lex.lex()) {
 984                 case et_encoding:
 985                 {
 986                         lex.next();
 987                         string const name = lex.getString();
 988                         lex.next();
 989                         string const latexname = lex.getString();
 990                         lex.next();
 991                         string const guiname = lex.getString();
 992                         lex.next();
 993                         string const iconvname = lex.getString();
 994                         lex.next();
 995                         string const width = lex.getString();
 996                         bool fixedwidth = false;
 997                         bool unsafe = false;
 998                         if (width == "fixed")
 999                                 fixedwidth = true;
1000                         else if (width == "variable")
1001                                 fixedwidth = false;
1002                         else if (width == "variableunsafe") {
1003                                 fixedwidth = false;
1004                                 unsafe = true;
1005                         }
1006                         else
1007                                 lex.printError("Unknown width");
1008
1009                         lex.next();
1010                         string const p = lex.getString();
1011                         Encoding::Package package = Encoding::none;
1012                         if (p == "none")
1013                                 package = Encoding::none;
1014                         else if (p == "inputenc")
1015                                 package = Encoding::inputenc;
1016                         else if (p == "CJK")
1017                                 package = Encoding::CJK;
1018                         else if (p == "japanese")
1019                                 package = Encoding::japanese;
1020                         else
1021                                 lex.printError("Unknown package");
1022
1023                         LYXERR(Debug::INFO, "Reading encoding " << name);
1024                         encodinglist[name] = Encoding(name, latexname,
1025                                 guiname, iconvname, fixedwidth, unsafe,
1026                                 package);
1027
1028                         if (lex.lex() != et_end)
1029                                 lex.printError("Missing end");
1030                         break;
1031                 }
1032                 case et_end:
1033                         lex.printError("Misplaced end");
1034                         break;
1035                 case Lexer::LEX_FEOF:
1036                         break;
1037                 default:
1038                         lex.printError("Unknown tag");
1039                         break;
1040                 }
1041         }
1042
1043         // Move all information from forcednotselected to forcedselected
1044         for (CharSetMap::const_iterator it1 = forcednotselected.begin(); it1 != forcednotselected.end(); ++it1) {
1045                 for (CharSetMap::iterator it2 = forcedselected.begin(); it2 != forcedselected.end(); ++it2) {
1046                         if (it2->first != it1->first)
1047                                 it2->second.insert(it1->second.begin(), it1->second.end());
1048                 }
1049         }
1050
1051 }
1052
1053
1054 } // namespace lyx