]> git.lyx.org Git - lyx.git/blobdiff - src/Encoding.cpp
Update fr.po
[lyx.git] / src / Encoding.cpp
index 98ad8519effb100e98d5672d278bdcf1ab17b8d0..dab24b72ddecf3c2c6eb0a8ba170cab1cf905e92 100644 (file)
 
 #include "Encoding.h"
 
-#include "Buffer.h"
-#include "BufferList.h"
-#include "InsetIterator.h"
-#include "LaTeXFeatures.h"
 #include "Lexer.h"
-#include "LyXRC.h"
 
 #include "support/debug.h"
+#include "support/docstring.h"
 #include "support/gettext.h"
-#include "support/FileName.h"
 #include "support/lstrings.h"
+#include "support/mutex.h"
 #include "support/textutils.h"
 #include "support/unicode.h"
 
-#include <boost/cstdint.hpp>
-
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
 #include <sstream>
 
 using namespace std;
@@ -47,257 +44,22 @@ Encodings::MathSymbolSet  Encodings::mathsym;
 
 namespace {
 
-char_type arabic_table[172][4] = {
-       {0xfe80, 0xfe80, 0xfe80, 0xfe80}, // 0x0621 = hamza
-       {0xfe81, 0xfe82, 0xfe81, 0xfe82}, // 0x0622 = ligature madda on alef
-       {0xfe83, 0xfe84, 0xfe83, 0xfe84}, // 0x0623 = ligature hamza on alef
-       {0xfe85, 0xfe86, 0xfe85, 0xfe86}, // 0x0624 = ligature hamza on waw
-       {0xfe87, 0xfe88, 0xfe87, 0xfe88}, // 0x0625 = ligature hamza under alef
-       {0xfe89, 0xfe8a, 0xfe8b, 0xfe8c}, // 0x0626 = ligature hamza on ya
-       {0xfe8d, 0xfe8e, 0xfe8d, 0xfe8e}, // 0x0627 = alef
-       {0xfe8f, 0xfe90, 0xfe91, 0xfe92}, // 0x0628 = baa
-       {0xfe93, 0xfe94, 0xfe93, 0xfe94}, // 0x0629 = taa marbuta
-       {0xfe95, 0xfe96, 0xfe97, 0xfe98}, // 0x062a = taa
-       {0xfe99, 0xfe9a, 0xfe9b, 0xfe9c}, // 0x062b = thaa
-       {0xfe9d, 0xfe9e, 0xfe9f, 0xfea0}, // 0x062c = jeem
-       {0xfea1, 0xfea2, 0xfea3, 0xfea4}, // 0x062d = haa
-       {0xfea5, 0xfea6, 0xfea7, 0xfea8}, // 0x062e = khaa
-       {0xfea9, 0xfeaa, 0xfea9, 0xfeaa}, // 0x062f = dal
-
-       {0xfeab, 0xfeac, 0xfeab, 0xfeac}, // 0x0630 = thal
-       {0xfead, 0xfeae, 0xfead, 0xfeae}, // 0x0631 = ra
-       {0xfeaf, 0xfeb0, 0xfeaf, 0xfeb0}, // 0x0632 = zain
-       {0xfeb1, 0xfeb2, 0xfeb3, 0xfeb4}, // 0x0633 = seen
-       {0xfeb5, 0xfeb6, 0xfeb7, 0xfeb8}, // 0x0634 = sheen
-       {0xfeb9, 0xfeba, 0xfebb, 0xfebc}, // 0x0635 = sad
-       {0xfebd, 0xfebe, 0xfebf, 0xfec0}, // 0x0636 = dad
-       {0xfec1, 0xfec2, 0xfec3, 0xfec4}, // 0x0637 = tah
-       {0xfec5, 0xfec6, 0xfec7, 0xfec8}, // 0x0638 = zah
-       {0xfec9, 0xfeca, 0xfecb, 0xfecc}, // 0x0639 = ain
-       {0xfecd, 0xfece, 0xfecf, 0xfed0}, // 0x063a = ghain
-       {0, 0, 0, 0}, // 0x063b
-       {0, 0, 0, 0}, // 0x063c
-       {0, 0, 0, 0}, // 0x063d
-       {0, 0, 0, 0}, // 0x063e
-       {0, 0, 0, 0}, // 0x063f
-
-       {0, 0, 0, 0}, // 0x0640
-       {0xfed1, 0xfed2, 0xfed3, 0xfed4}, // 0x0641 = fa
-       {0xfed5, 0xfed6, 0xfed7, 0xfed8}, // 0x0642 = qaf
-       {0xfed9, 0xfeda, 0xfedb, 0xfedc}, // 0x0643 = kaf
-       {0xfedd, 0xfede, 0xfedf, 0xfee0}, // 0x0644 = lam
-       {0xfee1, 0xfee2, 0xfee3, 0xfee4}, // 0x0645 = meem
-       {0xfee5, 0xfee6, 0xfee7, 0xfee8}, // 0x0646 = noon
-       {0xfee9, 0xfeea, 0xfeeb, 0xfeec}, // 0x0647 = ha
-       {0xfeed, 0xfeee, 0xfeed, 0xfeee}, // 0x0648 = waw
-       {0xfeef, 0xfef0, 0xfeef, 0xfef0}, // 0x0649 = alef maksura
-       {0xfef1, 0xfef2, 0xfef3, 0xfef4}, // 0x064a = ya
-       {0x065b, 0x065b, 0x065b, 0x065b}, // 0x064b = fathatan
-       {0x065c, 0x065c, 0x065c, 0x065c}, // 0x064c = dammatan
-       {0x064d, 0x064d, 0x064d, 0x064d}, // 0x064d = kasratan
-       {0x064e, 0x064e, 0x064e, 0x064e}, // 0x064e = fatha
-       {0x064f, 0x064f, 0x064f, 0x064f}, // 0x064f = damma
-
-       {0x0650, 0x0650, 0x0650, 0x0650}, // 0x0650 = kasra
-       {0x0651, 0x0651, 0x0651, 0x0651}, // 0x0651 = shadda
-       {0x0652, 0x0652, 0x0652, 0x0652}, // 0x0652 = sukun
-
-       {0, 0, 0, 0}, // 0x0653
-       {0, 0, 0, 0}, // 0x0654
-       {0, 0, 0, 0}, // 0x0655
-       {0, 0, 0, 0}, // 0x0656
-       {0, 0, 0, 0}, // 0x0657
-       {0, 0, 0, 0}, // 0x0658
-       {0, 0, 0, 0}, // 0x0659
-       {0, 0, 0, 0}, // 0x065a
-       {0, 0, 0, 0}, // 0x065b
-       {0, 0, 0, 0}, // 0x065c
-       {0, 0, 0, 0}, // 0x065d
-       {0, 0, 0, 0}, // 0x065e
-       {0, 0, 0, 0}, // 0x065f
-       {0, 0, 0, 0}, // 0x0660
-       {0, 0, 0, 0}, // 0x0661
-       {0, 0, 0, 0}, // 0x0662
-       {0, 0, 0, 0}, // 0x0663
-       {0, 0, 0, 0}, // 0x0664
-       {0, 0, 0, 0}, // 0x0665
-       {0, 0, 0, 0}, // 0x0666
-       {0, 0, 0, 0}, // 0x0667
-       {0, 0, 0, 0}, // 0x0668
-       {0, 0, 0, 0}, // 0x0669
-       {0, 0, 0, 0}, // 0x066a
-       {0, 0, 0, 0}, // 0x066b
-       {0, 0, 0, 0}, // 0x066c
-       {0, 0, 0, 0}, // 0x066d
-       {0, 0, 0, 0}, // 0x066e
-       {0, 0, 0, 0}, // 0x066f
-       {0, 0, 0, 0}, // 0x0670
-       {0, 0, 0, 0}, // 0x0671
-       {0, 0, 0, 0}, // 0x0672
-       {0, 0, 0, 0}, // 0x0673
-       {0, 0, 0, 0}, // 0x0674
-       {0, 0, 0, 0}, // 0x0675
-       {0, 0, 0, 0}, // 0x0676
-       {0, 0, 0, 0}, // 0x0677
-       {0, 0, 0, 0}, // 0x0678
-       {0, 0, 0, 0}, // 0x0679
-       {0, 0, 0, 0}, // 0x067a
-       {0, 0, 0, 0}, // 0x067b
-       {0, 0, 0, 0}, // 0x067c
-       {0, 0, 0, 0}, // 0x067d
-       {0xfb56, 0xfb57, 0xfb58, 0xfb59}, // 0x067e = peh
-       {0, 0, 0, 0}, // 0x067f
-       {0, 0, 0, 0}, // 0x0680
-       {0, 0, 0, 0}, // 0x0681
-       {0, 0, 0, 0}, // 0x0682
-       {0, 0, 0, 0}, // 0x0683
-       {0, 0, 0, 0}, // 0x0684
-       {0, 0, 0, 0}, // 0x0685
-       {0xfb7a, 0xfb7b, 0xfb7c, 0xfb7d}, // 0x0686 = tcheh
-       {0, 0, 0, 0}, // 0x0687
-       {0, 0, 0, 0}, // 0x0688
-       {0, 0, 0, 0}, // 0x0689
-       {0, 0, 0, 0}, // 0x068a
-       {0, 0, 0, 0}, // 0x068b
-       {0, 0, 0, 0}, // 0x068c
-       {0, 0, 0, 0}, // 0x068d
-       {0, 0, 0, 0}, // 0x068e
-       {0, 0, 0, 0}, // 0x068f
-       {0, 0, 0, 0}, // 0x0690
-       {0, 0, 0, 0}, // 0x0691
-       {0, 0, 0, 0}, // 0x0692
-       {0, 0, 0, 0}, // 0x0693
-       {0, 0, 0, 0}, // 0x0694
-       {0, 0, 0, 0}, // 0x0695
-       {0, 0, 0, 0}, // 0x0696
-       {0, 0, 0, 0}, // 0x0697
-       {0xfb8a, 0xfb8b, 0xfb8a, 0xfb8b}, // 0x0698 = jeh
-       {0, 0, 0, 0}, // 0x0699
-       {0, 0, 0, 0}, // 0x069a
-       {0, 0, 0, 0}, // 0x069b
-       {0, 0, 0, 0}, // 0x069c
-       {0, 0, 0, 0}, // 0x069d
-       {0, 0, 0, 0}, // 0x069e
-       {0, 0, 0, 0}, // 0x069f
-       {0, 0, 0, 0}, // 0x06a0
-       {0, 0, 0, 0}, // 0x06a1
-       {0, 0, 0, 0}, // 0x06a2
-       {0, 0, 0, 0}, // 0x06a3
-       {0, 0, 0, 0}, // 0x06a4
-       {0, 0, 0, 0}, // 0x06a5
-       {0, 0, 0, 0}, // 0x06a6
-       {0, 0, 0, 0}, // 0x06a7
-       {0, 0, 0, 0}, // 0x06a8
-       {0xfb8e, 0xfb8f, 0xfb90, 0xfb91}, // 0x06a9 = farsi kaf
-       {0, 0, 0, 0}, // 0x06aa
-       {0, 0, 0, 0}, // 0x06ab
-       {0, 0, 0, 0}, // 0x06ac
-       {0, 0, 0, 0}, // 0x06ad
-       {0, 0, 0, 0}, // 0x06ae
-       {0xfb92, 0xfb93, 0xfb94, 0xfb95}, // 0x06af = gaf
-       {0, 0, 0, 0}, // 0x06b0
-       {0, 0, 0, 0}, // 0x06b1
-       {0, 0, 0, 0}, // 0x06b2
-       {0, 0, 0, 0}, // 0x06b3
-       {0, 0, 0, 0}, // 0x06b4
-       {0, 0, 0, 0}, // 0x06b5
-       {0, 0, 0, 0}, // 0x06b6
-       {0, 0, 0, 0}, // 0x06b7
-       {0, 0, 0, 0}, // 0x06b8
-       {0, 0, 0, 0}, // 0x06b9
-       {0, 0, 0, 0}, // 0x06ba
-       {0, 0, 0, 0}, // 0x06bb
-       {0, 0, 0, 0}, // 0x06bc
-       {0, 0, 0, 0}, // 0x06bd
-       {0, 0, 0, 0}, // 0x06be
-       {0, 0, 0, 0}, // 0x06bf
-       {0, 0, 0, 0}, // 0x06c0
-       {0, 0, 0, 0}, // 0x06c1
-       {0, 0, 0, 0}, // 0x06c2
-       {0, 0, 0, 0}, // 0x06c3
-       {0, 0, 0, 0}, // 0x06c4
-       {0, 0, 0, 0}, // 0x06c5
-       {0, 0, 0, 0}, // 0x06c6
-       {0, 0, 0, 0}, // 0x06c7
-       {0, 0, 0, 0}, // 0x06c8
-       {0, 0, 0, 0}, // 0x06c9
-       {0, 0, 0, 0}, // 0x06ca
-       {0, 0, 0, 0}, // 0x06cb
-       {0xfbfc, 0xfbfd, 0xfbfe, 0xfbff} // 0x06cc = farsi yeh
-};
-
-
-char_type const arabic_start = 0x0621;
-char_type const arabic_end = 0x06cc;
-
-
-enum CharInfoFlags {
-       ///
-       CharInfoCombining = 1,
-       ///
-       CharInfoTextFeature = 2,
-       ///
-       CharInfoMathFeature = 4,
-       ///
-       CharInfoForce = 8,
-       ///
-       CharInfoTextNoTermination = 16,
-       ///
-       CharInfoMathNoTermination = 32,
-       ///
-       CharInfoForceSelected = 64,
-};
-
-/// Information about a single UCS4 character
-struct CharInfo {
-       /// LaTeX command (text mode) for this character
-       docstring textcommand;
-       /// LaTeX command (math mode) for this character
-       docstring mathcommand;
-       /// Needed LaTeX preamble (or feature) for text mode
-       string textpreamble;
-       /// Needed LaTeX preamble (or feature) for math mode
-       string mathpreamble;
-       /// Is this a combining character?
-       bool combining() const { return flags & CharInfoCombining ? true : false; }
-       /// Is \c textpreamble a feature known by LaTeXFeatures, or a raw LaTeX
-       /// command?
-       bool textfeature() const { return flags & CharInfoTextFeature ? true : false; }
-       /// Is \c mathpreamble a feature known by LaTeXFeatures, or a raw LaTeX
-       /// command?
-       bool mathfeature() const { return flags & CharInfoMathFeature ? true : false; }
-       /// Always force the LaTeX command, even if the encoding contains
-       /// this character?
-       bool force() const { return flags & CharInfoForce ? true : false; }
-       /// Force the LaTeX command for some encodings?
-       bool forceselected() const { return flags & CharInfoForceSelected ? true : false; }
-       /// TIPA shortcut
-       string tipashortcut;
-       /// \c textcommand needs no termination (such as {} or space).
-       bool textnotermination() const { return flags & CharInfoTextNoTermination ? true : false; }
-       /// \c mathcommand needs no termination (such as {} or space).
-       bool mathnotermination() const { return flags & CharInfoMathNoTermination ? true : false; }
-       ///
-       unsigned int flags;
-};
-
-
 typedef map<char_type, CharInfo> CharInfoMap;
 CharInfoMap unicodesymbols;
 
-typedef std::set<char_type> CharSet;
-typedef std::map<string, CharSet> CharSetMap;
+typedef set<char_type> CharSet;
+typedef map<string, CharSet> CharSetMap;
 CharSet forced;
-CharSetMap forcedselected;
+CharSetMap forcedSelected;
 
-typedef std::set<char_type> MathAlphaSet;
+typedef set<char_type> MathAlphaSet;
 MathAlphaSet mathalpha;
 
 
 /// The highest code point in UCS4 encoding (1<<20 + 1<<16)
 char_type const max_ucs4 = 0x110000;
 
-} // namespace anon
+} // namespace
 
 
 EncodingException::EncodingException(char_type c)
@@ -306,16 +68,38 @@ EncodingException::EncodingException(char_type c)
 }
 
 
-const char * EncodingException::what() const throw()
+const char * EncodingException::what() const noexcept
 {
        return "Could not find LaTeX command for a character";
 }
 
 
+CharInfo::CharInfo(
+               docstring const & text_command, docstring const & math_command,
+               std::string const & text_preamble, std::string const & math_preamble,
+               std::string const & tipa_shortcut, unsigned int flags)
+       : text_commands_({text_command}), math_commands_({math_command}),
+         text_preamble_(text_preamble), math_preamble_(math_preamble),
+         tipa_shortcut_(tipa_shortcut), flags_(flags)
+{
+}
+
+
+CharInfo::CharInfo(
+               std::vector<docstring> const & text_commands, std::vector<docstring> const & math_commands,
+               std::string const & text_preamble, std::string const & math_preamble,
+               std::string const & tipa_shortcut, unsigned int flags)
+       : text_commands_(text_commands), math_commands_(math_commands),
+         text_preamble_(text_preamble), math_preamble_(math_preamble),
+         tipa_shortcut_(tipa_shortcut), flags_(flags)
+{
+}
+
+
 Encoding::Encoding(string const & n, string const & l, string const & g,
                   string const & i, bool f, bool u, Encoding::Package p)
        : name_(n), latexName_(l), guiName_(g), iconvName_(i), fixedwidth_(f),
-         unsafe_(u), forced_(&forcedselected[n]), package_(p)
+         unsafe_(u), forced_(&forcedSelected[n]), package_(p)
 {
        if (n == "ascii") {
                // ASCII can encode 128 code points and nothing else
@@ -334,10 +118,28 @@ Encoding::Encoding(string const & n, string const & l, string const & g,
 
 void Encoding::init() const
 {
+       // Since the the constructor is the only method which sets complete_
+       // to false the test for complete_ is thread-safe without mutex.
+       if (complete_)
+               return;
+
+       static Mutex mutex;
+       Mutex::Locker lock(&mutex);
+
+       // We need to test again for complete_, since another thread could
+       // have set it to true while we were waiting for the lock and we must
+       // not modify an encoding which is already complete.
        if (complete_)
                return;
 
-       start_encodable_ = 0;
+       // We do not make any member mutable  so that it can be easily verified
+       // that all const methods are thread-safe: init() is the only const
+       // method which changes complete_, encodable_ and start_encodable_, and
+       // it uses a mutex to ensure thread-safety.
+       CharSet & encodable = const_cast<Encoding *>(this)->encodable_;
+       char_type & start_encodable = const_cast<Encoding *>(this)->start_encodable_;
+
+       start_encodable = 0;
        // temporarily switch off lyxerr, since we will generate iconv errors
        lyxerr.disable();
        if (fixedwidth_) {
@@ -351,10 +153,10 @@ void Encoding::init() const
                        char_type const uc = ucs4[0];
                        CharInfoMap::const_iterator const it = unicodesymbols.find(uc);
                        if (it == unicodesymbols.end())
-                               encodable_.insert(uc);
+                               encodable.insert(uc);
                        else if (!it->second.force()) {
                                if (forced_->empty() || forced_->find(uc) == forced_->end())
-                                       encodable_.insert(uc);
+                                       encodable.insert(uc);
                        }
                }
        } else {
@@ -367,22 +169,22 @@ void Encoding::init() const
                        if (!eightbit.empty()) {
                                CharInfoMap::const_iterator const it = unicodesymbols.find(c);
                                if (it == unicodesymbols.end())
-                                       encodable_.insert(c);
+                                       encodable.insert(c);
                                else if (!it->second.force()) {
                                        if (forced_->empty() || forced_->find(c) == forced_->end())
-                                               encodable_.insert(c);
+                                               encodable.insert(c);
                                }
                        }
                }
        }
        lyxerr.enable();
-       CharSet::iterator it = encodable_.find(start_encodable_);
-       while (it != encodable_.end()) {
-               encodable_.erase(it);
-               ++start_encodable_;
-               it = encodable_.find(start_encodable_);
+       CharSet::iterator it = encodable.find(start_encodable);
+       while (it != encodable.end()) {
+               encodable.erase(it);
+               ++start_encodable;
+               it = encodable.find(start_encodable);
        }
-       complete_ = true;
+       const_cast<Encoding *>(this)->complete_ = true;
 }
 
 
@@ -398,9 +200,16 @@ bool Encoding::encodable(char_type c) const
 {
        // assure the used encoding is properly initialized
        init();
-
        if (iconvName_ == "UTF-8" && package_ == none)
                return true;
+       // platex does not load inputenc: force conversion of supported characters
+       if (package_ == Encoding::japanese
+           && ((0xb7 <= c && c <= 0x05ff) // Latin-1 Supplement ... Hebrew
+                       || (0x1d00 <= c && c <= 0x218f) // Phonetic Extensions ... Number Forms
+                       || (0x2193 <= c && c <= 0x2aff) // Arrows ... Supplemental Mathematical Operators
+                       || (0xfb00 <= c && c <= 0xfb4f) // Alphabetic Presentation Forms
+                       || (0x1d400 <= c && c <= 0x1d7ff))) // Mathematical Alphanumeric Symbols
+               return false;
        if (c < start_encodable_ && !isForced(c))
                return true;
        if (encodable_.find(c) != encodable_.end())
@@ -418,22 +227,21 @@ pair<docstring, bool> Encoding::latexChar(char_type c) const
        CharInfoMap::const_iterator const it = unicodesymbols.find(c);
        if (it == unicodesymbols.end())
                throw EncodingException(c);
-       // at least one of mathcommand and textcommand is nonempty
-       if (it->second.textcommand.empty())
+       // at least one of mathCommand and textCommand is nonempty
+       if (it->second.textCommand().empty())
                return make_pair(
-                       "\\ensuremath{" + it->second.mathcommand + '}', false);
-       return make_pair(it->second.textcommand, !it->second.textnotermination());
+                               "\\ensuremath{" + it->second.mathCommand() + '}', false);
+       return make_pair(it->second.textCommand(), !it->second.textNoTermination());
 }
 
 
-pair<docstring, docstring> Encoding::latexString(docstring const input, bool dryrun) const
+pair<docstring, docstring> Encoding::latexString(docstring const input, bool dryrun) const
 {
        docstring result;
        docstring uncodable;
        bool terminate = false;
-       for (size_t n = 0; n < input.size(); ++n) {
+       for (char_type const c : input) {
                try {
-                       char_type const c = input[n];
                        pair<docstring, bool> latex_char = latexChar(c);
                        docstring const latex = latex_char.first;
                        if (terminate && !prefixIs(latex, '\\')
@@ -450,14 +258,15 @@ pair<docstring, docstring> Encoding::latexString(docstring const input, bool dry
                        result += latex;
                        terminate = latex_char.second;
                } catch (EncodingException & /* e */) {
-                       LYXERR0("Uncodable character in latexString!");
+                       LYXERR0("Uncodable character <" << docstring(1, c) 
+                                       << "> in latexString!");
                        if (dryrun) {
                                result += "<" + _("LyX Warning: ")
                                           + _("uncodable character") + " '";
-                               result += docstring(1, input[n]);
+                               result += docstring(1, c);
                                result += "'>";
                        } else
-                               uncodable += input[n];
+                               uncodable += c;
                }
        }
        return make_pair(result, uncodable);
@@ -469,16 +278,19 @@ vector<char_type> Encoding::symbolsList() const
        // assure the used encoding is properly initialized
        init();
 
-       // first all encodable characters
-       vector<char_type> symbols(encodable_.begin(), encodable_.end());
-       // add those below start_encodable_
+       // first all those below start_encodable_
+       vector<char_type> symbols;
        for (char_type c = 0; c < start_encodable_; ++c)
                symbols.push_back(c);
-       // now the ones from the unicodesymbols file
-       CharInfoMap::const_iterator const end = unicodesymbols.end();
-       CharInfoMap::const_iterator it = unicodesymbols.begin();
-       for (; it != end; ++it)
-               symbols.push_back(it->first);
+       // add all encodable characters
+       copy(encodable_.begin(), encodable_.end(), back_inserter(symbols));
+       // now the ones from the unicodesymbols file that are not already there
+       for (auto const & elem : unicodesymbols) {
+               if (find(symbols.begin(), symbols.end(), elem.first) == symbols.end())
+                       symbols.push_back(elem.first);
+       }
+       // finally, sort the vector
+       sort(symbols.begin(), symbols.end());
        return symbols;
 }
 
@@ -501,21 +313,22 @@ bool Encodings::latexMathChar(char_type c, bool mathmode,
                        addMathSym(c);
                return false;
        }
-       // at least one of mathcommand and textcommand is nonempty
-       bool use_math = (mathmode && !it->second.mathcommand.empty()) ||
-                       (!mathmode && it->second.textcommand.empty());
+       // at least one of mathCommand and textCommand is nonempty
+       bool use_math = (mathmode && !it->second.mathCommand().empty()) ||
+                       (!mathmode && it->second.textCommand().empty());
        if (use_math) {
-               command = it->second.mathcommand;
-               needsTermination = !it->second.mathnotermination();
+               command = it->second.mathCommand();
+               needsTermination = !it->second.mathNoTermination();
                addMathCmd(c);
        } else {
                if (!encoding || command.empty()) {
-                       command = it->second.textcommand;
-                       needsTermination = !it->second.textnotermination();
-                       addTextCmd(c);
+                       command = it->second.textCommand();
+                       needsTermination = !it->second.textNoTermination();
                }
                if (mathmode)
                        addMathSym(c);
+               else
+                       addTextCmd(c);
        }
        return use_math;
 }
@@ -527,23 +340,33 @@ char_type Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
        CharInfoMap::const_iterator const end = unicodesymbols.end();
        CharInfoMap::const_iterator it = unicodesymbols.begin();
        for (combining = false; it != end; ++it) {
-               docstring const math = it->second.mathcommand;
-               docstring const text = it->second.textcommand;
-               if ((cmdtype & MATH_CMD) && math == cmd) {
-                       combining = it->second.combining();
-                       needsTermination = !it->second.mathnotermination();
-                       if (req && it->second.mathfeature() &&
-                           !it->second.mathpreamble.empty())
-                               req->insert(it->second.mathpreamble);
-                       return it->first;
+               if (it->second.deprecated())
+                       continue;
+
+               if (cmdtype & MATH_CMD) {
+                       for (const docstring& math : it->second.mathCommands()) {
+                               if ((cmdtype & MATH_CMD) && math == cmd) {
+                                       combining = it->second.combining();
+                                       needsTermination = !it->second.mathNoTermination();
+                                       if (req && it->second.mathFeature() &&
+                                           !it->second.mathPreamble().empty())
+                                               req->insert(it->second.mathPreamble());
+                                       return it->first;
+                               }
+                       }
                }
-               if ((cmdtype & TEXT_CMD) && text == cmd) {
-                       combining = it->second.combining();
-                       needsTermination = !it->second.textnotermination();
-                       if (req && it->second.textfeature() &&
-                           !it->second.textpreamble.empty())
-                               req->insert(it->second.textpreamble);
-                       return it->first;
+
+               if (cmdtype & TEXT_CMD) {
+                       for (const docstring& text : it->second.textCommands()) {
+                               if (text == cmd) {
+                                       combining = it->second.combining();
+                                       needsTermination = !it->second.textNoTermination();
+                                       if (req && it->second.textFeature() &&
+                                           !it->second.textPreamble().empty())
+                                               req->insert(it->second.textPreamble());
+                                       return it->first;
+                               }
+                       }
                }
        }
        needsTermination = false;
@@ -558,20 +381,69 @@ docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
        rem = empty_docstring();
        bool const mathmode = cmdtype & MATH_CMD;
        bool const textmode = cmdtype & TEXT_CMD;
+
+       // Easy case: the command is a complete entry of unicodesymbols.
+       for (const auto & unicodeSymbol : unicodesymbols) {
+               if (mathmode) {
+                       for (const auto & command : unicodeSymbol.second.mathCommands()) {
+                               if (command == cmd) {
+                                       docstring value;
+                                       value += unicodeSymbol.first;
+                                       needsTermination = !unicodeSymbol.second.mathNoTermination();
+                                       if (req && unicodeSymbol.second.mathFeature()
+                                               && !unicodeSymbol.second.mathPreamble().empty())
+                                                       req->insert(unicodeSymbol.second.mathPreamble());
+                                       return value;
+                               }
+                       }
+               }
+               if (textmode) {
+                       for (const auto & command : unicodeSymbol.second.textCommands()) {
+                               if (command == cmd) {
+                                       docstring value;
+                                       value += unicodeSymbol.first;
+                                       needsTermination = !unicodeSymbol.second.textNoTermination();
+                                       if (req && unicodeSymbol.second.textFeature()
+                                               && !unicodeSymbol.second.textPreamble().empty())
+                                               req->insert(unicodeSymbol.second.textPreamble());
+                                       return value;
+                               }
+                       }
+               }
+       }
+
+       // Otherwise, try to map as many commands as possible, matching prefixes of the command.
        docstring symbols;
-       size_t i = 0;
        size_t const cmdend = cmd.size();
+       size_t prefix = 0;
        CharInfoMap::const_iterator const uniend = unicodesymbols.end();
-       for (size_t j = 0; j < cmdend; ++j) {
+       for (size_t i = 0, j = 0; j < cmdend; ++j) {
                // Also get the char after a backslash
-               if (j + 1 < cmdend && cmd[j] == '\\')
+               if (j + 1 < cmdend && cmd[j] == '\\') {
                        ++j;
+                       prefix = 1;
+                       // Detect things like \=*{e} as well
+                       if (j + 3 < cmdend && cmd[j+1] == '*' &&
+                           cmd[j+2] == '{') {
+                               ++j;
+                               prefix = 2;
+                       }
+               }
+               // position of the last character before a possible macro
+               // argument
+               size_t m = j;
                // If a macro argument follows, get it, too
+               // Do it here only for single character commands. Other
+               // combining commands need this too, but they are handled in
+               // the loop below for performance reasons.
                if (j + 1 < cmdend && cmd[j + 1] == '{') {
                        size_t k = j + 1;
                        int count = 1;
-                       while (k < cmdend && count && k != docstring::npos) {
+                       while (k < cmdend && count) {
                                k = cmd.find_first_of(from_ascii("{}"), k + 1);
+                               // braces may not be balanced
+                               if (k == docstring::npos)
+                                       break;
                                if (cmd[k] == '{')
                                        ++count;
                                else
@@ -579,19 +451,31 @@ docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
                        }
                        if (k != docstring::npos)
                                j = k;
+               } else if (m + 1 < cmdend && isAlphaASCII(cmd[m])) {
+                       while (m + 2 < cmdend && isAlphaASCII(cmd[m+1]))
+                               m++;
                }
                // Start with this substring and try augmenting it when it is
                // the prefix of some command in the unicodesymbols file
-               docstring const subcmd = cmd.substr(i, j - i + 1);
+               docstring subcmd = cmd.substr(i, j - i + 1);
 
-               CharInfoMap::const_iterator it = unicodesymbols.begin();
+               // First part of subcmd which might be a combining character
+               docstring combcmd = (m == j) ? docstring() : cmd.substr(i, m - i + 1);
+               // The combining character of combcmd if it exists
                size_t unicmd_size = 0;
                char_type c = 0;
+               CharInfoMap::const_iterator it = unicodesymbols.begin();
+               CharInfoMap::const_iterator combining = uniend;
                for (; it != uniend; ++it) {
-                       docstring const math = mathmode ? it->second.mathcommand
+                       if (it->second.deprecated())
+                               continue;
+                       docstring const math = mathmode ? it->second.mathCommand()
                                                        : docstring();
-                       docstring const text = textmode ? it->second.textcommand
+                       docstring const text = textmode ? it->second.textCommand()
                                                        : docstring();
+                       if (!combcmd.empty() && it->second.combining() &&
+                           (math == combcmd || text == combcmd))
+                               combining = it;
                        size_t cur_size = max(math.size(), text.size());
                        // The current math or text unicode command cannot
                        // match, or we already matched a longer one
@@ -618,6 +502,29 @@ docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
 
                        // If this is an exact match, we found a (longer)
                        // matching entry in the unicodesymbols file.
+                       if (math != tmp && text != tmp)
+                               continue;
+                       // If we found a combining command, we need to append
+                       // the macro argument if this has not been done above.
+                       if (tmp == combcmd && combining != uniend &&
+                           k < cmdend && cmd[k] == '{') {
+                               size_t l = k;
+                               int count = 1;
+                               while (l < cmdend && count) {
+                                       l = cmd.find_first_of(from_ascii("{}"), l + 1);
+                                       // braces may not be balanced
+                                       if (l == docstring::npos)
+                                               break;
+                                       if (cmd[l] == '{')
+                                               ++count;
+                                       else
+                                               --count;
+                               }
+                               if (l != docstring::npos) {
+                                       j = l;
+                                       subcmd = cmd.substr(i, j - i + 1);
+                               }
+                       }
                        // If the entry doesn't start with '\', we take note
                        // of the match and continue (this is not a ultimate
                        // acceptance, as some other entry may match a longer
@@ -627,34 +534,68 @@ docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
                        // (nonletter) char macro, or nothing else follows,
                        // or what follows is a nonletter char, or the last
                        // character is a }.
-                       if ((math == tmp || text == tmp)
-                           && (tmp[0] != '\\'
-                                  || (tmp.size() == 2 && !isAlphaASCII(tmp[1]))
-                                  || k == cmdend 
+                       else if (tmp[0] != '\\'
+                                  || (tmp.size() == prefix + 1 &&
+                                      !isAlphaASCII(tmp[1]) &&
+                                      (prefix == 1 || !isAlphaASCII(tmp[2])))
+                                  || k == cmdend
                                   || !isAlphaASCII(cmd[k])
-                                  || tmp[tmp.size() - 1] == '}')
+                                  || tmp[tmp.size() - 1] == '}'
                                 ) {
                                c = it->first;
                                j = k - 1;
                                i = j + 1;
                                unicmd_size = cur_size;
                                if (math == tmp)
-                                       needsTermination = !it->second.mathnotermination();
+                                       needsTermination = !it->second.mathNoTermination();
                                else
-                                       needsTermination = !it->second.textnotermination();
+                                       needsTermination = !it->second.textNoTermination();
                                if (req) {
-                                       if (math == tmp && it->second.mathfeature() &&
-                                           !it->second.mathpreamble.empty())
-                                               req->insert(it->second.mathpreamble);
-                                       if (text == tmp && it->second.textfeature() &&
-                                           !it->second.textpreamble.empty())
-                                               req->insert(it->second.textpreamble);
+                                       if (math == tmp && it->second.mathFeature() &&
+                                           !it->second.mathPreamble().empty())
+                                               req->insert(it->second.mathPreamble());
+                                       if (text == tmp && it->second.textFeature() &&
+                                           !it->second.textPreamble().empty())
+                                               req->insert(it->second.textPreamble());
                                }
                        }
                }
                if (unicmd_size)
                        symbols += c;
-               else if (j + 1 == cmdend) {
+               else if (combining != uniend &&
+                        prefixIs(subcmd, combcmd + '{')) {
+                       // We know that subcmd starts with combcmd and
+                       // contains an argument in braces.
+                       docstring const arg = subcmd.substr(
+                               combcmd.length() + 1,
+                               subcmd.length() - combcmd.length() - 2);
+                       // If arg is a single character we can construct a
+                       // combining sequence.
+                       char_type a;
+                       bool argcomb = false;
+                       if (arg.size() == 1 && isAlnumASCII(arg[0]))
+                               a = arg[0];
+                       else {
+                               // Use the version of fromLaTeXCommand() that
+                               // parses only one command, since we cannot
+                               // use more than one character.
+                               bool dummy = false;
+                               set<string> r;
+                               a = fromLaTeXCommand(arg, cmdtype, argcomb,
+                                                    dummy, &r);
+                               if (a && req && !argcomb)
+                                       req->insert(r.begin(), r.end());
+                       }
+                       if (a && !argcomb) {
+                               // In unicode the combining character comes
+                               // after its base
+                               symbols += a;
+                               symbols += combining->first;
+                               i = j + 1;
+                               unicmd_size = 2;
+                       }
+               }
+               if (j + 1 == cmdend && !unicmd_size) {
                        // No luck. Return what remains
                        rem = cmd.substr(i);
                        if (needsTermination && !rem.empty()) {
@@ -673,131 +614,11 @@ docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
 }
 
 
-void Encodings::initUnicodeMath(Buffer const & buffer, bool for_master)
+CharInfo const & Encodings::unicodeCharInfo(char_type c)
 {
-#ifdef TEX2LYX
-       // The code below is not needed in tex2lyx and requires additional stuff
-       (void)buffer;
-       (void)for_master;
-#else
-       if (for_master) {
-               mathcmd.clear();
-               textcmd.clear();
-               mathsym.clear();
-       }
-
-       // Check this buffer
-       Inset & inset = buffer.inset();
-       InsetIterator it = inset_iterator_begin(inset);
-       InsetIterator const end = inset_iterator_end(inset);
-       for (; it != end; ++it)
-               it->initUnicodeMath();
-
-       if (!for_master)
-               return;
-
-       // Check children
-       ListOfBuffers blist = buffer.getDescendents();
-       ListOfBuffers::const_iterator bit = blist.begin();
-       ListOfBuffers::const_iterator const bend = blist.end();
-       for (; bit != bend; ++bit)
-               initUnicodeMath(**bit, false);
-#endif
-}
-
-
-void Encodings::validate(char_type c, LaTeXFeatures & features, bool for_mathed)
-{
-#ifdef TEX2LYX
-       // The code below is not needed in tex2lyx and requires additional stuff
-       (void)c;
-       (void)features;
-       (void)for_mathed;
-#else
+       static CharInfo empty;
        CharInfoMap::const_iterator const it = unicodesymbols.find(c);
-       if (it != unicodesymbols.end()) {
-               // In mathed, c could be used both in textmode and mathmode
-               bool const math_mode = for_mathed && isMathCmd(c);
-               bool const use_math = math_mode ||
-                                     (!for_mathed && it->second.textcommand.empty());
-               bool const use_text = (for_mathed && isTextCmd(c)) ||
-                                     (!for_mathed && !it->second.textcommand.empty());
-               bool const plain_utf8 = (features.runparams().encoding->name() == "utf8-plain");
-               bool const unicode_math = (features.isRequired("unicode-math")
-                       && features.isAvailable("unicode-math"));
-               // with utf8-plain, we only load packages when in mathed (see #7766)
-               // and if we do not use unicode-math
-               if ((math_mode && !unicode_math)
-                    || (use_math && !plain_utf8)) {
-                       if (!it->second.mathpreamble.empty()) {
-                               if (it->second.mathfeature()) {
-                                       string feats = it->second.mathpreamble;
-                                       while (!feats.empty()) {
-                                               string feat;
-                                               feats = split(feats, feat, ',');
-                                               features.require(feat);
-                                       }
-                               } else
-                                       features.addPreambleSnippet(it->second.mathpreamble);
-                       }
-               }
-               // with utf8-plain, we do not load packages (see #7766)
-               if (use_text && !plain_utf8) {
-                       if (!it->second.textpreamble.empty()) {
-                               if (it->second.textfeature()) {
-                                       string feats = it->second.textpreamble;
-                                       while (!feats.empty()) {
-                                               string feat;
-                                               feats = split(feats, feat, ',');
-                                               features.require(feat);
-                                       }
-                               } else
-                                       features.addPreambleSnippet(it->second.textpreamble);
-                       }
-               }
-       }
-       if (for_mathed && isMathSym(c)) {
-               features.require("amstext");
-               features.require("lyxmathsym");
-       }
-#endif
-}
-
-
-bool Encodings::isHebrewComposeChar(char_type c)
-{
-       return c <= 0x05c2 && c >= 0x05b0 && c != 0x05be && c != 0x05c0;
-}
-
-
-// Special Arabic letters are ones that do not get connected from left
-// they are hamza, alef_madda, alef_hamza, waw_hamza, alef_hamza_under,
-// alef, tah_marbota, dal, thal, rah, zai, wow, alef_maksoura
-
-bool Encodings::isArabicSpecialChar(char_type c)
-{
-       return (c >= 0x0621 && c <= 0x0625) || (c >= 0x0630 && c <= 0x0632)
-               || c == 0x0627 || c == 0x0629 || c == 0x062f || c == 0x0648
-               || c == 0x0649 || c == 0x0698;
-}
-
-
-bool Encodings::isArabicComposeChar(char_type c)
-{
-       return c >= 0x064b && c <= 0x0652;
-}
-
-
-bool Encodings::isArabicChar(char_type c)
-{
-       return c >= arabic_start && c <= arabic_end
-               && arabic_table[c-arabic_start][0];
-}
-
-
-char_type Encodings::transformChar(char_type c, Encodings::LetterForm form)
-{
-       return isArabicChar(c) ? arabic_table[c-arabic_start][form] : c;
+       return it != unicodesymbols.end() ? it->second : empty;
 }
 
 
@@ -814,26 +635,34 @@ string const Encodings::TIPAShortcut(char_type c)
 {
        CharInfoMap::const_iterator const it = unicodesymbols.find(c);
        if (it != unicodesymbols.end())
-               return it->second.tipashortcut;
+               return it->second.tipaShortcut();
        return string();
 }
 
 
-bool Encodings::isKnownScriptChar(char_type const c, string & preamble)
+string const Encodings::isKnownScriptChar(char_type const c)
 {
        CharInfoMap::const_iterator const it = unicodesymbols.find(c);
 
        if (it == unicodesymbols.end())
-               return false;
+               return string();
+       // FIXME: parse complex textPreamble (may be list or alternatives,
+       //                e.g., "subscript,textgreek" or "textcomp|textgreek")
+       if (it->second.textPreamble() == "textgreek"
+               || it->second.textPreamble() == "textcyrillic")
+               return it->second.textPreamble();
+       return string();
+}
 
-       if (it->second.textpreamble != "textgreek" && it->second.textpreamble != "textcyr")
-               return false;
 
-       if (preamble.empty()) {
-               preamble = it->second.textpreamble;
-               return true;
-       }
-       return it->second.textpreamble == preamble;
+bool Encodings::fontencSupportsScript(string const & fontenc, string const & script)
+{
+       if (script == "textgreek")
+               return (fontenc == "LGR" || fontenc == "TU");
+       if (script == "textcyrillic")
+               return (fontenc == "T2A" || fontenc == "T2B" || fontenc == "T2C"
+                               || fontenc == "X2" || fontenc == "TU");
+       return false;
 }
 
 
@@ -843,18 +672,30 @@ bool Encodings::isMathAlpha(char_type c)
 }
 
 
+bool Encodings::isUnicodeTextOnly(char_type c)
+{
+       if (isASCII(c) || isMathAlpha(c))
+               return false;
+
+       CharInfoMap::const_iterator const it = unicodesymbols.find(c);
+       return it == unicodesymbols.end() || it->second.mathCommand().empty();
+}
+
+
 Encoding const *
 Encodings::fromLyXName(string const & name, bool allowUnsafe) const
 {
        EncodingList::const_iterator const it = encodinglist.find(name);
+       if (it == encodinglist.end())
+               return nullptr;
        if (!allowUnsafe && it->second.unsafe())
-               return 0;
-       return it != encodinglist.end() ? &it->second : 0;
+               return nullptr;
+       return &it->second;
 }
 
 
 Encoding const *
-Encodings::fromLaTeXName(string const & n, int const & p, bool allowUnsafe) const
+Encodings::fromLaTeXName(string const & n, int p, bool allowUnsafe) const
 {
        string name = n;
        // FIXME: if we have to test for too many of these synonyms,
@@ -872,19 +713,19 @@ Encodings::fromLaTeXName(string const & n, int const & p, bool allowUnsafe) cons
                if ((it->second.latexName() == name) && (it->second.package() & p)
                                && (!it->second.unsafe() || allowUnsafe))
                        return &it->second;
-       return 0;
+       return nullptr;
 }
 
 
 Encoding const *
-Encodings::fromIconvName(string const & n, int const & p, bool allowUnsafe) const
+Encodings::fromIconvName(string const & n, int p, bool allowUnsafe) const
 {
        EncodingList::const_iterator const end = encodinglist.end();
        for (EncodingList::const_iterator it = encodinglist.begin(); it != end; ++it)
                if ((it->second.iconvName() == n) && (it->second.package() & p)
                                && (!it->second.unsafe() || allowUnsafe))
                        return &it->second;
-       return 0;
+       return nullptr;
 }
 
 
@@ -896,71 +737,96 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
 {
        // We must read the symbolsfile first, because the Encoding
        // constructor depends on it.
-       Lexer symbolslex;
-       symbolslex.setFile(symbolsfile);
+       CharSetMap forcedNotSelected;
+       Lexer symbolsLex;
+       symbolsLex.setFile(symbolsfile);
        bool getNextToken = true;
-       while (symbolslex.isOK()) {
+       while (symbolsLex.isOK()) {
                char_type symbol;
-               CharInfo info;
-               string flags;
 
                if (getNextToken) {
-                       if (!symbolslex.next(true))
+                       if (!symbolsLex.next(true))
                                break;
                } else
                        getNextToken = true;
 
-               istringstream is(symbolslex.getString());
+               istringstream is(symbolsLex.getString());
                // reading symbol directly does not work if
                // char_type == wchar_t.
-               boost::uint32_t tmp;
+               uint32_t tmp;
                if(!(is >> hex >> tmp))
                        break;
                symbol = tmp;
 
-               if (!symbolslex.next(true))
+               // Special case: more than one entry for one character (to add other LaTeX commands).
+               if (unicodesymbols.find(symbol) != unicodesymbols.end()) {
+                       if (!symbolsLex.next(true))
+                               break;
+                       docstring textCommand = symbolsLex.getDocString();
+                       if (!symbolsLex.next(true))
+                               break;
+                       string mathCommand = symbolsLex.getString();
+
+                       if (!textCommand.empty())
+                               unicodesymbols.at(symbol).addTextCommand(textCommand);
+                       if (!mathCommand.empty())
+                               unicodesymbols.at(symbol).addMathCommand(textCommand);
+
+                       continue;
+               }
+
+               // If the symbol is not the same as the previous entry, consider it is a totally new symbol.
+               if (!symbolsLex.next(true))
                        break;
-               info.textcommand = symbolslex.getDocString();
-               if (!symbolslex.next(true))
+               docstring textCommand = symbolsLex.getDocString();
+               if (!symbolsLex.next(true))
                        break;
-               info.textpreamble = symbolslex.getString();
-               if (!symbolslex.next(true))
+               string textPreamble = symbolsLex.getString();
+               if (!symbolsLex.next(true))
                        break;
-               flags = symbolslex.getString();
-
-               info.flags = 0;
-               if (suffixIs(info.textcommand, '}'))
-                       info.flags |= CharInfoTextNoTermination;
-               if (suffixIs(info.mathcommand, '}'))
-                       info.flags |= CharInfoMathNoTermination;
-               while (!flags.empty()) {
+               string sflags = symbolsLex.getString();
+
+               string tipaShortcut;
+               int flags = 0;
+
+               if (suffixIs(textCommand, '}'))
+                       flags |= CharInfoTextNoTermination;
+               while (!sflags.empty()) {
                        string flag;
-                       flags = split(flags, flag, ',');
+                       sflags = split(sflags, flag, ',');
                        if (flag == "combining") {
-                               info.flags |= CharInfoCombining;
+                               flags |= CharInfoCombining;
                        } else if (flag == "force") {
-                               info.flags |= CharInfoForce;
+                               flags |= CharInfoForce;
                                forced.insert(symbol);
                        } else if (prefixIs(flag, "force=")) {
-                               vector<string> encodings =
+                               vector<string> encs =
                                        getVectorFromString(flag.substr(6), ";");
-                               for (size_t i = 0; i < encodings.size(); ++i)
-                                       forcedselected[encodings[i]].insert(symbol);
-                               info.flags |= CharInfoForceSelected;
+                               for (auto const & enc : encs)
+                                       forcedSelected[enc].insert(symbol);
+                               flags |= CharInfoForceSelected;
+                       } else if (prefixIs(flag, "force!=")) {
+                               vector<string> encs =
+                                       getVectorFromString(flag.substr(7), ";");
+                               for (auto const & enc : encs)
+                                       forcedNotSelected[enc].insert(symbol);
+                               flags |= CharInfoForceSelected;
                        } else if (flag == "mathalpha") {
                                mathalpha.insert(symbol);
                        } else if (flag == "notermination=text") {
-                               info.flags |= CharInfoTextNoTermination;
+                               flags |= CharInfoTextNoTermination;
                        } else if (flag == "notermination=math") {
-                               info.flags |= CharInfoMathNoTermination;
+                               flags |= CharInfoMathNoTermination;
                        } else if (flag == "notermination=both") {
-                               info.flags |= CharInfoTextNoTermination;
-                               info.flags |= CharInfoMathNoTermination;
+                               flags |= CharInfoTextNoTermination;
+                               flags |= CharInfoMathNoTermination;
                        } else if (flag == "notermination=none") {
-                               info.flags &= ~CharInfoTextNoTermination;
-                               info.flags &= ~CharInfoMathNoTermination;
+                               flags &= ~CharInfoTextNoTermination;
+                               flags &= ~CharInfoMathNoTermination;
                        } else if (contains(flag, "tipashortcut=")) {
-                               info.tipashortcut = split(flag, '=');
+                               tipaShortcut = split(flag, '=');
+                       } else if (flag == "deprecated") {
+                               flags |= CharInfoDeprecated;
                        } else {
                                lyxerr << "Ignoring unknown flag `" << flag
                                       << "' for symbol `0x"
@@ -968,23 +834,27 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
                                       << "'." << endl;
                        }
                }
-               // mathcommand and mathpreamble have been added for 1.6.0.
+               // mathCommand and mathPreamble have been added for 1.6.0.
                // make them optional so that old files still work.
-               int const lineno = symbolslex.lineNumber();
+               int const lineNo = symbolsLex.lineNumber();
                bool breakout = false;
-               if (symbolslex.next(true)) {
-                       if (symbolslex.lineNumber() != lineno) {
-                               // line in old format without mathcommand and mathpreamble
+               docstring mathCommand;
+               string mathPreamble;
+               if (symbolsLex.next(true)) {
+                       if (symbolsLex.lineNumber() != lineNo) {
+                               // line in old format without mathCommand and mathPreamble
                                getNextToken = false;
                        } else {
-                               info.mathcommand = symbolslex.getDocString();
-                               if (symbolslex.next(true)) {
-                                       if (symbolslex.lineNumber() != lineno) {
-                                               // line in new format with mathcommand only
+                               mathCommand = symbolsLex.getDocString();
+                               if (suffixIs(mathCommand, '}'))
+                                       flags |= CharInfoMathNoTermination;
+                               if (symbolsLex.next(true)) {
+                                       if (symbolsLex.lineNumber() != lineNo) {
+                                               // line in new format with mathCommand only
                                                getNextToken = false;
                                        } else {
-                                               // line in new format with mathcommand and mathpreamble
-                                               info.mathpreamble = symbolslex.getString();
+                                               // line in new format with mathCommand and mathPreamble
+                                               mathPreamble = symbolsLex.getString();
                                        }
                                } else
                                        breakout = true;
@@ -994,27 +864,32 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
                }
 
                // backward compatibility
-               if (info.mathpreamble == "esintoramsmath")
-                       info.mathpreamble = "esint|amsmath";
-
-               if (!info.textpreamble.empty())
-                       if (info.textpreamble[0] != '\\')
-                               info.flags |= CharInfoTextFeature;
-               if (!info.mathpreamble.empty())
-                       if (info.mathpreamble[0] != '\\')
-                               info.flags |= CharInfoMathFeature;
-
+               if (mathPreamble == "esintoramsmath")
+                       mathPreamble = "esint|amsmath";
+
+               if (!textPreamble.empty())
+                       if (textPreamble[0] != '\\')
+                               flags |= CharInfoTextFeature;
+               if (!mathPreamble.empty())
+                       if (mathPreamble[0] != '\\')
+                               flags |= CharInfoMathFeature;
+
+               CharInfo info = CharInfo(
+                               textCommand, mathCommand,
+                               textPreamble, mathPreamble,
+                               tipaShortcut, flags);
                LYXERR(Debug::INFO, "Read unicode symbol " << symbol << " '"
-                       << to_utf8(info.textcommand) << "' '" << info.textpreamble
-                       << " '" << info.textfeature() << ' ' << info.textnotermination()
-                       << ' ' << to_utf8(info.mathcommand) << "' '" << info.mathpreamble
-                       << "' " << info.mathfeature() << ' ' << info.mathnotermination()
-                       << ' ' << info.combining() << ' ' << info.force()
-                       << ' ' << info.forceselected());
+                                                          << to_utf8(info.textCommand()) << "' '" << info.textPreamble()
+                                                          << " '" << info.textFeature() << ' ' << info.textNoTermination()
+                                                          << ' ' << to_utf8(info.mathCommand()) << "' '" << info.mathPreamble()
+                                                          << "' " << info.mathFeature() << ' ' << info.mathNoTermination()
+                          << ' ' << info.combining() << ' ' << info.force()
+                          << ' ' << info.forceSelected());
 
                // we assume that at least one command is nonempty when using unicodesymbols
-               if (!info.textcommand.empty() || !info.mathcommand.empty())
+               if (info.isUnicodeSymbol()) {
                        unicodesymbols[symbol] = info;
+               }
 
                if (breakout)
                        break;
@@ -1026,12 +901,12 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
                et_end
        };
 
-       LexerKeyword encodingtags[] = {
+       LexerKeyword encodingTags[] = {
                { "encoding", et_encoding },
                { "end", et_end }
        };
 
-       Lexer lex(encodingtags);
+       Lexer lex(encodingTags);
        lex.setFile(encfile);
        lex.setContext("Encodings::read");
        while (lex.isOK()) {
@@ -1041,21 +916,21 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
                        lex.next();
                        string const name = lex.getString();
                        lex.next();
-                       string const latexname = lex.getString();
+                       string const latexName = lex.getString();
                        lex.next();
-                       string const guiname = lex.getString();
+                       string const guiName = lex.getString();
                        lex.next();
-                       string const iconvname = lex.getString();
+                       string const iconvName = lex.getString();
                        lex.next();
                        string const width = lex.getString();
-                       bool fixedwidth = false;
+                       bool fixedWidth = false;
                        bool unsafe = false;
                        if (width == "fixed")
-                               fixedwidth = true;
+                               fixedWidth = true;
                        else if (width == "variable")
-                               fixedwidth = false;
+                               fixedWidth = false;
                        else if (width == "variableunsafe") {
-                               fixedwidth = false;
+                               fixedWidth = false;
                                unsafe = true;
                        }
                        else
@@ -1076,9 +951,9 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
                                lex.printError("Unknown package");
 
                        LYXERR(Debug::INFO, "Reading encoding " << name);
-                       encodinglist[name] = Encoding(name, latexname,
-                               guiname, iconvname, fixedwidth, unsafe,
-                               package);
+                       encodinglist[name] = Encoding(name, latexName,
+                                                     guiName, iconvName, fixedWidth, unsafe,
+                                                     package);
 
                        if (lex.lex() != et_end)
                                lex.printError("Missing end");
@@ -1094,6 +969,15 @@ void Encodings::read(FileName const & encfile, FileName const & symbolsfile)
                        break;
                }
        }
+
+       // Move all information from forcedNotSelected to forcedSelected
+       for (CharSetMap::const_iterator it1 = forcedNotSelected.begin(); it1 != forcedNotSelected.end(); ++it1) {
+               for (CharSetMap::iterator it2 = forcedSelected.begin(); it2 != forcedSelected.end(); ++it2) {
+                       if (it2->first != it1->first)
+                               it2->second.insert(it1->second.begin(), it1->second.end());
+               }
+       }
+
 }