]> git.lyx.org Git - features.git/blobdiff - src/lyxfind.cpp
CALS tables: base implementation of row separators on the new code from XHTML.
[features.git] / src / lyxfind.cpp
index a34baa372650934aa13c742fa3712ce14f93b7e0..53058ddace7f247fceb0448e6a75c663f1ca393e 100644 (file)
@@ -106,6 +106,8 @@ class IgnoreFormats {
        ///
        void setIgnoreDeleted(bool value);
        ///
+       bool getNonContent() const { return searchNonContent_; }
+       ///
        void setIgnoreFormat(string const & type, bool value, bool fromUser = true);
 
 private:
@@ -132,6 +134,8 @@ private:
        bool userSelectedIgnoreLanguage_ = false;
        ///
        bool ignoreDeleted_ = true;
+       ///
+       bool searchNonContent_ = true;
 };
 
 void IgnoreFormats::setIgnoreFormat(string const & type, bool value, bool fromUser)
@@ -177,6 +181,9 @@ void IgnoreFormats::setIgnoreFormat(string const & type, bool value, bool fromUs
        else if (type == "deleted") {
                ignoreDeleted_ = value;
        }
+       else if (type == "non-output-content") {
+               searchNonContent_ = !value;
+       }
 }
 
 // The global variable that can be changed from outside
@@ -185,7 +192,7 @@ IgnoreFormats ignoreFormats;
 
 void setIgnoreFormat(string const & type, bool value, bool fromUser)
 {
-  ignoreFormats.setIgnoreFormat(type, value, fromUser);
+       ignoreFormats.setIgnoreFormat(type, value, fromUser);
 }
 
 
@@ -276,7 +283,7 @@ bool searchAllowed(docstring const & str)
 
 bool findOne(BufferView * bv, docstring const & searchstr,
             bool case_sens, bool whole, bool forward,
-            bool find_del, bool check_wrap, bool auto_wrap,
+            bool find_del, bool check_wrap, bool const auto_wrap,
             bool instant, bool onlysel)
 {
        // Clean up previous selections with empty searchstr on instant
@@ -285,7 +292,7 @@ bool findOne(BufferView * bv, docstring const & searchstr,
                        bv->setCursor(bv->cursor().selectionBegin());
                        bv->clearSelection();
                }
-               return false;
+               return true;
        }
 
        if (!searchAllowed(searchstr))
@@ -338,6 +345,7 @@ bool findOne(BufferView * bv, docstring const & searchstr,
        }
        else if (check_wrap) {
                DocIterator cur_orig(bv->cursor());
+               bool wrap = auto_wrap;
                if (!auto_wrap) {
                        docstring q;
                        if (forward)
@@ -348,9 +356,9 @@ bool findOne(BufferView * bv, docstring const & searchstr,
                                  "Continue searching from the end?");
                        int wrap_answer = frontend::Alert::prompt(_("Wrap search?"),
                                q, 0, 1, _("&Yes"), _("&No"));
-                       auto_wrap = wrap_answer == 0;
+                       wrap = wrap_answer == 0;
                }
-               if (auto_wrap) {
+               if (wrap) {
                        if (forward) {
                                bv->cursor().clear();
                                bv->cursor().push_back(CursorSlice(bv->buffer().inset()));
@@ -358,6 +366,12 @@ bool findOne(BufferView * bv, docstring const & searchstr,
                                bv->cursor().setCursor(doc_iterator_end(&bv->buffer()));
                                bv->cursor().backwardPos();
                        }
+                       if (auto_wrap) {
+                               docstring const msg = forward
+                                 ? _("Search reached end of document, continuing from beginning.")
+                                 : _("Search reached beginning of document, continuing from end.");
+                               bv->message(msg);
+                       }
                        bv->clearSelection();
                        if (findOne(bv, searchstr, case_sens, whole, forward,
                                    find_del, false, false, false, false))
@@ -816,16 +830,22 @@ string string2regex(string in)
        return temp2;
 }
 
+static void buildAccentsMap();
+
 string correctRegex(string t, bool withformat)
 {
        /* Convert \backslash => \
         * and \{, \}, \[, \] => {, }, [, ]
         */
        string s("");
-       regex wordre("(\\\\)*(\\\\((backslash|mathcircumflex) ?|[\\[\\]\\{\\}]))");
+       regex wordre("(\\\\)*(\\\\(([A-Za-z]+|[\\{\\}])( |\\{\\})?|[\\[\\]\\{\\}]))");
        size_t lastpos = 0;
        smatch sub;
        bool backslashed = false;
+       if (accents.empty())
+               buildAccentsMap();
+
+       //LYXERR0("correctRegex input '" << t << "'");
        for (sregex_iterator it(t.begin(), t.end(), wordre), end; it != end; ++it) {
                sub = *it;
                string replace;
@@ -835,9 +855,8 @@ string correctRegex(string t, bool withformat)
                else {
                        if (sub.str(4) == "backslash") {
                                replace = "\\";
-                               if (withformat) {
+                               {
                                        // transforms '\backslash \{' into '\{'
-                                       // and '\{' into '{'
                                        string next = t.substr(sub.position(2) + sub.str(2).length(), 2);
                                        if ((next == "\\{") || (next == "\\}")) {
                                                replace = "";
@@ -849,17 +868,40 @@ string correctRegex(string t, bool withformat)
                                replace = "^";
                        else if (backslashed) {
                                backslashed = false;
-                               if (withformat && (sub.str(3) == "{"))
-                                       replace = accents["braceleft"];
-                               else if (withformat && (sub.str(3) == "}"))
-                                       replace = accents["braceright"];
+                               if (withformat) {
+                                       if (sub.str(3) == "{")
+                                               replace = accents["braceleft"];
+                                       else if (sub.str(3) == "}")
+                                               replace = accents["braceright"];
+                                       else {
+                                               // else part should not exist
+                                               LASSERT(1, /**/);
+                                       }
+                               }
                                else {
-                                       // else part should not exist
-                                       LASSERT(1, /**/);
+                                       if (sub.str(3) == "{")
+                                               replace = "\\{";
+                                       else if (sub.str(3) == "}")
+                                               replace = "\\}";
+                                       else {
+                                               // else part should not exist
+                                               LASSERT(1, /**/);
+                                       }
+                               }
+                       }
+                       else if (sub.str(4) == "{") // transforms '\{' into '{'
+                               replace = "{";
+                       else if (sub.str(4) == "}")
+                                replace = "}";
+                       else {
+                               AccentsIterator it_ac = accents.find(sub.str(4));
+                               if (it_ac == accents.end()) {
+                                       replace = sub.str(2);
+                               }
+                               else {
+                                       replace = it_ac->second;
                                }
                        }
-                       else
-                               replace = sub.str(3);
                }
                if (lastpos < (size_t) sub.position(2))
                        s += t.substr(lastpos, sub.position(2) - lastpos);
@@ -870,6 +912,7 @@ string correctRegex(string t, bool withformat)
                return t;
        else if (lastpos < t.length())
                s += t.substr(lastpos, t.length() - lastpos);
+       //LYXERR0("correctRegex output '" << s << "'");
        return s;
 }
 
@@ -891,7 +934,7 @@ string escape_for_regex(string s, bool withformat)
                        if (lastpos == s.size())
                                break;
                }
-               size_t end_pos = s.find("\\endregexp{}}", regex_pos + 8);
+               size_t end_pos = s.find("\\endregexp", regex_pos + 8);
                result += correctRegex(s.substr(regex_pos + 8, end_pos -(regex_pos + 8)), withformat);
                lastpos = end_pos + 13;
        }
@@ -928,7 +971,7 @@ public:
        int pos_len;
        int searched_size;
        vector <string> result = vector <string>();
-       MatchResult(int len = 0): match_len(len),match_prefix(0),match2end(0), pos(0),leadsize(0),pos_len(-1),searched_size(0) {};
+       MatchResult(int len = 0): match_len(len),match_prefix(0),match2end(0), pos(0),leadsize(0),pos_len(-1),searched_size(0) {}
 };
 
 static MatchResult::range interpretMatch(MatchResult &oldres, MatchResult &newres)
@@ -994,7 +1037,7 @@ private:
         ** @todo Normalization should also expand macros, if the corresponding
         ** search option was checked.
         **/
-       string normalize(docstring const & s) const;
+       string normalize(docstring const & s, bool ignore_fomat) const;
        // normalized string to search
        string par_as_string;
        // regular expression to use for searching
@@ -1052,6 +1095,9 @@ static docstring buffer_to_latex(Buffer & buffer)
                runparams.for_searchAdv = OutputParams::SearchWithoutDeleted;
        else
                runparams.for_searchAdv = OutputParams::SearchWithDeleted;
+       if (ignoreFormats.getNonContent()) {
+               runparams.for_searchAdv |= OutputParams::SearchNonOutput;
+       }
        pit_type const endpit = buffer.paragraphs().size();
        for (pit_type pit = 0; pit != endpit; ++pit) {
                TeXOnePar(buffer, buffer.text(), pit, os, runparams);
@@ -1060,6 +1106,52 @@ static docstring buffer_to_latex(Buffer & buffer)
        return ods.str();
 }
 
+static string latexNamesToUtf8(docstring strIn)
+{
+       string addtmp = to_utf8(strIn);
+       static regex const rmAcc("(\\\\)*("
+                                        "\\\\([A-Za-z]+\\{.\\})"       // e.g. "ddot{A}" == sub.str(3)
+                                       "|\\\\([A-Za-z]+)( |\\{\\})?"   // e.g. "LyX", "LyX{}", "LyX " == sub.str(4)
+                                       ")"
+                               );
+       size_t lastpos = 0;
+       smatch sub;
+       string replace;
+       string add("");
+       if (accents.empty())
+               buildAccentsMap();
+       for (sregex_iterator it_add(addtmp.begin(), addtmp.end(), rmAcc), end; it_add != end; ++it_add) {
+               sub = *it_add;
+               if ((sub.position(2) - sub.position(0)) % 2 == 1) {
+                       continue;
+               }
+               else {
+                       string key;
+                       if (sub.length(3) > 0)
+                               key = sub.str(3);
+                       else
+                               key = sub.str(4);
+                       AccentsIterator it_ac = accents.find(key);
+                       if (it_ac == accents.end()) {
+                               replace = sub.str(2);
+                       }
+                       else {
+                               replace = it_ac->second;
+                       }
+               }
+               if (lastpos < (size_t) sub.position(2))
+                       add += addtmp.substr(lastpos, sub.position(2) - lastpos);
+               add += replace;
+               lastpos = sub.position(2) + sub.length(2);
+       }
+       if (lastpos == 0)
+               add = addtmp;
+       else if (addtmp.length() > lastpos)
+               add += addtmp.substr(lastpos, addtmp.length() - lastpos);
+       LYXERR(Debug::FIND, "Adding to search string: '"
+                       << add << "'");
+       return add;
+}
 
 static docstring stringifySearchBuffer(Buffer & buffer, FindAndReplaceOptions const & opt)
 {
@@ -1081,19 +1173,20 @@ static docstring stringifySearchBuffer(Buffer & buffer, FindAndReplaceOptions co
                else {
                        runparams.for_searchAdv = OutputParams::SearchWithDeleted;
                }
+               if (ignoreFormats.getNonContent()) {
+                       runparams.for_searchAdv |= OutputParams::SearchNonOutput;
+               }
+               string t("");
                for (pos_type pit = pos_type(0); pit < (pos_type)buffer.paragraphs().size(); ++pit) {
                        Paragraph const & par = buffer.paragraphs().at(pit);
+                       string add = latexNamesToUtf8(par.asString(pos_type(0), par.size(),
+                                                               option,
+                                                               &runparams));
                        LYXERR(Debug::FIND, "Adding to search string: '"
-                              << par.asString(pos_type(0), par.size(),
-                                              option,
-                                              &runparams)
-                              << "'");
-                       str += par.asString(pos_type(0), par.size(),
-                                           option,
-                                           &runparams);
+                               << add << "'");
+                       t += add;
                }
                // Even in ignore-format we have to remove "\text{}, \lyxmathsym{}" parts
-               string t = to_utf8(str);
                while (regex_replace(t, t, "\\\\(text|lyxmathsym|ensuremath)\\{([^\\}]*)\\}", "$2"));
                str = from_utf8(t);
        }
@@ -1241,7 +1334,7 @@ class KeyInfo {
 
 class Border {
  public:
- Border(int l=0, int u=0) : low(l), upper(u) {};
+ Border(int l=0, int u=0) : low(l), upper(u) {}
   int low;
   int upper;
 };
@@ -1486,6 +1579,294 @@ static string getutf8(unsigned uchar)
        return(ret);
 }
 
+static void addAccents(string latex_in, string unicode_out)
+{
+  latex_in = latex_in.substr(1);
+  AccentsIterator it_ac = accents.find(latex_in);
+  if (it_ac == accents.end()) {
+    accents[latex_in] = unicode_out;
+  }
+  else {
+    LYXERR0("Key " << latex_in  << " already set");
+  }
+}
+
+void static fillMissingUnicodesymbols()
+{
+  addAccents("\\pounds", getutf8(0x00a3));
+  addAccents("\\textsterling", getutf8(0x00a3));
+  addAccents("\\textyen", getutf8(0x00a5));
+  addAccents("\\yen", getutf8(0x00a5));
+  addAccents("\\textsection", getutf8(0x00a7));
+  addAccents("\\mathsection", getutf8(0x00a7));
+  addAccents("\\textcopyright", getutf8(0x00a9));
+  addAccents("\\copyright", getutf8(0x00a9));
+  addAccents("\\textlnot", getutf8(0x00ac));
+  addAccents("\\neg", getutf8(0x00ac));
+  addAccents("\\textregistered", getutf8(0x00ae));
+  addAccents("\\circledR", getutf8(0x00ae));
+  addAccents("\\textpm", getutf8(0x00b1));
+  addAccents("\\pm", getutf8(0x00b1));
+  addAccents("\\textparagraph", getutf8(0x00b6));
+  addAccents("\\mathparagraph", getutf8(0x00b6));
+  addAccents("\\textperiodcentered", getutf8(0x00b7));
+  addAccents("\\texttimes", getutf8(0x00d7));
+  addAccents("\\times", getutf8(0x00d7));
+  addAccents("\\O", getutf8(0x00d8));
+  addAccents("\\dh", getutf8(0x00f0));
+  addAccents("\\eth", getutf8(0x00f0));
+  addAccents("\\textdiv", getutf8(0x00f7));
+  addAccents("\\div", getutf8(0x00f7));
+  addAccents("\\o", getutf8(0x00f8));
+  addAccents("\\textcrlambda", getutf8(0x019b));
+  addAccents("\\j", getutf8(0x0237));
+  addAccents("\\textrevepsilon", getutf8(0x025c));
+  addAccents("\\textbaru", getutf8(0x0289));
+  addAccents("\\textquoteleft", getutf8(0x02bb));
+  addAccents("\\textGamma", getutf8(0x0393));
+  addAccents("\\Gamma", getutf8(0x0393));
+  addAccents("\\textDelta", getutf8(0x0394));
+  addAccents("\\Delta", getutf8(0x0394));
+  addAccents("\\textTheta", getutf8(0x0398));
+  addAccents("\\Theta", getutf8(0x0398));
+  addAccents("\\textLambda", getutf8(0x039b));
+  addAccents("\\Lambda", getutf8(0x039b));
+  addAccents("\\textXi", getutf8(0x039e));
+  addAccents("\\Xi", getutf8(0x039e));
+  addAccents("\\textPi", getutf8(0x03a0));
+  addAccents("\\Pi", getutf8(0x03a0));
+  addAccents("\\textSigma", getutf8(0x03a3));
+  addAccents("\\Sigma", getutf8(0x03a3));
+  addAccents("\\textUpsilon", getutf8(0x03a5));
+  addAccents("\\Upsilon", getutf8(0x03a5));
+  addAccents("\\textPhi", getutf8(0x03a6));
+  addAccents("\\Phi", getutf8(0x03a6));
+  addAccents("\\textPsi", getutf8(0x03a8));
+  addAccents("\\Psi", getutf8(0x03a8));
+  addAccents("\\textOmega", getutf8(0x03a9));
+  addAccents("\\Omega", getutf8(0x03a9));
+  addAccents("\\textalpha", getutf8(0x03b1));
+  addAccents("\\alpha", getutf8(0x03b1));
+  addAccents("\\textbeta", getutf8(0x03b2));
+  addAccents("\\beta", getutf8(0x03b2));
+  addAccents("\\textgamma", getutf8(0x03b3));
+  addAccents("\\gamma", getutf8(0x03b3));
+  addAccents("\\textdelta", getutf8(0x03b4));
+  addAccents("\\delta", getutf8(0x03b4));
+  addAccents("\\textepsilon", getutf8(0x03b5));
+  addAccents("\\varepsilon", getutf8(0x03b5));
+  addAccents("\\textzeta", getutf8(0x03b6));
+  addAccents("\\zeta", getutf8(0x03b6));
+  addAccents("\\texteta", getutf8(0x03b7));
+  addAccents("\\eta", getutf8(0x03b7));
+  addAccents("\\texttheta", getutf8(0x03b8));
+  addAccents("\\theta", getutf8(0x03b8));
+  addAccents("\\textiota", getutf8(0x03b9));
+  addAccents("\\iota", getutf8(0x03b9));
+  addAccents("\\textkappa", getutf8(0x03ba));
+  addAccents("\\kappa", getutf8(0x03ba));
+  addAccents("\\textlambda", getutf8(0x03bb));
+  addAccents("\\lambda", getutf8(0x03bb));
+  addAccents("\\textmu", getutf8(0x03bc));
+  addAccents("\\mu", getutf8(0x03bc));
+  addAccents("\\textnu", getutf8(0x03bd));
+  addAccents("\\nu", getutf8(0x03bd));
+  addAccents("\\textxi", getutf8(0x03be));
+  addAccents("\\xi", getutf8(0x03be));
+  addAccents("\\textpi", getutf8(0x03c0));
+  addAccents("\\pi", getutf8(0x03c0));
+  addAccents("\\textrho", getutf8(0x03c1));
+  addAccents("\\rho", getutf8(0x03c1));
+  addAccents("\\textfinalsigma", getutf8(0x03c2));
+  addAccents("\\varsigma", getutf8(0x03c2));
+  addAccents("\\textsigma", getutf8(0x03c3));
+  addAccents("\\sigma", getutf8(0x03c3));
+  addAccents("\\texttau", getutf8(0x03c4));
+  addAccents("\\tau", getutf8(0x03c4));
+  addAccents("\\textupsilon", getutf8(0x03c5));
+  addAccents("\\upsilon", getutf8(0x03c5));
+  addAccents("\\textphi", getutf8(0x03c6));
+  addAccents("\\varphi", getutf8(0x03c6));
+  addAccents("\\textchi", getutf8(0x03c7));
+  addAccents("\\chi", getutf8(0x03c7));
+  addAccents("\\textpsi", getutf8(0x03c8));
+  addAccents("\\psi", getutf8(0x03c8));
+  addAccents("\\textomega", getutf8(0x03c9));
+  addAccents("\\omega", getutf8(0x03c9));
+  addAccents("\\textdigamma", getutf8(0x03dd));
+  addAccents("\\digamma", getutf8(0x03dd));
+  addAccents("\\hebalef", getutf8(0x05d0));
+  addAccents("\\aleph", getutf8(0x05d0));
+  addAccents("\\hebbet", getutf8(0x05d1));
+  addAccents("\\beth", getutf8(0x05d1));
+  addAccents("\\hebgimel", getutf8(0x05d2));
+  addAccents("\\gimel", getutf8(0x05d2));
+  addAccents("\\hebdalet", getutf8(0x05d3));
+  addAccents("\\daleth", getutf8(0x05d3));
+  addAccents("\\hebhe", getutf8(0x05d4));
+  addAccents("\\hebvav", getutf8(0x05d5));
+  addAccents("\\hebzayin", getutf8(0x05d6));
+  addAccents("\\hebhet", getutf8(0x05d7));
+  addAccents("\\hebtet", getutf8(0x05d8));
+  addAccents("\\hebyod", getutf8(0x05d9));
+  addAccents("\\hebfinalkaf", getutf8(0x05da));
+  addAccents("\\hebkaf", getutf8(0x05db));
+  addAccents("\\heblamed", getutf8(0x05dc));
+  addAccents("\\hebfinalmem", getutf8(0x05dd));
+  addAccents("\\hebmem", getutf8(0x05de));
+  addAccents("\\hebfinalnun", getutf8(0x05df));
+  addAccents("\\hebnun", getutf8(0x05e0));
+  addAccents("\\hebsamekh", getutf8(0x05e1));
+  addAccents("\\hebayin", getutf8(0x05e2));
+  addAccents("\\hebfinalpe", getutf8(0x05e3));
+  addAccents("\\hebpe", getutf8(0x05e4));
+  addAccents("\\hebfinaltsadi", getutf8(0x05e5));
+  addAccents("\\hebtsadi", getutf8(0x05e6));
+  addAccents("\\hebqof", getutf8(0x05e7));
+  addAccents("\\hebresh", getutf8(0x05e8));
+  addAccents("\\hebshin", getutf8(0x05e9));
+  addAccents("\\hebtav", getutf8(0x05ea));
+
+  // Thai characters
+  addAccents("\\thaiKoKai", getutf8(0x0e01));
+  addAccents("\\thaiKhoKhai", getutf8(0x0e02));
+  addAccents("\\thaiKhoKhuat", getutf8(0x0e03));
+  addAccents("\\thaiKhoKhwai", getutf8(0x0e04));
+  addAccents("\\thaiKhoKhon", getutf8(0x0e05));
+  addAccents("\\thaiKhoRakhang", getutf8(0x0e06));
+  addAccents("\\thaiNgoNgu", getutf8(0x0e07));
+  addAccents("\\thaiChoChan", getutf8(0x0e08));
+  addAccents("\\thaiChoChing", getutf8(0x0e09));
+  addAccents("\\thaiChoChang", getutf8(0x0e0a));
+  addAccents("\\thaiSoSo", getutf8(0x0e0b));
+  addAccents("\\thaiChoChoe", getutf8(0x0e0c));
+  addAccents("\\thaiYoYing", getutf8(0x0e0d));
+  addAccents("\\thaiDoChada", getutf8(0x0e0e));
+  addAccents("\\thaiToPatak", getutf8(0x0e0f));
+  addAccents("\\thaiThoThan", getutf8(0x0e10));
+  addAccents("\\thaiThoNangmontho", getutf8(0x0e11));
+  addAccents("\\thaiThoPhuthao", getutf8(0x0e12));
+  addAccents("\\thaiNoNen", getutf8(0x0e13));
+  addAccents("\\thaiDoDek", getutf8(0x0e14));
+  addAccents("\\thaiToTao", getutf8(0x0e15));
+  addAccents("\\thaiThoThung", getutf8(0x0e16));
+  addAccents("\\thaiThoThahan", getutf8(0x0e17));
+  addAccents("\\thaiThoThong", getutf8(0x0e18));
+  addAccents("\\thaiNoNu", getutf8(0x0e19));
+  addAccents("\\thaiBoBaimai", getutf8(0x0e1a));
+  addAccents("\\thaiPoPla", getutf8(0x0e1b));
+  addAccents("\\thaiPhoPhung", getutf8(0x0e1c));
+  addAccents("\\thaiFoFa", getutf8(0x0e1d));
+  addAccents("\\thaiPhoPhan", getutf8(0x0e1e));
+  addAccents("\\thaiFoFan", getutf8(0x0e1f));
+  addAccents("\\thaiPhoSamphao", getutf8(0x0e20));
+  addAccents("\\thaiMoMa", getutf8(0x0e21));
+  addAccents("\\thaiYoYak", getutf8(0x0e22));
+  addAccents("\\thaiRoRua", getutf8(0x0e23));
+  addAccents("\\thaiRu", getutf8(0x0e24));
+  addAccents("\\thaiLoLing", getutf8(0x0e25));
+  addAccents("\\thaiLu", getutf8(0x0e26));
+  addAccents("\\thaiWoWaen", getutf8(0x0e27));
+  addAccents("\\thaiSoSala", getutf8(0x0e28));
+  addAccents("\\thaiSoRusi", getutf8(0x0e29));
+  addAccents("\\thaiSoSua", getutf8(0x0e2a));
+  addAccents("\\thaiHoHip", getutf8(0x0e2b));
+  addAccents("\\thaiLoChula", getutf8(0x0e2c));
+  addAccents("\\thaiOAng", getutf8(0x0e2d));
+  addAccents("\\thaiHoNokhuk", getutf8(0x0e2e));
+  addAccents("\\thaiPaiyannoi", getutf8(0x0e2f));
+  addAccents("\\thaiSaraA", getutf8(0x0e30));
+  addAccents("\\thaiMaiHanakat", getutf8(0x0e31));
+  addAccents("\\thaiSaraAa", getutf8(0x0e32));
+  addAccents("\\thaiSaraAm", getutf8(0x0e33));
+  addAccents("\\thaiSaraI", getutf8(0x0e34));
+  addAccents("\\thaiSaraIi", getutf8(0x0e35));
+  addAccents("\\thaiSaraUe", getutf8(0x0e36));
+  addAccents("\\thaiSaraUee", getutf8(0x0e37));
+  addAccents("\\thaiSaraU", getutf8(0x0e38));
+  addAccents("\\thaiSaraUu", getutf8(0x0e39));
+  addAccents("\\thaiPhinthu", getutf8(0x0e3a));
+  addAccents("\\thaiSaraE", getutf8(0x0e40));
+  addAccents("\\thaiSaraAe", getutf8(0x0e41));
+  addAccents("\\thaiSaraO", getutf8(0x0e42));
+  addAccents("\\thaiSaraAiMaimuan", getutf8(0x0e43));
+  addAccents("\\thaiSaraAiMaimalai", getutf8(0x0e44));
+  addAccents("\\thaiLakkhangyao", getutf8(0x0e45));
+  addAccents("\\thaiMaiyamok", getutf8(0x0e46));
+  addAccents("\\thaiMaitaikhu", getutf8(0x0e47));
+  addAccents("\\thaiMaiEk", getutf8(0x0e48));
+  addAccents("\\thaiMaiTho", getutf8(0x0e49));
+  addAccents("\\thaiMaiTri", getutf8(0x0e4a));
+  addAccents("\\thaiMaiChattawa", getutf8(0x0e4b));
+  addAccents("\\thaiThanthakhat", getutf8(0x0e4c));
+  addAccents("\\thaiNikhahit", getutf8(0x0e4d));
+  addAccents("\\thaiYamakkan", getutf8(0x0e4e));
+  addAccents("\\thaiFongman", getutf8(0x0e4f));
+  addAccents("\\thaizero", getutf8(0x0e50));
+  addAccents("\\thaione", getutf8(0x0e51));
+  addAccents("\\thaitwo", getutf8(0x0e52));
+  addAccents("\\thaithree", getutf8(0x0e53));
+  addAccents("\\thaifour", getutf8(0x0e54));
+  addAccents("\\thaifive", getutf8(0x0e55));
+  addAccents("\\thaisix", getutf8(0x0e56));
+  addAccents("\\thaiseven", getutf8(0x0e57));
+  addAccents("\\thaieight", getutf8(0x0e58));
+  addAccents("\\thainine", getutf8(0x0e59));
+  addAccents("\\thaiAngkhankhu", getutf8(0x0e5a));
+  addAccents("\\thaiKhomut", getutf8(0x0e5b));
+  addAccents("\\dag", getutf8(0x2020));
+  addAccents("\\dagger", getutf8(0x2020));
+  addAccents("\\textdagger", getutf8(0x2020));
+  addAccents("\\ddag", getutf8(0x2021));
+  addAccents("\\ddagger", getutf8(0x2021));
+  addAccents("\\textdaggerdbl", getutf8(0x2021));
+  addAccents("\\textbullet", getutf8(0x2022));
+  addAccents("\\bullet", getutf8(0x2022));
+  addAccents("\\dots", getutf8(0x2026));
+  addAccents("\\ldots", getutf8(0x2026));
+  addAccents("\\textellipsis", getutf8(0x2026));
+  addAccents("\\textasciiacute", getutf8(0x2032));
+  addAccents("\\prime", getutf8(0x2032));
+  addAccents("\\textacutedbl", getutf8(0x2033));
+  addAccents("\\dprime", getutf8(0x2033));
+  addAccents("\\textasciigrave", getutf8(0x2035));
+  addAccents("\\backprime", getutf8(0x2035));
+  addAccents("\\textsubcircum{ }", getutf8(0x2038));
+  addAccents("\\caretinsert", getutf8(0x2038));
+  addAccents("\\textasteriskcentered", getutf8(0x204e));
+  addAccents("\\ast", getutf8(0x204e));
+  addAccents("\\textmho", getutf8(0x2127));
+  addAccents("\\mho", getutf8(0x2127));
+  addAccents("\\textleftarrow", getutf8(0x2190));
+  addAccents("\\leftarrow", getutf8(0x2190));
+  addAccents("\\textuparrow", getutf8(0x2191));
+  addAccents("\\uparrow", getutf8(0x2191));
+  addAccents("\\textrightarrow", getutf8(0x2192));
+  addAccents("\\rightarrow", getutf8(0x2192));
+  addAccents("\\textdownarrow", getutf8(0x2193));
+  addAccents("\\downarrow", getutf8(0x2193));
+  addAccents("\\textglobrise", getutf8(0x2197));
+  addAccents("\\nearrow", getutf8(0x2197));
+  addAccents("\\textglobfall", getutf8(0x2198));
+  addAccents("\\searrow", getutf8(0x2198));
+  addAccents("\\textsurd", getutf8(0x221a));
+  addAccents("\\surd", getutf8(0x221a));
+  addAccents("\\textbigcircle", getutf8(0x25ef));
+  addAccents("\\bigcirc", getutf8(0x25ef));
+  addAccents("\\FiveStar", getutf8(0x2605));
+  addAccents("\\bigstar", getutf8(0x2605));
+  addAccents("\\FiveStarOpen", getutf8(0x2606));
+  addAccents("\\bigwhitestar", getutf8(0x2606));
+  addAccents("\\Checkmark", getutf8(0x2713));
+  addAccents("\\checkmark", getutf8(0x2713));
+  addAccents("\\CrossMaltese", getutf8(0x2720));
+  addAccents("\\maltese", getutf8(0x2720));
+  addAccents("\\textlangle", getutf8(0x27e8));
+  addAccents("\\langle", getutf8(0x27e8));
+  addAccents("\\textrangle", getutf8(0x27e9));
+  addAccents("\\rangle", getutf8(0x27e9));
+}
+
 static void buildAccentsMap()
 {
   accents["imath"] = "ı";
@@ -1525,8 +1906,8 @@ static void buildAccentsMap()
   accents["backslash LaTeXe"]        = getutf8(0xf0013);
   accents["backslash lyxarrow"]      = getutf8(0xf0020);
   accents["ddot{\\imath}"] = "ï";
-  buildaccent("ddot", "aAeEhHiIioOtuUwWxXyY",
-                      "äÃ\84ëÃ\8bḧḦïÃ\8fïöÃ\96áº\97üÃ\9cáº\85áº\84áº\8dáº\8cÿŸ");      // umlaut
+  buildaccent("ddot", "aAeEhHiIoOtuUwWxXyY",
+                      "äÃ\84ëÃ\8bḧḦïÃ\8föÃ\96áº\97üÃ\9cáº\85áº\84áº\8dáº\8cÿŸ");        // umlaut
   buildaccent("dot|.", "aAbBcCdDeEfFGghHIimMnNoOpPrRsStTwWxXyYzZ",
                        "ȧȦḃḂċĊḋḊėĖḟḞĠġḣḢİİṁṀṅṄȯȮṗṖṙṘṡṠṫṪẇẆẋẊẏẎżŻ");  // dot{i} can only happen if ignoring case, but there is no lowercase of 'İ'
   accents["acute{\\imath}"] = "í";
@@ -1571,12 +1952,14 @@ static void buildAccentsMap()
   accents["textdoublegrave{\\i}"] = "ȉ";
   buildaccent("dgrave|textdoublegrave", "AaEeIiOoRrUu",
                                         "ȀȁȄȅȈȉȌȍȐȑȔȕ"); // double grave
-  accents["rcap{\\imath}"] = "È\89";
-  accents["textroundcap{\\i}"] = "È\89";
+  accents["rcap{\\imath}"] = "È\8b";
+  accents["textroundcap{\\i}"] = "È\8b";
   buildaccent("rcap|textroundcap", "AaEeIiOoRrUu",
                                    "ȂȃȆȇȊȋȎȏȒȓȖȗ"); // inverted breve
   buildaccent("slashed", "oO",
                          "øØ"); // slashed
+  fillMissingUnicodesymbols(); // Add some still not handled entries contained in 'unicodesynbols'
+  // LYXERR0("Number of accents " << accents.size());
 }
 
 /*
@@ -1587,9 +1970,13 @@ void Intervall::removeAccents()
 {
   if (accents.empty())
     buildAccentsMap();
-  static regex const accre("\\\\(([\\S]|grave|breve|ddot|dot|acute|dacute|mathring|check|hat|bar|tilde|subdot|ogonek|"
-         "cedilla|subring|textsubring|subhat|textsubcircum|subtilde|textsubtilde|dgrave|textdoublegrave|rcap|textroundcap|slashed)\\{[^\\{\\}]+\\}"
-      "|((i|imath|jmath|cdot|[a-z]+space)|((backslash )?([lL]y[xX]|[tT]e[xX]|[lL]a[tT]e[xX]e?|lyxarrow))|(brace|guillemot)(left|right)|textasciicircum|mathcircumflex|sim)(?![a-zA-Z]))");
+  static regex const accre("\\\\("
+      "([\\S]|[A-Za-z]+)\\{[^\\{\\}]+\\}"
+      "|("
+        "(backslash ([lL]y[xX]|[tT]e[xX]|[lL]a[tT]e[xX]e?|lyxarrow))"
+       "|[A-Za-z]+"
+      ")"
+      "(?![a-zA-Z]))");
   smatch sub;
   for (sregex_iterator itacc(par.begin(), par.end(), accre), end; itacc != end; ++itacc) {
     sub = *itacc;
@@ -1694,7 +2081,7 @@ class LatexInfo {
     buildKeys(isPatternString);
     entries_ = vector<KeyInfo>();
     buildEntries(isPatternString);
-  };
+  }
   int getFirstKey() {
     entidx_ = 0;
     if (entries_.empty()) {
@@ -1715,7 +2102,7 @@ class LatexInfo {
         return -1;
     }
     return 0;
-  };
+  }
   int getNextKey() {
     entidx_++;
     if (int(entries_.size()) > entidx_) {
@@ -1724,7 +2111,7 @@ class LatexInfo {
     else {
       return -1;
     }
-  };
+  }
   bool setNextKey(int idx) {
     if ((idx == entidx_) && (entidx_ >= 0)) {
       entidx_--;
@@ -1732,7 +2119,7 @@ class LatexInfo {
     }
     else
       return false;
-  };
+  }
   int find(int start, KeyInfo::KeyType keytype) const {
     if (start < 0)
       return -1;
@@ -1743,20 +2130,20 @@ class LatexInfo {
       tmpIdx++;
     }
     return -1;
-  };
+  }
   int process(ostringstream & os, KeyInfo const & actual);
   int dispatch(ostringstream & os, int previousStart, KeyInfo & actual);
-  // string show(int lastpos) { return interval.show(lastpos);};
-  int nextNotIgnored(int start) { return interval_.nextNotIgnored(start);};
+  // string show(int lastpos) { return interval.show(lastpos);}
+  int nextNotIgnored(int start) { return interval_.nextNotIgnored(start);}
   KeyInfo &getKeyInfo(int keyinfo) {
     static KeyInfo invalidInfo = KeyInfo();
     if ((keyinfo < 0) || ( keyinfo >= int(entries_.size())))
       return invalidInfo;
     else
       return entries_[keyinfo];
-  };
-  void setForDefaultLang(KeyInfo const & defLang) {interval_.setForDefaultLang(defLang);};
-  void addIntervall(int low, int up) { interval_.addIntervall(low, up); };
+  }
+  void setForDefaultLang(KeyInfo const & defLang) {interval_.setForDefaultLang(defLang);}
+  void addIntervall(int low, int up) { interval_.addIntervall(low, up); }
 };
 
 
@@ -1810,7 +2197,7 @@ class MathInfo {
     m.mathSize = m.mathEnd - m.mathStart;
     entries_.push_back(m);
   }
-  bool empty() const { return entries_.empty(); };
+  bool empty() const { return entries_.empty(); }
   size_t getEndPos() const {
     if (entries_.empty() || (actualIdx_ >= entries_.size())) {
       return 0;
@@ -1845,12 +2232,12 @@ class MathInfo {
     }
     return entries_[actualIdx_].mathSize;
   }
-  void incrEntry() { actualIdx_++; };
+  void incrEntry() { actualIdx_++; }
 };
 
 void LatexInfo::buildEntries(bool isPatternString)
 {
-  static regex const rmath("(\\\\)*(\\$|\\\\\\[|\\\\\\]|\\\\(begin|end)\\{((eqnarray|equation|flalign|gather|multline|align|alignat)\\*?)\\})");
+  static regex const rmath("(\\\\)*(\\$|\\\\\\[|\\\\\\]|\\\\(begin|end)\\{((eqnarray|equation|flalign|gather|multline|align|x?x?alignat)\\*?\\})(\\{[0-9]+\\})?)");
   static regex const rkeys("(\\\\)*(\\$|\\\\\\[|\\\\\\]|\\\\((([a-zA-Z]+\\*?)(\\{([a-z]+\\*?)\\}|=[0-9]+[a-z]+)?)))");
   static bool disableLanguageOverride = false;
   smatch sub, submath;
@@ -1891,7 +2278,7 @@ void LatexInfo::buildEntries(bool isPatternString)
         math_end_waiting = false;
       }
       else if ((submath.str(3).compare("end") == 0) &&
-          (submath.str(4).compare(math_end) == 0)) {
+          (submath.str(5).compare(math_end) == 0)) {
         mi.insert(math_end, math_pos, math_prefix_size, pos, submath.str(2).length());
         math_end_waiting = false;
       }
@@ -1901,7 +2288,7 @@ void LatexInfo::buildEntries(bool isPatternString)
     else {
       if (submath.str(3).compare("begin") == 0) {
         math_end_waiting = true;
-        math_end = submath.str(4);
+        math_end = submath.str(5);
         math_pos = submath.position(size_t(2));
         math_prefix_size = submath.str(2).length();
       }
@@ -2030,6 +2417,13 @@ void LatexInfo::buildEntries(bool isPatternString)
          interval_.addIntervall(found._tokenstart, found._tokenstart + mi.getPrefixSize());
          interval_.addIntervall(found._dataEnd - mi.getPostfixSize(), found._dataEnd);
        }
+       else {
+         // Treate all math constructs as simple math
+         interval_.par[found._tokenstart] = '$';
+         interval_.par[found._dataEnd - mi.getPostfixSize()] = '$';
+         interval_.addIntervall(found._tokenstart + 1, found._tokenstart + mi.getPrefixSize());
+         interval_.addIntervall(found._dataEnd - mi.getPostfixSize() + 1, found._dataEnd);
+       }
         evaluatingMath = true;
       }
       else {
@@ -2635,7 +3029,7 @@ int LatexInfo::dispatch(ostringstream &os, int previousStart, KeyInfo &actual)
       // Remove the key with all parameters and following spaces
       size_t pos;
       size_t start;
-      if (interval_.par[actual._dataEnd-1] == ' ')
+      if (interval_.par[actual._dataEnd-1] == ' ' || interval_.par[actual._dataEnd-1] == '}')
         start = actual._dataEnd;
       else
         start = actual._dataEnd+1;
@@ -3071,9 +3465,9 @@ void MatchStringAdv::CreateRegexp(FindAndReplaceOptions const & opt, string rege
                                balanced--;
                                if (balanced < 0)
                                        break;
-                               }
-                               skip = 1;
                        }
+                       skip = 1;
+               }
                if (balanced != 0) {
                        regexIsValid = false;
                        regexError = "Unbalanced curly brackets in regexp \"" + regexp_str + "\"";
@@ -3087,6 +3481,7 @@ void MatchStringAdv::CreateRegexp(FindAndReplaceOptions const & opt, string rege
                        regexError += "Invalid regexp2 \"" + regexp2_str + "\", error = " + regexp2.errorString().toStdString();
        }
 #else
+       (void)par_as_string;
        if (opt.casesensitive) {
                regexp = regex(regexp_str);
                regexp2 = regex(regexp2_str);
@@ -3141,7 +3536,7 @@ MatchStringAdv::MatchStringAdv(lyx::Buffer & buf, FindAndReplaceOptions & opt)
                previous_single_replace = true;
        }
        // When using regexp, braces are hacked already by escape_for_regex()
-       par_as_string = normalize(ds);
+       par_as_string = normalize(ds, opt.ignoreformat);
        open_braces = 0;
        close_wildcards = 0;
 
@@ -3189,7 +3584,8 @@ MatchStringAdv::MatchStringAdv(lyx::Buffer & buf, FindAndReplaceOptions & opt)
                string lead_as_regexp;
                if (lead_size > 0) {
                        lead_as_regexp = string2regex(par_as_string.substr(0, lead_size));
-                       regex_replace(par_as_string_nolead, par_as_string_nolead, "}$", "");
+                       (void)regex_replace(par_as_string_nolead, par_as_string_nolead, "\\$$", "");
+                       (void)regex_replace(par_as_string_nolead, par_as_string_nolead, "}$", "");
                        par_as_string = par_as_string_nolead;
                        LYXERR(Debug::FIND, "lead_as_regexp is '" << lead_as_regexp << "'");
                        LYXERR(Debug::FIND, "par_as_string now is '" << par_as_string << "'");
@@ -3264,7 +3660,7 @@ MatchResult MatchStringAdv::findAux(DocIterator const & cur, int len, bool at_be
 
        docstring docstr = stringifyFromForSearch(opt, cur, len);
        string str;
-       str = normalize(docstr);
+       str = normalize(docstr, opt.ignoreformat);
        if (!opt.ignoreformat) {
                str = correctlanguagesetting(str, false, !opt.ignoreformat);
                // remove closing '}' and '\n' to allow for use of '$' in regex
@@ -3458,16 +3854,22 @@ static bool simple_replace(string &t, string from, string to)
 }
 #endif
 
-string MatchStringAdv::normalize(docstring const & s) const
+string MatchStringAdv::normalize(docstring const & s, bool ignore_format) const
 {
        string t;
        t = lyx::to_utf8(s);
        // Remove \n at begin
        while (!t.empty() && t[0] == '\n')
                t = t.substr(1);
-       // Remove \n at end
-       while (!t.empty() && t[t.size() - 1] == '\n')
-               t = t.substr(0, t.size() - 1);
+       // Remove [%]*\n at end
+       while (!t.empty() && t[t.size() - 1] == '\n') {
+               int count = 1;
+               if (!ignore_format) {
+                       while ((t.size() > 1 + count) && (t[t.size() - 1 - count] == '%'))
+                               count++;
+               }
+               t = t.substr(0, t.size() - count);
+       }
        size_t pos;
        // Handle all other '\n'
        while ((pos = t.find("\n")) != string::npos) {
@@ -3480,13 +3882,24 @@ string MatchStringAdv::normalize(docstring const & s) const
                                t.replace(pos-2, 3, "");
                        }
                }
-               else if (!isAlnumASCII(t[pos+1]) || !isAlnumASCII(t[pos-1])) {
-                       // '\n' adjacent to non-alpha-numerics, discard
-                       t.replace(pos, 1, "");
-               }
                else {
-                       // Replace all other \n with spaces
-                       t.replace(pos, 1, " ");
+                       if (!isAlnumASCII(t[pos+1]) || !isAlnumASCII(t[pos-1])) {
+                               // '\n' adjacent to non-alpha-numerics, discard
+                               t.replace(pos, 1, "");
+                       }
+                       else {
+                               // Replace all other \n with spaces
+                               t.replace(pos, 1, " ");
+                       }
+                       if (!ignore_format) {
+                               int count = 0;
+                               while ((pos > count + 1) && (t[pos - 1 -count] == '%')) {
+                                       count++;
+                               }
+                               if (count > 0) {
+                                       t.replace(pos - count, count, "");
+                               }
+                       }
                }
        }
        // Remove stale empty \emph{}, \textbf{} and similar blocks from latexify
@@ -3527,11 +3940,14 @@ docstring stringifyFromCursor(DocIterator const & cur, int len)
                else {
                        runparams.for_searchAdv = OutputParams::SearchWithDeleted;
                }
+               if (ignoreFormats.getNonContent()) {
+                       runparams.for_searchAdv |= OutputParams::SearchNonOutput;
+               }
                LYXERR(Debug::FIND, "Stringifying with cur: "
-                      << cur << ", from pos: " << cur.pos() << ", end: " << end);
-               return par.asString(cur.pos(), end,
-                       option,
-                       &runparams);
+                       << cur << ", from pos: " << cur.pos() << ", end: " << end);
+               return from_utf8(latexNamesToUtf8(par.asString(cur.pos(), end,
+                                                               option,
+                                                               &runparams)));
        } else if (cur.inMathed()) {
                CursorSlice cs = cur.top();
                MathData md = cs.cell();
@@ -3540,10 +3956,9 @@ docstring stringifyFromCursor(DocIterator const & cur, int len)
                         ? md.end()
                         : md.begin() + cs.pos() + len );
                MathData md2;
-               for (MathData::const_iterator it = md.begin() + cs.pos();
-                    it != it_end; ++it)
+               for (MathData::const_iterator it = md.begin() + cs.pos(); it != it_end; ++it)
                        md2.push_back(*it);
-               docstring s = asString(md2);
+               docstring s = from_utf8(latexNamesToUtf8(asString(md2)));
                LYXERR(Debug::FIND, "Stringified math: '" << s << "'");
                return s;
        }
@@ -3551,7 +3966,6 @@ docstring stringifyFromCursor(DocIterator const & cur, int len)
        return docstring();
 }
 
-
 /** Computes the LaTeX export of buf starting from cur and ending len positions
  * after cur, if len is positive, or at the paragraph or innermost inset end
  * if len is -1.
@@ -3580,6 +3994,9 @@ docstring latexifyFromCursor(DocIterator const & cur, int len)
        else {
                runparams.for_searchAdv = OutputParams::SearchWithDeleted;
        }
+       if (ignoreFormats.getNonContent()) {
+               runparams.for_searchAdv |= OutputParams::SearchNonOutput;
+       }
 
        if (cur.inTexted()) {
                // @TODO what about searching beyond/across paragraph breaks ?
@@ -3756,14 +4173,14 @@ MatchResult findAdvFinalize(DocIterator & cur, MatchStringAdv const & match, Mat
              max_match.match_len = actual_match_len;
               maxl = len;
               if (maxl - minl < 4)
-                len = (int)((maxl + minl)/2);
+                len = (maxl + minl)/2;
               else
-                len = (int)(minl + (maxl - minl + 3)/4);
+                len = minl + (maxl - minl + 3)/4;
             }
             else {
               // (actual_match_len < max_match.match_len)
               minl = len + 1;
-              len = (int)((maxl + minl)/2);
+              len = (maxl + minl)/2;
             }
           }
          len = minl;
@@ -4145,10 +4562,10 @@ static int findAdvReplace(BufferView * bv, FindAndReplaceOptions const & opt, Ma
        ostringstream oss;
        repl_buffer_orig.write(oss);
        string lyx = oss.str();
-       if (matchAdv.valid_matches > 0) {
-         replaceMatches(lyx, matchAdv.valid_matches, matchAdv.matches);
-       }
-       Buffer repl_buffer("", false);
+       if (matchAdv.valid_matches > 0)
+               replaceMatches(lyx, matchAdv.valid_matches, matchAdv.matches);
+       Buffer repl_buffer(string(), false);
+       repl_buffer.setInternal(true);
        repl_buffer.setUnnamed(true);
        LASSERT(repl_buffer.readString(lyx), return 0);
        if (opt.keep_case && sel_len >= 2) {