1 # This file is part of lyx2lyx
2 # -*- coding: utf-8 -*-
3 # Copyright (C) 2011 The LyX team
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 This module offers several free functions to help with lyx2lyx'ing.
21 More documentaton is below, but here is a quick guide to what
22 they do. Optional arguments are marked by brackets.
24 add_to_preamble(document, text):
25 Here, text can be either a single line or a list of lines. It
26 is bad practice to pass something with embedded newlines, but
27 we will handle that properly.
28 The routine checks to see whether the provided material is
29 already in the preamble. If not, it adds it.
30 Prepends a comment "% Added by lyx2lyx" to text.
32 insert_to_preamble(document, text[, index]):
33 Here, text can be either a single line or a list of lines. It
34 is bad practice to pass something with embedded newlines, but
35 we will handle that properly.
36 The routine inserts text at document.preamble[index], where by
37 default index is 0, so the material is inserted at the beginning.
38 Prepends a comment "% Added by lyx2lyx" to text.
41 Here cmd should be a list of strings (lines), which we want to
42 wrap in ERT. Returns a list of strings so wrapped.
43 A call to this routine will often go something like this:
44 i = find_token('\\begin_inset FunkyInset', ...)
45 j = find_end_of_inset(document.body, i)
46 content = lyx2latex(document[i:j + 1])
47 ert = put_cmd_in_ert(content)
48 document.body[i:j+1] = ert
50 get_ert(lines, i[, verbatim]):
51 Here, lines is a list of lines of LyX material containing an ERT inset,
52 whose content we want to convert to LaTeX. The ERT starts at index i.
53 If the optional (by default: False) bool verbatim is True, the content
54 of the ERT is returned verbatim, that is in LyX syntax (not LaTeX syntax)
55 for the use in verbatim insets.
57 lyx2latex(document, lines):
58 Here, lines is a list of lines of LyX material we want to convert
59 to LaTeX. We do the best we can and return a string containing
60 the translated material.
62 lyx2verbatim(document, lines):
63 Here, lines is a list of lines of LyX material we want to convert
64 to verbatim material (used in ERT an the like). We do the best we
65 can and return a string containing the translated material.
68 Convert lengths (in LyX form) to their LaTeX representation. Returns
69 (bool, length), where the bool tells us if it was a percentage, and
70 the length is the LaTeX representation.
72 convert_info_insets(document, type, func):
73 Applies func to the argument of all info insets matching certain types
74 type : the type to match. This can be a regular expression.
75 func : function from string to string to apply to the "arg" field of
78 is_document_option(document, option):
79 Find if _option_ is a document option (\\options in the header).
81 insert_document_option(document, option):
82 Insert _option_ as a document option.
84 remove_document_option(document, option):
85 Remove _option_ as a document option.
87 revert_language(document, lyxname, babelname="", polyglossianame=""):
88 Reverts native language support to ERT
89 If babelname or polyglossianame is empty, it is assumed
90 this language package is not supported for the given language.
93 from __future__ import print_function
95 from parser_tools import (find_token, find_end_of_inset, get_containing_layout,
96 get_containing_inset, get_value, get_bool_value)
97 from unicode_symbols import unicode_reps
99 # This will accept either a list of lines or a single line.
100 # It is bad practice to pass something with embedded newlines,
101 # though we will handle that.
102 def add_to_preamble(document, text):
103 " Add text to the preamble if it is not already there. "
105 if not type(text) is list:
106 # split on \n just in case
107 # it'll give us the one element list we want
108 # if there's no \n, too
109 text = text.split('\n')
112 prelen = len(document.preamble)
114 i = find_token(document.preamble, text[0], i)
117 # we need a perfect match
120 if i >= prelen or line != document.preamble[i]:
127 document.preamble.extend(["% Added by lyx2lyx"])
128 document.preamble.extend(text)
131 # Note that text can be either a list of lines or a single line.
132 # It should really be a list.
133 def insert_to_preamble(document, text, index = 0):
134 """ Insert text to the preamble at a given line"""
136 if not type(text) is list:
137 # split on \n just in case
138 # it'll give us the one element list we want
139 # if there's no \n, too
140 text = text.split('\n')
142 text.insert(0, "% Added by lyx2lyx")
143 document.preamble[index:index] = text
146 # A dictionary of Unicode->LICR mappings for use in a Unicode string's translate() method
147 # Created from the reversed list to keep the first of alternative definitions.
148 licr_table = {ord(ch): cmd for cmd, ch in unicode_reps[::-1]}
150 def put_cmd_in_ert(cmd, is_open=False, as_paragraph=False):
152 Return ERT inset wrapping `cmd` as a list of strings.
154 `cmd` can be a string or list of lines. Non-ASCII characters are converted
155 to the respective LICR macros if defined in unicodesymbols,
156 `is_open` is a boolean setting the inset status to "open",
157 `as_paragraph` wraps the ERT inset in a Standard paragraph.
160 status = {False:"collapsed", True:"open"}
161 ert_inset = ["\\begin_inset ERT", "status %s"%status[is_open], "",
162 "\\begin_layout Plain Layout", "",
163 # content here ([5:5])
164 "\\end_layout", "", "\\end_inset"]
166 paragraph = ["\\begin_layout Standard",
167 # content here ([1:1])
168 "", "", "\\end_layout", ""]
169 # ensure cmd is an unicode instance and make it "LyX safe".
170 if isinstance(cmd, list):
171 cmd = u"\n".join(cmd)
172 elif sys.version_info[0] == 2 and isinstance(cmd, str):
173 cmd = cmd.decode('utf8')
174 cmd = cmd.translate(licr_table)
175 cmd = cmd.replace("\\", "\n\\backslash\n")
177 ert_inset[5:5] = cmd.splitlines()
180 paragraph[1:1] = ert_inset
184 def get_ert(lines, i, verbatim = False):
185 'Convert an ERT inset into LaTeX.'
186 if not lines[i].startswith("\\begin_inset ERT"):
188 j = find_end_of_inset(lines, i)
191 while i < j and not lines[i].startswith("status"):
197 if lines[i] == "\\begin_layout Plain Layout":
202 while i + 1 < j and lines[i+1] == "":
204 elif lines[i] == "\\end_layout":
205 while i + 1 < j and lines[i+1] == "":
207 elif lines[i] == "\\backslash":
209 ret = ret + "\n" + lines[i] + "\n"
218 def lyx2latex(document, lines):
219 'Convert some LyX stuff into corresponding LaTeX stuff, as best we can.'
226 for curline in range(len(lines)):
227 line = lines[curline]
228 if line.startswith("\\begin_inset Note Note"):
229 # We want to skip LyX notes, so remember where the inset ends
230 note_end = find_end_of_inset(lines, curline + 1)
232 elif note_end >= curline:
235 elif line.startswith("\\begin_inset ERT"):
236 # We don't want to replace things inside ERT, so figure out
237 # where the end of the inset is.
238 ert_end = find_end_of_inset(lines, curline + 1)
240 elif line.startswith("\\begin_inset Formula"):
242 elif line.startswith("\\begin_inset Quotes"):
243 # For now, we do a very basic reversion. Someone who understands
244 # quotes is welcome to fix it up.
245 qtype = line[20:].strip()
259 elif line.startswith("\\begin_inset Newline newline"):
261 elif line.startswith("\\noindent"):
262 line = "\\noindent " # we need the space behind the command
263 elif line.startswith("\\begin_inset space"):
264 line = line[18:].strip()
265 if line.startswith("\\hspace"):
266 # Account for both \hspace and \hspace*
269 elif line == "\\space{}":
271 elif line == "\\thinspace{}":
274 # The LyX length is in line[8:], after the \length keyword
275 length = latex_length(line[8:])[1]
276 line = hspace + "{" + length + "}"
278 elif line.isspace() or \
279 line.startswith("\\begin_layout") or \
280 line.startswith("\\end_layout") or \
281 line.startswith("\\begin_inset") or \
282 line.startswith("\\end_inset") or \
283 line.startswith("\\lang") or \
284 line.strip() == "status collapsed" or \
285 line.strip() == "status open":
289 # this needs to be added to the preamble because of cases like
290 # \textmu, \textbackslash, etc.
291 add_to_preamble(document, ['% added by lyx2lyx for converted index entries',
292 '\\@ifundefined{textmu}',
293 ' {\\usepackage{textcomp}}{}'])
294 # a lossless reversion is not possible
295 # try at least to handle some common insets and settings
296 if ert_end >= curline:
297 line = line.replace(r'\backslash', '\\')
299 # No need to add "{}" after single-nonletter macros
300 line = line.replace('&', '\\&')
301 line = line.replace('#', '\\#')
302 line = line.replace('^', '\\textasciicircum{}')
303 line = line.replace('%', '\\%')
304 line = line.replace('_', '\\_')
305 line = line.replace('$', '\\$')
307 # Do the LyX text --> LaTeX conversion
308 for rep in unicode_reps:
309 line = line.replace(rep[1], rep[0])
310 line = line.replace(r'\backslash', r'\textbackslash{}')
311 line = line.replace(r'\series bold', r'\bfseries{}').replace(r'\series default', r'\mdseries{}')
312 line = line.replace(r'\shape italic', r'\itshape{}').replace(r'\shape smallcaps', r'\scshape{}')
313 line = line.replace(r'\shape slanted', r'\slshape{}').replace(r'\shape default', r'\upshape{}')
314 line = line.replace(r'\emph on', r'\em{}').replace(r'\emph default', r'\em{}')
315 line = line.replace(r'\noun on', r'\scshape{}').replace(r'\noun default', r'\upshape{}')
316 line = line.replace(r'\bar under', r'\underbar{').replace(r'\bar default', r'}')
317 line = line.replace(r'\family sans', r'\sffamily{}').replace(r'\family default', r'\normalfont{}')
318 line = line.replace(r'\family typewriter', r'\ttfamily{}').replace(r'\family roman', r'\rmfamily{}')
319 line = line.replace(r'\InsetSpace ', r'').replace(r'\SpecialChar ', r'')
324 def lyx2verbatim(document, lines):
325 'Convert some LyX stuff into corresponding verbatim stuff, as best we can.'
327 content = lyx2latex(document, lines)
328 content = re.sub(r'\\(?!backslash)', r'\n\\backslash\n', content)
333 def latex_length(slen):
335 Convert lengths to their LaTeX representation. Returns (bool, length),
336 where the bool tells us if it was a percentage, and the length is the
337 LaTeX representation.
341 # the slen has the form
342 # ValueUnit+ValueUnit-ValueUnit or
343 # ValueUnit+-ValueUnit
344 # the + and - (glue lengths) are optional
345 # the + always precedes the -
347 # Convert relative lengths to LaTeX units
348 units = {"col%": "\\columnwidth",
349 "text%": "\\textwidth",
350 "page%": "\\paperwidth",
351 "line%": "\\linewidth",
352 "theight%": "\\textheight",
353 "pheight%": "\\paperheight",
354 "baselineskip%": "\\baselineskip"
356 for unit in list(units.keys()):
361 minus = slen.rfind("-", 1, i)
362 plus = slen.rfind("+", 0, i)
363 latex_unit = units[unit]
364 if plus == -1 and minus == -1:
366 value = str(float(value)/100)
367 end = slen[i + len(unit):]
368 slen = value + latex_unit + end
370 value = slen[plus + 1:i]
371 value = str(float(value)/100)
372 begin = slen[:plus + 1]
373 end = slen[i+len(unit):]
374 slen = begin + value + latex_unit + end
376 value = slen[minus + 1:i]
377 value = str(float(value)/100)
378 begin = slen[:minus + 1]
379 slen = begin + value + latex_unit
381 # replace + and -, but only if the - is not the first character
382 slen = slen[0] + slen[1:].replace("+", " plus ").replace("-", " minus ")
383 # handle the case where "+-1mm" was used, because LaTeX only understands
384 # "plus 1mm minus 1mm"
385 if slen.find("plus minus"):
386 lastvaluepos = slen.rfind(" ")
387 lastvalue = slen[lastvaluepos:]
388 slen = slen.replace(" ", lastvalue + " ")
389 return (percent, slen)
392 def length_in_bp(length):
393 " Convert a length in LyX format to its value in bp units "
395 em_width = 10.0 / 72.27 # assume 10pt font size
396 text_width = 8.27 / 1.7 # assume A4 with default margins
397 # scale factors are taken from Length::inInch()
398 scales = {"bp" : 1.0,
399 "cc" : (72.0 / (72.27 / (12.0 * 0.376 * 2.845))),
400 "cm" : (72.0 / 2.54),
401 "dd" : (72.0 / (72.27 / (0.376 * 2.845))),
402 "em" : (72.0 * em_width),
403 "ex" : (72.0 * em_width * 0.4305),
405 "mm" : (72.0 / 25.4),
406 "mu" : (72.0 * em_width / 18.0),
407 "pc" : (72.0 / (72.27 / 12.0)),
408 "pt" : (72.0 / (72.27)),
409 "sp" : (72.0 / (72.27 * 65536.0)),
410 "text%" : (72.0 * text_width / 100.0),
411 "col%" : (72.0 * text_width / 100.0), # assume 1 column
412 "page%" : (72.0 * text_width * 1.7 / 100.0),
413 "line%" : (72.0 * text_width / 100.0),
414 "theight%" : (72.0 * text_width * 1.787 / 100.0),
415 "pheight%" : (72.0 * text_width * 2.2 / 100.0)}
417 rx = re.compile(r'^\s*([^a-zA-Z%]+)([a-zA-Z%]+)\s*$')
420 document.warning("Invalid length value: " + length + ".")
424 if not unit in scales.keys():
425 document.warning("Unknown length unit: " + unit + ".")
427 return "%g" % (float(value) * scales[unit])
430 def revert_flex_inset(lines, name, LaTeXname):
431 " Convert flex insets to TeX code "
434 i = find_token(lines, '\\begin_inset Flex ' + name, i)
437 z = find_end_of_inset(lines, i)
439 document.warning("Can't find end of Flex " + name + " inset.")
442 # remove the \end_inset
443 lines[z - 2:z + 1] = put_cmd_in_ert("}")
444 # we need to reset character layouts if necessary
445 j = find_token(lines, '\\emph on', i, z)
446 k = find_token(lines, '\\noun on', i, z)
447 l = find_token(lines, '\\series', i, z)
448 m = find_token(lines, '\\family', i, z)
449 n = find_token(lines, '\\shape', i, z)
450 o = find_token(lines, '\\color', i, z)
451 p = find_token(lines, '\\size', i, z)
452 q = find_token(lines, '\\bar under', i, z)
453 r = find_token(lines, '\\uuline on', i, z)
454 s = find_token(lines, '\\uwave on', i, z)
455 t = find_token(lines, '\\strikeout on', i, z)
457 lines.insert(z - 2, "\\emph default")
459 lines.insert(z - 2, "\\noun default")
461 lines.insert(z - 2, "\\series default")
463 lines.insert(z - 2, "\\family default")
465 lines.insert(z - 2, "\\shape default")
467 lines.insert(z - 2, "\\color inherit")
469 lines.insert(z - 2, "\\size default")
471 lines.insert(z - 2, "\\bar default")
473 lines.insert(z - 2, "\\uuline default")
475 lines.insert(z - 2, "\\uwave default")
477 lines.insert(z - 2, "\\strikeout default")
478 lines[i:i + 4] = put_cmd_in_ert(LaTeXname + "{")
482 def revert_font_attrs(lines, name, LaTeXname):
483 " Reverts font changes to TeX code "
487 i = find_token(lines, name + ' on', i)
490 j = find_token(lines, name + ' default', i)
491 k = find_token(lines, name + ' on', i + 1)
492 # if there is no default set, the style ends with the layout
493 # assure hereby that we found the correct layout end
494 if j != -1 and (j < k or k == -1):
495 lines[j:j + 1] = put_cmd_in_ert("}")
497 j = find_token(lines, '\\end_layout', i)
498 lines[j:j] = put_cmd_in_ert("}")
499 lines[i:i + 1] = put_cmd_in_ert(LaTeXname + "{")
503 # now delete all remaining lines that manipulate this attribute
506 i = find_token(lines, name, i)
514 def revert_layout_command(lines, name, LaTeXname):
515 " Reverts a command from a layout to TeX code "
518 i = find_token(lines, '\\begin_layout ' + name, i)
522 # find the next layout
525 j = find_token(lines, '\\begin_layout', j)
527 # if nothing was found it was the last layout of the document
529 lines[l - 4:l - 4] = put_cmd_in_ert("}")
531 # exclude plain layout because this can be TeX code or another inset
532 elif lines[j] != '\\begin_layout Plain Layout':
533 lines[j - 2:j - 2] = put_cmd_in_ert("}")
537 lines[i] = '\\begin_layout Standard'
538 lines[i + 1:i + 1] = put_cmd_in_ert(LaTeXname + "{")
543 " Converts an RRGGBB-type hexadecimal string to a float in [0.0,1.0] "
550 return str(val / 256.0)
554 "'true' goes to True, case-insensitively, and we strip whitespace."
555 s = s.strip().lower()
559 def convert_info_insets(document, type, func):
560 "Convert info insets matching type using func."
562 type_re = re.compile(r'^type\s+"(%s)"$' % type)
563 arg_re = re.compile(r'^arg\s+"(.*)"$')
565 i = find_token(document.body, "\\begin_inset Info", i)
568 t = type_re.match(document.body[i + 1])
570 arg = arg_re.match(document.body[i + 2])
572 new_arg = func(arg.group(1))
573 document.body[i + 2] = 'arg "%s"' % new_arg
577 def insert_document_option(document, option):
578 "Insert _option_ as a document option."
580 # Find \options in the header
581 i = find_token(document.header, "\\options", 0)
582 # if the options does not exists add it after the textclass
584 i = find_token(document.header, "\\textclass", 0) + 1
585 document.header.insert(i, r"\options %s" % option)
587 # otherwise append to options
588 if not is_document_option(document, option):
589 document.header[i] += ",%s" % option
592 def remove_document_option(document, option):
593 """ Remove _option_ as a document option."""
595 i = find_token(document.header, "\\options")
596 options = get_value(document.header, "\\options", i)
597 options = [op.strip() for op in options.split(',')]
599 # Remove `option` from \options
600 options = [op for op in options if op != option]
603 document.header[i] = "\\options " + ','.join(options)
605 del document.header[i]
608 def is_document_option(document, option):
609 "Find if _option_ is a document option"
611 options = get_value(document.header, "\\options")
612 options = [op.strip() for op in options.split(',')]
613 return option in options
616 singlepar_insets = [s.strip() for s in
617 u"Argument, Caption Above, Caption Below, Caption Bicaption,"
618 u"Caption Centered, Caption FigCaption, Caption Standard, Caption Table,"
619 u"Flex Chemistry, Flex Fixme_Note, Flex Latin, Flex ListOfSlides,"
620 u"Flex Missing_Figure, Flex PDF-Annotation, Flex PDF-Comment-Setup,"
621 u"Flex Reflectbox, Flex S/R expression, Flex Sweave Input File,"
622 u"Flex Sweave Options, Flex Thanks_Reference, Flex URL, Foot InTitle,"
623 u"IPADeco, Index, Info, Phantom, Script".split(',')]
624 # print(singlepar_insets)
626 def revert_language(document, lyxname, babelname="", polyglossianame=""):
627 " Revert native language support "
629 # Does the document use polyglossia?
630 use_polyglossia = False
631 if get_bool_value(document.header, "\\use_non_tex_fonts"):
632 i = find_token(document.header, "\\language_package")
634 document.warning("Malformed document! Missing \\language_package")
636 pack = get_value(document.header, "\\language_package", i)
637 if pack in ("default", "auto"):
638 use_polyglossia = True
640 # Do we use this language with polyglossia?
641 with_polyglossia = use_polyglossia and polyglossianame != ""
642 # Do we use this language with babel?
643 with_babel = with_polyglossia == False and babelname != ""
645 # Are we dealing with a primary or secondary language?
646 primary = document.language == lyxname
649 # Main language first
650 orig_doc_language = document.language
652 # Change LyX document language to English (we will tell LaTeX
653 # to use the original language at the end of this function):
654 document.language = "english"
655 i = find_token(document.header, "\\language %s" % lyxname, 0)
657 document.header[i] = "\\language english"
659 # Now look for occurences in the body
662 i = find_token(document.body, "\\lang", i+1)
665 if document.body[i].startswith("\\lang %s" % lyxname):
667 texname = use_polyglossia and polyglossianame or babelname
668 elif primary and document.body[i].startswith("\\lang english"):
669 # Since we switched the main language manually, English parts need to be marked
674 parent = get_containing_layout(document.body, i)
675 i_e = parent[2] # end line no,
676 # print(i, texname, parent, document.body[i+1], file=sys.stderr)
678 # Move leading space to the previous line:
679 if document.body[i+1].startswith(" "):
680 document.body[i+1] = document.body[i+1][1:]
681 document.body.insert(i, " ")
684 # TODO: handle nesting issues with font attributes, e.g.
685 # \begin_layout Standard
691 # — јужнословенски јазик, дел од групата на словенски јазици од јазичното
692 # семејство на индоевропски јазици.
693 # Македонскиот е службен и национален јазик во Македонија.
696 # Ensure correct handling of list labels
697 if (parent[0] in ["Labeling", "Description"]
698 and not " " in "\n".join(document.body[parent[3]:i])):
699 # line `i+1` is first line of a list item,
700 # part before a space character is the label
701 # TODO: insets or language change before first space character
702 labelline = document.body[i+1].split(' ', 1)
703 if len(labelline) > 1:
704 # Insert a space in the (original) document language
705 # between label and remainder.
706 # print(" Label:", labelline, file=sys.stderr)
707 lines = [labelline[0],
708 "\\lang %s" % orig_doc_language,
710 "\\lang %s" % (primary and "english" or lyxname),
712 document.body[i+1:i+2] = lines
715 # Find out where to end the language change.
718 langswitch = find_token(document.body, "\\lang", langswitch+1, i_e)
721 # print(" ", langswitch, document.body[langswitch], file=sys.stderr)
723 i_a = parent[3] # paragraph start line
724 container = get_containing_inset(document.body[i_a:i_e], langswitch-i_a)
725 if container and container[1] < langswitch-i_a and container[2] > langswitch-i_a:
726 # print(" inset", container, file=sys.stderr)
731 # use function or environment?
732 singlepar = i_e - i < 3
733 if not singlepar and parent[0] == "Plain Layout":
734 # environment not allowed in some insets
735 container = get_containing_inset(document.body, i)
736 singlepar = container[0] in singlepar_insets
738 # Delete empty language switches:
739 if not "".join(document.body[i+1:i_e]):
740 del document.body[i:i_e]
746 begin_cmd = "\\text%s{"%texname
748 begin_cmd = "\\foreignlanguage{%s}{" % texname
752 begin_cmd = "\\begin{%s}"%texname
753 end_cmd = "\\end{%s}"%texname
755 begin_cmd = "\\begin{otherlanguage}{%s}" % texname
756 end_cmd = "\\end{otherlanguage}"
758 if (not primary or texname == "english"):
760 document.body[i_e:i_e] = put_cmd_in_ert(end_cmd)
761 document.body[i+1:i+1] = put_cmd_in_ert(begin_cmd)
762 except UnboundLocalError:
766 if not (primary or secondary):
769 # Make the language known to Babel/Polyglossia and ensure the correct
773 # add as global option
774 insert_document_option(document, babelname)
775 # Since user options are appended to the document options,
776 # Babel will treat `babelname` as primary language.
778 doc_lang_switch = "\\selectlanguage{%s}" % orig_doc_language
780 # Define language in the user preamble
781 # (don't use \AtBeginDocument, this fails with some languages).
782 add_to_preamble(document, ["\\usepackage{polyglossia}",
783 "\\setotherlanguage{%s}" % polyglossianame])
785 # Changing the main language must be done in the document body.
786 doc_lang_switch = "\\resetdefaultlanguage{%s}" % polyglossianame
788 # Reset LaTeX main language if required and not already done
789 if doc_lang_switch and doc_lang_switch[1:] not in document.body[8:20]:
790 document.body[2:2] = put_cmd_in_ert(doc_lang_switch,
791 is_open=True, as_paragraph=True)