[lyx.git] / lib / lyx2lyx / lyx_2_0.py

# -*- coding: utf-8 -*-
# This file is part of lyx2lyx
# -*- coding: utf-8 -*-
# Copyright (C) 2010 The LyX team
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

""" Convert files to the file format generated by lyx 2.0"""

import re, string
import unicodedata
import sys, os

from parser_tools import find_token, find_end_of, find_tokens, get_value, get_value_string

####################################################################
# Private helper functions

def remove_option(document, m, option):
    l = document.body[m].find(option)
    if l != -1:
        val = document.body[m][l:].split('"')[1]
        document.body[m] = document.body[m][:l - 1] + document.body[m][l+len(option + '="' + val + '"'):]
    return l

def find_end_of_inset(lines, i):
    " Find end of inset, where lines[i] is included."
    return find_end_of(lines, i, "\\begin_inset", "\\end_inset")


# Note that text can be either a list of lines or a single line.
def add_to_preamble(document, text):
    """ Add text to the preamble if it is not already there.
    Only the first line is checked!"""

    if not type(text) is list:
      # split on \n just in case
      # it'll give us the one element list we want
      # if there's no \n, too
      text = text.split('\n')

    if find_token(document.preamble, text[0], 0) != -1:
        return

    document.preamble.extend(text)


def insert_to_preamble(index, document, text):
    """ Insert text to the preamble at a given line"""

    document.preamble.insert(index, text)


def read_unicodesymbols():
    " Read the unicodesymbols list of unicode characters and corresponding commands."
    pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
    fp = open(os.path.join(pathname.strip('lyx2lyx'), 'unicodesymbols'))
    spec_chars = []
    # Two backslashes, followed by some non-word character, and then a character
    # in brackets. The idea is to check for constructs like: \"{u}, which is how
    # they are written in the unicodesymbols file; but they can also be written
    # as: \"u or even \" u.
    r = re.compile(r'\\\\(\W)\{(\w)\}')
    for line in fp.readlines():
        if line[0] != '#' and line.strip() != "":
            line=line.replace(' "',' ') # remove all quotation marks with spaces before
            line=line.replace('" ',' ') # remove all quotation marks with spaces after
            line=line.replace(r'\"','"') # replace \" by " (for characters with diaeresis)
            try:
                [ucs4,command,dead] = line.split(None,2)
                if command[0:1] != "\\":
                    continue
                spec_chars.append([command, unichr(eval(ucs4))])
            except:
                continue
            m = r.match(command)
            if m != None:
                command = "\\\\"
                # If the character is a double-quote, then we need to escape it, too,
                # since it is done that way in the LyX file.
                if m.group(1) == "\"":
                    command += "\\"
                commandbl = command
                command += m.group(1) + m.group(2)
                commandbl += m.group(1) + ' ' + m.group(2)
                spec_chars.append([command, unichr(eval(ucs4))])
                spec_chars.append([commandbl, unichr(eval(ucs4))])
    fp.close()
    return spec_chars


unicode_reps = read_unicodesymbols()


# DO NOT USE THIS ROUTINE ANY MORE. Better yet, replace the uses that
# have been made of it with uses of put_cmd_in_ert.
def old_put_cmd_in_ert(string):
    for rep in unicode_reps:
        string = string.replace(rep[1], rep[0].replace('\\\\', '\\'))
    string = string.replace('\\', "\\backslash\n")
    string = "\\begin_inset ERT\nstatus collapsed\n\\begin_layout Plain Layout\n" \
      + string + "\n\\end_layout\n\\end_inset"
    return string


# This routine wraps some content in an ERT inset. 
#
# NOTE: The function accepts either a single string or a LIST of strings as
# argument. But it returns a LIST of strings, split on \n, so that it does 
# not have embedded newlines.
# 
# This is how lyx2lyx represents a LyX document: as a list of strings, 
# each representing a line of a LyX file. Embedded newlines confuse 
# lyx2lyx very much.
#
# A call to this routine will often go something like this:
#   i = find_token('\\begin_inset FunkyInset', ...)
#   ...
#   j = find_end_of_inset(document.body, i)
#   content = ...extract content from insets
#   # that could be as simple as: 
#   # content = lyx2latex(document[i:j + 1])
#   ert = put_cmd_in_ert(content)
#   document.body[i:j] = ert
# Now, before we continue, we need to reset i appropriately. Normally,
# this would be: 
#   i += len(ert)
# That puts us right after the ERT we just inserted.
#
def put_cmd_in_ert(arg):
    ret = ["\\begin_inset ERT", "status collapsed", "\\begin_layout Plain Layout", ""]
    # Despite the warnings just given, it will be faster for us to work
    # with a single string internally. That way, we only go through the
    # unicode_reps loop once.
    if type(arg) is list:
      s = "\n".join(arg)
    else:
      s = arg
    for rep in unicode_reps:
      s = s.replace(rep[1], rep[0].replace('\\\\', '\\'))
    s = s.replace('\\', "\\backslash\n")
    ret += s.splitlines()
    ret += ["\\end_layout", "\\end_inset"]
    return ret

            
def lyx2latex(document, lines):
    'Convert some LyX stuff into corresponding LaTeX stuff, as best we can.'
    # clean up multiline stuff
    content = ""
    ert_end = 0
    note_end = 0
    hspace = ""

    for curline in range(len(lines)):
      line = lines[curline]
      if line.startswith("\\begin_inset Note Note"):
          # We want to skip LyX notes, so remember where the inset ends
          note_end = find_end_of_inset(lines, curline + 1)
          continue
      elif note_end >= curline:
          # Skip LyX notes
          continue
      elif line.startswith("\\begin_inset ERT"):
          # We don't want to replace things inside ERT, so figure out
          # where the end of the inset is.
          ert_end = find_end_of_inset(lines, curline + 1)
          continue
      elif line.startswith("\\begin_inset Formula"):
          line = line[20:]
      elif line.startswith("\\begin_inset Quotes"):
          # For now, we do a very basic reversion. Someone who understands
          # quotes is welcome to fix it up.
          qtype = line[20:].strip()
          # lang = qtype[0]
          side = qtype[1]
          dbls = qtype[2]
          if side == "l":
              if dbls == "d":
                  line = "``"
              else:
                  line = "`"
          else:
              if dbls == "d":
                  line = "''"
              else:
                  line = "'"
      elif line.startswith("\\begin_inset space"):
          line = line[18:].strip()
          if line.startswith("\\hspace"):
              # Account for both \hspace and \hspace*
              hspace = line[:-2]
              continue
          elif line == "\\space{}":
              line = "\\ "
          elif line == "\\thinspace{}":
              line = "\\,"
      elif hspace != "":
          # The LyX length is in line[8:], after the \length keyword
          # latex_length returns "bool,length"
          length = latex_length(line[8:]).split(",")[1]
          line = hspace + "{" + length + "}"
          hspace = ""
      elif line.isspace() or \
            line.startswith("\\begin_layout") or \
            line.startswith("\\end_layout") or \
            line.startswith("\\begin_inset") or \
            line.startswith("\\end_inset") or \
            line.startswith("\\lang") or \
            line.strip() == "status collapsed" or \
            line.strip() == "status open":
          #skip all that stuff
          continue

      # this needs to be added to the preamble because of cases like
      # \textmu, \textbackslash, etc.
      add_to_preamble(document, ['% added by lyx2lyx for converted index entries',
                                 '\\@ifundefined{textmu}',
                                 ' {\\usepackage{textcomp}}{}'])
      # a lossless reversion is not possible
      # try at least to handle some common insets and settings
      if ert_end >= curline:
          line = line.replace(r'\backslash', '\\')
      else:
          # No need to add "{}" after single-nonletter macros
          line = line.replace('&', '\\&')
          line = line.replace('#', '\\#')
          line = line.replace('^', '\\textasciicircum{}')
          line = line.replace('%', '\\%')
          line = line.replace('_', '\\_')
          line = line.replace('$', '\\$')

          # Do the LyX text --> LaTeX conversion
          for rep in unicode_reps:
            line = line.replace(rep[1], rep[0] + "{}")
          line = line.replace(r'\backslash', r'\textbackslash{}')
          line = line.replace(r'\series bold', r'\bfseries{}').replace(r'\series default', r'\mdseries{}')
          line = line.replace(r'\shape italic', r'\itshape{}').replace(r'\shape smallcaps', r'\scshape{}')
          line = line.replace(r'\shape slanted', r'\slshape{}').replace(r'\shape default', r'\upshape{}')
          line = line.replace(r'\emph on', r'\em{}').replace(r'\emph default', r'\em{}')
          line = line.replace(r'\noun on', r'\scshape{}').replace(r'\noun default', r'\upshape{}')
          line = line.replace(r'\bar under', r'\underbar{').replace(r'\bar default', r'}')
          line = line.replace(r'\family sans', r'\sffamily{}').replace(r'\family default', r'\normalfont{}')
          line = line.replace(r'\family typewriter', r'\ttfamily{}').replace(r'\family roman', r'\rmfamily{}')
          line = line.replace(r'\InsetSpace ', r'').replace(r'\SpecialChar ', r'')
      content += line
    return content


def latex_length(string):
    'Convert lengths to their LaTeX representation.'
    i = 0
    percent = False
    # the string has the form
    # ValueUnit+ValueUnit-ValueUnit or
    # ValueUnit+-ValueUnit
    # the + and - (glue lengths) are optional
    # the + always precedes the -

    # Convert relative lengths to LaTeX units
    units = {"text%":"\\textwidth", "col%":"\\columnwidth",
             "page%":"\\paperwidth", "line%":"\\linewidth",
             "theight%":"\\textheight", "pheight%":"\\paperheight"}
    for unit in units.keys():
        i = string.find(unit)
        if i != -1:
            percent = True
            minus = string.rfind("-", 1, i)
            plus = string.rfind("+", 0, i)
            latex_unit = units[unit]
            if plus == -1 and minus == -1:
                value = string[:i]
                value = str(float(value)/100)
                end = string[i + len(unit):]
                string = value + latex_unit + end
            if plus > minus:
                value = string[plus + 1:i]
                value = str(float(value)/100)
                begin = string[:plus + 1]
                end = string[i+len(unit):]
                string = begin + value + latex_unit + end
            if plus < minus:
                value = string[minus + 1:i]
                value = str(float(value)/100)
                begin = string[:minus + 1]
                string = begin + value + latex_unit

    # replace + and -, but only if the - is not the first character
    string = string[0] + string[1:].replace("+", " plus ").replace("-", " minus ")
    # handle the case where "+-1mm" was used, because LaTeX only understands
    # "plus 1mm minus 1mm"
    if string.find("plus  minus"):
        lastvaluepos = string.rfind(" ")
        lastvalue = string[lastvaluepos:]
        string = string.replace("  ", lastvalue + " ")
    if percent ==  False:
        return "False," + string
    else:
        return "True," + string


def revert_flex_inset(document, name, LaTeXname, position):
  " Convert flex insets to TeX code "
  i = position
  while True:
    i = find_token(document.body, '\\begin_inset Flex ' + name, i)
    if i == -1:
      return
    z = find_end_of_inset(document.body, i)
    if z == -1:
      document.warning("Malformed LyX document: Can't find end of Flex " + name + " inset.")
      return
    # remove the \end_inset
    document.body[z - 2:z + 1] = put_cmd_in_ert("}")
    # we need to reset character layouts if necessary
    j = find_token(document.body, '\\emph on', i, z)
    k = find_token(document.body, '\\noun on', i, z)
    l = find_token(document.body, '\\series', i, z)
    m = find_token(document.body, '\\family', i, z)
    n = find_token(document.body, '\\shape', i, z)
    o = find_token(document.body, '\\color', i, z)
    p = find_token(document.body, '\\size', i, z)
    q = find_token(document.body, '\\bar under', i, z)
    r = find_token(document.body, '\\uuline on', i, z)
    s = find_token(document.body, '\\uwave on', i, z)
    t = find_token(document.body, '\\strikeout on', i, z)
    if j != -1:
      document.body.insert(z - 2, "\\emph default")
    if k != -1:
      document.body.insert(z - 2, "\\noun default")
    if l != -1:
      document.body.insert(z - 2, "\\series default")
    if m != -1:
      document.body.insert(z - 2, "\\family default")
    if n != -1:
      document.body.insert(z - 2, "\\shape default")
    if o != -1:
      document.body.insert(z - 2, "\\color inherit")
    if p != -1:
      document.body.insert(z - 2, "\\size default")
    if q != -1:
      document.body.insert(z - 2, "\\bar default")
    if r != -1:
      document.body.insert(z - 2, "\\uuline default")
    if s != -1:
      document.body.insert(z - 2, "\\uwave default")
    if t != -1:
      document.body.insert(z - 2, "\\strikeout default")
    document.body[i:i + 4] = put_cmd_in_ert(LaTeXname + "{")
    i += 1


def revert_font_attrs(document, name, LaTeXname):
  " Reverts font changes to TeX code "
  i = 0
  changed = False
  while True:
    i = find_token(document.body, name + ' on', i)
    if i == -1:
      return changed
    j = find_token(document.body, name + ' default', i)
    k = find_token(document.body, name + ' on', i + 1)
    # if there is no default set, the style ends with the layout
    # assure hereby that we found the correct layout end
    if j != -1 and (j < k or k == -1):
      document.body[j:j + 1] = put_cmd_in_ert("}")
    else:
      j = find_token(document.body, '\\end_layout', i)
      document.body[j:j] = put_cmd_in_ert("}")
    document.body[i:i + 1] = put_cmd_in_ert(LaTeXname + "{")
    changed = True
    i += 1


def revert_layout_command(document, name, LaTeXname, position):
  " Reverts a command from a layout to TeX code "
  i = position
  while True:
    i = find_token(document.body, '\\begin_layout ' + name, i)
    if i == -1:
      return
    k = -1
    # find the next layout
    j = i + 1
    while k == -1:
      j = find_token(document.body, '\\begin_layout', j)
      l = len(document.body)
      # if nothing was found it was the last layout of the document
      if j == -1:
        document.body[l - 4:l - 4] = put_cmd_in_ert("}")
        k = 0
      # exclude plain layout because this can be TeX code or another inset
      elif document.body[j] != '\\begin_layout Plain Layout':
        document.body[j - 2:j - 2] = put_cmd_in_ert("}")
        k = 0
      else:
        j += 1
    document.body[i] = '\\begin_layout Standard'
    document.body[i + 1:i + 1] = put_cmd_in_ert(LaTeXname + "{")
    i += 1


###############################################################################
###
### Conversion and reversion routines
###
###############################################################################

def revert_swiss(document):
    " Set language german-ch to ngerman "
    i = 0
    if document.language == "german-ch":
        document.language = "ngerman"
        i = find_token(document.header, "\\language", 0)
        if i != -1:
            document.header[i] = "\\language ngerman"
    j = 0
    while True:
        j = find_token(document.body, "\\lang german-ch", j)
        if j == -1:
            return
        document.body[j] = document.body[j].replace("\\lang german-ch", "\\lang ngerman")
        j = j + 1


def revert_tabularvalign(document):
   " Revert the tabular valign option "
   i = 0
   while True:
      i = find_token(document.body, "\\begin_inset Tabular", i)
      if i == -1:
          return
      end = find_end_of_inset(document.body, i)
      if end == -1:
          document.warning("Can't find end of inset at line " + str(i))
          i = j
          continue
      fline = find_token(document.body, "<features", i, end)
      if fline == -1:
          document.warning("Can't find features for inset at line " + str(i))
          i = end
          continue
      p = document.body[fline].find("islongtable")
      if p != -1:
          q = document.body[fline].find("tabularvalignment")
          if q != -1:
              # FIXME
              # This seems wrong: It removes everything after 
              # tabularvalignment, too.
              document.body[fline] = document.body[fline][:q - 1] + '>'
          i = end
          continue

       # no longtable
      tabularvalignment = 'c'
      # which valignment is specified?
      m = document.body[fline].find('tabularvalignment="top"')
      if m != -1:
          tabularvalignment = 't'
      m = document.body[fline].find('tabularvalignment="bottom"')
      if m != -1:
          tabularvalignment = 'b'
      # delete tabularvalignment
      q = document.body[fline].find("tabularvalignment")
      if q != -1:
          # FIXME
          # This seems wrong: It removes everything after 
          # tabularvalignment, too.
          document.body[fline] = document.body[fline][:q - 1] + '>'

      # don't add a box when centered
      if tabularvalignment == 'c':
          i = end
          continue
      subst = ['\\end_layout', '\\end_inset']
      document.body[end:end] = subst # just inserts those lines
      subst = ['\\begin_inset Box Frameless',
          'position "' + tabularvalignment +'"',
          'hor_pos "c"',
          'has_inner_box 1',
          'inner_pos "c"',
          'use_parbox 0',
          # we don't know the width, assume 50%
          'width "50col%"',
          'special "none"',
          'height "1in"',
          'height_special "totalheight"',
          'status open',
          '',
          '\\begin_layout Plain Layout']
      document.body[i:i] = subst # this just inserts the array at i
      i = end + len(subst) # adjust i to save a few cycles


def revert_phantom_types(document, ptype, cmd):
    " Reverts phantom to ERT "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset Phantom " + ptype, i)
      if i == -1:
          return
      end = find_end_of_inset(document.body, i)
      if end == -1:
          document.warning("Can't find end of inset at line " + str(i))
          i += 1
          continue
      blay = find_token(document.body, "\\begin_layout Plain Layout", i, end)
      if blay == -1:
          document.warning("Can't find layout for inset at line " + str(i))
          i = end
          continue
      bend = find_token(document.body, "\\end_layout", blay, end)
      if bend == -1:
          document.warning("Malformed LyX document: Could not find end of Phantom inset's layout.")
          i = end
          continue
      substi = ["\\begin_inset ERT", "status collapsed", "",
                "\\begin_layout Plain Layout", "", "", "\\backslash", 
                cmd + "{", "\\end_layout", "", "\\end_inset"]
      substj = ["\\size default", "", "\\begin_inset ERT", "status collapsed", "",
                "\\begin_layout Plain Layout", "", "}", "\\end_layout", "", "\\end_inset"]
      # do the later one first so as not to mess up the numbering
      document.body[bend:end + 1] = substj
      document.body[i:blay + 1] = substi
      i = end + len(substi) + len(substj) - (end - bend) - (blay - i) - 2


def revert_phantom(document):
    revert_phantom_types(document, "Phantom", "phantom")
    
def revert_hphantom(document):
    revert_phantom_types(document, "HPhantom", "hphantom")

def revert_vphantom(document):
    revert_phantom_types(document, "VPhantom", "vphantom")


def revert_xetex(document):
    " Reverts documents that use XeTeX "
    i = find_token(document.header, '\\use_xetex', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\use_xetex.")
        return
    if get_value(document.header, "\\use_xetex", i) == 'false':
        del document.header[i]
        return
    del document.header[i]
    # 1.) set doc encoding to utf8-plain
    i = find_token(document.header, "\\inputencoding", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\inputencoding.")
    document.header[i] = "\\inputencoding utf8-plain"
    # 2.) check font settings
    l = find_token(document.header, "\\font_roman", 0)
    if l == -1:
        document.warning("Malformed LyX document: Missing \\font_roman.")
    line = document.header[l]
    l = re.compile(r'\\font_roman (.*)$')
    m = l.match(line)
    roman = m.group(1)
    l = find_token(document.header, "\\font_sans", 0)
    if l == -1:
        document.warning("Malformed LyX document: Missing \\font_sans.")
    line = document.header[l]
    l = re.compile(r'\\font_sans (.*)$')
    m = l.match(line)
    sans = m.group(1)
    l = find_token(document.header, "\\font_typewriter", 0)
    if l == -1:
        document.warning("Malformed LyX document: Missing \\font_typewriter.")
    line = document.header[l]
    l = re.compile(r'\\font_typewriter (.*)$')
    m = l.match(line)
    typewriter = m.group(1)
    osf = get_value(document.header, '\\font_osf', 0) == "true"
    sf_scale = float(get_value(document.header, '\\font_sf_scale', 0))
    tt_scale = float(get_value(document.header, '\\font_tt_scale', 0))
    # 3.) set preamble stuff
    pretext = '%% This document must be processed with xelatex!\n'
    pretext += '\\usepackage{fontspec}\n'
    if roman != "default":
        pretext += '\\setmainfont[Mapping=tex-text]{' + roman + '}\n'
    if sans != "default":
        pretext += '\\setsansfont['
        if sf_scale != 100:
            pretext += 'Scale=' + str(sf_scale / 100) + ','
        pretext += 'Mapping=tex-text]{' + sans + '}\n'
    if typewriter != "default":
        pretext += '\\setmonofont'
        if tt_scale != 100:
            pretext += '[Scale=' + str(tt_scale / 100) + ']'
        pretext += '{' + typewriter + '}\n'
    if osf:
        pretext += '\\defaultfontfeatures{Numbers=OldStyle}\n'
    pretext += '\usepackage{xunicode}\n'
    pretext += '\usepackage{xltxtra}\n'
    insert_to_preamble(0, document, pretext)
    # 4.) reset font settings
    i = find_token(document.header, "\\font_roman", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_roman.")
    document.header[i] = "\\font_roman default"
    i = find_token(document.header, "\\font_sans", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_sans.")
    document.header[i] = "\\font_sans default"
    i = find_token(document.header, "\\font_typewriter", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_typewriter.")
    document.header[i] = "\\font_typewriter default"
    i = find_token(document.header, "\\font_osf", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_osf.")
    document.header[i] = "\\font_osf false"
    i = find_token(document.header, "\\font_sc", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_sc.")
    document.header[i] = "\\font_sc false"
    i = find_token(document.header, "\\font_sf_scale", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_sf_scale.")
    document.header[i] = "\\font_sf_scale 100"
    i = find_token(document.header, "\\font_tt_scale", 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\font_tt_scale.")
    document.header[i] = "\\font_tt_scale 100"


def revert_outputformat(document):
    " Remove default output format param "
    i = find_token(document.header, '\\default_output_format', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\default_output_format.")
        return
    del document.header[i]


def hex2ratio(s):
    val = string.atoi(s, 16)
    if val != 0:
      val += 1
    return str(val / 256.0)


def revert_backgroundcolor(document):
    " Reverts background color to preamble code "
    i = find_token(document.header, "\\backgroundcolor", 0)
    if i == -1:
        return
    colorcode = get_value(document.header, '\\backgroundcolor', i)
    del document.header[i]
    # don't clutter the preamble if backgroundcolor is not set
    if colorcode == "#ffffff":
        return
    red   = hex2ratio(colorcode[1:3])
    green = hex2ratio(colorcode[3:5])
    blue  = hex2ratio(colorcode[5:7])
    insert_to_preamble(0, document,
                          '% Commands inserted by lyx2lyx to set the background color\n'
                          + '\\@ifundefined{definecolor}{\\usepackage{color}}{}\n'
                          + '\\definecolor{page_backgroundcolor}{rgb}{'
                          + red + ',' + green + ',' + blue + '}\n'
                          + '\\pagecolor{page_backgroundcolor}\n')


def revert_splitindex(document):
    " Reverts splitindex-aware documents "
    i = find_token(document.header, '\\use_indices', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\use_indices.")
        return
    indices = get_value(document.header, "\\use_indices", i)
    preamble = ""
    useindices = (indices == "true")
    if useindices:
         preamble += "\\usepackage{splitidx}\n"
    del document.header[i]
    
    # deal with index declarations in the preamble
    i = 0
    while True:
        i = find_token(document.header, "\\index", i)
        if i == -1:
            break
        k = find_token(document.header, "\\end_index", i)
        if k == -1:
            document.warning("Malformed LyX document: Missing \\end_index.")
            return
        if useindices:    
          line = document.header[i]
          l = re.compile(r'\\index (.*)$')
          m = l.match(line)
          iname = m.group(1)
          ishortcut = get_value(document.header, '\\shortcut', i, k)
          if ishortcut != "":
              preamble += "\\newindex[" + iname + "]{" + ishortcut + "}\n"
        del document.header[i:k + 1]
    if preamble != "":
        insert_to_preamble(0, document, preamble)
        
    # deal with index insets
    # these need to have the argument removed
    i = 0
    while True:
        i = find_token(document.body, "\\begin_inset Index", i)
        if i == -1:
            break
        line = document.body[i]
        l = re.compile(r'\\begin_inset Index (.*)$')
        m = l.match(line)
        itype = m.group(1)
        if itype == "idx" or indices == "false":
            document.body[i] = "\\begin_inset Index"
        else:
            k = find_end_of_inset(document.body, i)
            if k == -1:
                document.warning("Can't find end of index inset!")
                i += 1
                continue
            content = lyx2latex(document, document.body[i:k])
            # escape quotes
            content = content.replace('"', r'\"')
            subst = put_cmd_in_ert("\\sindex[" + itype + "]{" + content + "}")
            document.body[i:k + 1] = subst
        i = i + 1
        
    # deal with index_print insets
    i = 0
    while True:
        i = find_token(document.body, "\\begin_inset CommandInset index_print", i)
        if i == -1:
            return
        k = find_end_of_inset(document.body, i)
        ptype = get_value(document.body, 'type', i, k).strip('"')
        if ptype == "idx":
            j = find_token(document.body, "type", i, k)
            del document.body[j]
        elif not useindices:
            del document.body[i:k + 1]
        else:
            subst = put_cmd_in_ert("\\printindex[" + ptype + "]{}")
            document.body[i:k + 1] = subst
        i = i + 1


def convert_splitindex(document):
    " Converts index and printindex insets to splitindex-aware format "
    i = 0
    while True:
        i = find_token(document.body, "\\begin_inset Index", i)
        if i == -1:
            break
        document.body[i] = document.body[i].replace("\\begin_inset Index",
            "\\begin_inset Index idx")
        i = i + 1
    i = 0
    while True:
        i = find_token(document.body, "\\begin_inset CommandInset index_print", i)
        if i == -1:
            return
        if document.body[i + 1].find('LatexCommand printindex') == -1:
            document.warning("Malformed LyX document: Incomplete printindex inset.")
            return
        subst = ["LatexCommand printindex", 
            "type \"idx\""]
        document.body[i + 1:i + 2] = subst
        i = i + 1


def revert_subindex(document):
    " Reverts \\printsubindex CommandInset types "
    i = find_token(document.header, '\\use_indices', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\use_indices.")
        return
    indices = get_value(document.header, "\\use_indices", i)
    useindices = (indices == "true")
    i = 0
    while True:
        i = find_token(document.body, "\\begin_inset CommandInset index_print", i)
        if i == -1:
            return
        k = find_end_of_inset(document.body, i)
        ctype = get_value(document.body, 'LatexCommand', i, k)
        if ctype != "printsubindex":
            i = k + 1
            continue
        ptype = get_value(document.body, 'type', i, k).strip('"')
        if not useindices:
            del document.body[i:k + 1]
        else:
            subst = put_cmd_in_ert("\\printsubindex[" + ptype + "]{}")
            document.body[i:k + 1] = subst
        i = i + 1


def revert_printindexall(document):
    " Reverts \\print[sub]index* CommandInset types "
    i = find_token(document.header, '\\use_indices', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\use_indices.")
        return
    indices = get_value(document.header, "\\use_indices", i)
    useindices = (indices == "true")
    i = 0
    while True:
        i = find_token(document.body, "\\begin_inset CommandInset index_print", i)
        if i == -1:
            return
        k = find_end_of_inset(document.body, i)
        ctype = get_value(document.body, 'LatexCommand', i, k)
        if ctype != "printindex*" and ctype != "printsubindex*":
            i = k
            continue
        if not useindices:
            del document.body[i:k + 1]
        else:
            subst = put_cmd_in_ert("\\" + ctype + "{}")
            document.body[i:k + 1] = subst
        i = i + 1


def revert_strikeout(document):
  " Reverts \\strikeout font attribute "
  changed = revert_font_attrs(document, "\\uuline", "\\uuline")
  changed = revert_font_attrs(document, "\\uwave", "\\uwave") or changed
  changed = revert_font_attrs(document, "\\strikeout", "\\sout")  or changed
  if changed == True:
    insert_to_preamble(0, document,
        '% Commands inserted by lyx2lyx for proper underlining\n'
        + '\\PassOptionsToPackage{normalem}{ulem}\n'
        + '\\usepackage{ulem}\n')


def revert_ulinelatex(document):
    " Reverts \\uline font attribute "
    i = find_token(document.body, '\\bar under', 0)
    if i == -1:
        return
    insert_to_preamble(0, document,
            '% Commands inserted by lyx2lyx for proper underlining\n'
            + '\\PassOptionsToPackage{normalem}{ulem}\n'
            + '\\usepackage{ulem}\n'
            + '\\let\\cite@rig\\cite\n'
            + '\\newcommand{\\b@xcite}[2][\\%]{\\def\\def@pt{\\%}\\def\\pas@pt{#1}\n'
            + '  \\mbox{\\ifx\\def@pt\\pas@pt\\cite@rig{#2}\\else\\cite@rig[#1]{#2}\\fi}}\n'
            + '\\renewcommand{\\underbar}[1]{{\\let\\cite\\b@xcite\\uline{#1}}}\n')


def revert_custom_processors(document):
    " Remove bibtex_command and index_command params "
    i = find_token(document.header, '\\bibtex_command', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\bibtex_command.")
    else:
        del document.header[i]
    i = find_token(document.header, '\\index_command', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\index_command.")
    else:
        del document.header[i]


def convert_nomencl_width(document):
    " Add set_width param to nomencl_print "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset CommandInset nomencl_print", i)
      if i == -1:
        break
      document.body.insert(i + 2, "set_width \"none\"")
      i = i + 1


def revert_nomencl_width(document):
    " Remove set_width param from nomencl_print "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset CommandInset nomencl_print", i)
      if i == -1:
        break
      j = find_end_of_inset(document.body, i)
      l = find_token(document.body, "set_width", i, j)
      if l == -1:
            document.warning("Can't find set_width option for nomencl_print!")
            i = j
            continue
      del document.body[l]
      i = i + 1


def revert_nomencl_cwidth(document):
    " Remove width param from nomencl_print "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset CommandInset nomencl_print", i)
      if i == -1:
        break
      j = find_end_of_inset(document.body, i)
      l = find_token(document.body, "width", i, j)
      if l == -1:
            #Can't find width option for nomencl_print
            i = j
            continue
      width = get_value(document.body, "width", i, j).strip('"')
      del document.body[l]
      add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
      add_to_preamble(document, ["\\setlength{\\nomlabelwidth}{" + width + "}"])
      i = i + 1


def revert_applemac(document):
    " Revert applemac encoding to auto "
    i = 0
    if document.encoding == "applemac":
        document.encoding = "auto"
        i = find_token(document.header, "\\encoding", 0)
        if i != -1:
            document.header[i] = "\\encoding auto"


def revert_longtable_align(document):
    " Remove longtable alignment setting "
    i = 0
    j = 0
    while True:
      i = find_token(document.body, "\\begin_inset Tabular", i)
      if i == -1:
          break
      # the alignment is 2 lines below \\begin_inset Tabular
      j = document.body[i + 2].find("longtabularalignment")
      if j == -1:
          break
      document.body[i + 2] = document.body[i + 2][:j - 1]
      document.body[i + 2] = document.body[i + 2] + '>'
      i = i + 1


def revert_branch_filename(document):
    " Remove \\filename_suffix parameter from branches "
    i = 0
    while True:
        i = find_token(document.header, "\\filename_suffix", i)
        if i == -1:
            return
        del document.header[i]


def revert_paragraph_indentation(document):
    " Revert custom paragraph indentation to preamble code "
    i = 0
    while True:
      i = find_token(document.header, "\\paragraph_indentation", i)
      if i == -1:
          break
      # only remove the preamble line if default
      # otherwise also write the value to the preamble
      length = get_value(document.header, "\\paragraph_indentation", i)
      if length == "default":
          del document.header[i]
          break
      else:
          # handle percent lengths
          # latex_length returns "bool,length"
          length = latex_length(length).split(",")[1]
          add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
          add_to_preamble(document, ["\\setlength{\\parindent}{" + length + "}"])
          del document.header[i]
      i = i + 1


def revert_percent_skip_lengths(document):
    " Revert relative lengths for paragraph skip separation to preamble code "
    i = 0
    while True:
      i = find_token(document.header, "\\defskip", i)
      if i == -1:
          break
      length = get_value(document.header, "\\defskip", i)
      # only revert when a custom length was set and when
      # it used a percent length
      if length not in ('smallskip', 'medskip', 'bigskip'):
          # handle percent lengths
          length = latex_length(length)
          # latex_length returns "bool,length"
          percent = length.split(",")[0]
          length = length.split(",")[1]
          if percent == "True":
              add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
              add_to_preamble(document, ["\\setlength{\\parskip}{" + length + "}"])
              # set defskip to medskip as default
              document.header[i] = "\\defskip medskip"
      i = i + 1


def revert_percent_vspace_lengths(document):
    " Revert relative VSpace lengths to ERT "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset VSpace", i)
      if i == -1:
          break
      # only revert if a custom length was set and if
      # it used a percent length
      line = document.body[i]
      r = re.compile(r'\\begin_inset VSpace (.*)$')
      m = r.match(line)
      length = m.group(1)
      if length not in ('defskip', 'smallskip', 'medskip', 'bigskip', 'vfill'):
          # check if the space has a star (protected space)
          protected = (document.body[i].rfind("*") != -1)
          if protected:
              length = length.rstrip('*')
          # handle percent lengths
          length = latex_length(length)
          # latex_length returns "bool,length"
          percent = length.split(",")[0]
          length = length.split(",")[1]
          # revert the VSpace inset to ERT
          if percent == "True":
              if protected:
                  subst = [old_put_cmd_in_ert("\\vspace*{" + length + "}")]
              else:
                  subst = [old_put_cmd_in_ert("\\vspace{" + length + "}")]
              document.body[i:i + 2] = subst
      i = i + 1


def revert_percent_hspace_lengths(document):
    " Revert relative HSpace lengths to ERT "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset space \\hspace", i)
      if i == -1:
          break
      protected = (document.body[i].find("\\hspace*{}") != -1)
      # only revert if a custom length was set and if
      # it used a percent length
      length = get_value(document.body, '\\length', i + 1)
      if length == '':
          document.warning("Malformed lyx document: Missing '\\length' in Space inset.")
          return
      # handle percent lengths
      length = latex_length(length)
      # latex_length returns "bool,length"
      percent = length.split(",")[0]
      length = length.split(",")[1]
      # revert the HSpace inset to ERT
      if percent == "True":
          if protected:
              subst = [old_put_cmd_in_ert("\\hspace*{" + length + "}")]
          else:
              subst = [old_put_cmd_in_ert("\\hspace{" + length + "}")]
          document.body[i:i + 3] = subst
      i = i + 2


def revert_hspace_glue_lengths(document):
    " Revert HSpace glue lengths to ERT "
    i = 0
    while True:
      i = find_token(document.body, "\\begin_inset space \\hspace", i)
      if i == -1:
          break
      protected = (document.body[i].find("\\hspace*{}") != -1)
      length = get_value(document.body, '\\length', i + 1)
      if length == '':
          document.warning("Malformed lyx document: Missing '\\length' in Space inset.")
          return
      # only revert if the length contains a plus or minus at pos != 0
      glue  = re.compile(r'.+[\+-]')
      if glue.search(length):
          # handle percent lengths
          # latex_length returns "bool,length"
          length = latex_length(length).split(",")[1]
          # revert the HSpace inset to ERT
          if protected:
              subst = [old_put_cmd_in_ert("\\hspace*{" + length + "}")]
          else:
              subst = [old_put_cmd_in_ert("\\hspace{" + length + "}")]
          document.body[i:i + 3] = subst
      i = i + 2

def convert_author_id(document):
    " Add the author_id to the \\author definition and make sure 0 is not used"
    i = 0
    j = 1
    while True:
        i = find_token(document.header, "\\author", i)
        if i == -1:
            break
        
        r = re.compile(r'(\\author) (\".*\")\s?(.*)$')
        m = r.match(document.header[i])
        if m != None:
            name = m.group(2)
            
            email = ''
            if m.lastindex == 3:
                email = m.group(3)
            document.header[i] = "\\author %i %s %s" % (j, name, email)
        j = j + 1
        i = i + 1
        
    k = 0
    while True:
        k = find_token(document.body, "\\change_", k)
        if k == -1:
            break

        change = document.body[k].split(' ');
        if len(change) == 3:
            type = change[0]
            author_id = int(change[1])
            time = change[2]
            document.body[k] = "%s %i %s" % (type, author_id + 1, time)
        k = k + 1

def revert_author_id(document):
    " Remove the author_id from the \\author definition "
    i = 0
    j = 0
    idmap = dict()
    while True:
        i = find_token(document.header, "\\author", i)
        if i == -1:
            break
        
        r = re.compile(r'(\\author) (\d+) (\".*\")\s?(.*)$')
        m = r.match(document.header[i])
        if m != None:
            author_id = int(m.group(2))
            idmap[author_id] = j
            name = m.group(3)
            
            email = ''
            if m.lastindex == 4:
                email = m.group(4)
            document.header[i] = "\\author %s %s" % (name, email)
        i = i + 1
        j = j + 1

    k = 0
    while True:
        k = find_token(document.body, "\\change_", k)
        if k == -1:
            break

        change = document.body[k].split(' ');
        if len(change) == 3:
            type = change[0]
            author_id = int(change[1])
            time = change[2]
            document.body[k] = "%s %i %s" % (type, idmap[author_id], time)
        k = k + 1


def revert_suppress_date(document):
    " Revert suppressing of default document date to preamble code "
    i = 0
    while True:
      i = find_token(document.header, "\\suppress_date", i)
      if i == -1:
          break
      # remove the preamble line and write to the preamble
      # when suppress_date was true
      date = get_value(document.header, "\\suppress_date", i)
      if date == "true":
          add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
          add_to_preamble(document, ["\\date{}"])
      del document.header[i]
      i = i + 1


def revert_mhchem(document):
    "Revert mhchem loading to preamble code"
    i = 0
    j = 0
    k = 0
    mhchem = "off"
    i = find_token(document.header, "\\use_mhchem 1", 0)
    if i != -1:
        mhchem = "auto"
    else:
        i = find_token(document.header, "\\use_mhchem 2", 0)
        if i != -1:
            mhchem = "on"
    if mhchem == "auto":
        j = find_token(document.body, "\\cf{", 0)
        if j != -1:
            mhchem = "on"
        else:
            j = find_token(document.body, "\\ce{", 0)
            if j != -1:
                mhchem = "on"
    if mhchem == "on":
        add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
        add_to_preamble(document, ["\\PassOptionsToPackage{version=3}{mhchem}"])
        add_to_preamble(document, ["\\usepackage{mhchem}"])
    k = find_token(document.header, "\\use_mhchem", 0)
    if k == -1:
        document.warning("Malformed LyX document: Could not find mhchem setting.")
        return
    del document.header[k]


def revert_fontenc(document):
    " Remove fontencoding param "
    i = find_token(document.header, '\\fontencoding', 0)
    if i == -1:
        document.warning("Malformed LyX document: Missing \\fontencoding.")
        return
    del document.header[i]


def merge_gbrief(document):
    " Merge g-brief-en and g-brief-de to one class "

    if document.textclass != "g-brief-de":
        if document.textclass == "g-brief-en":
            document.textclass = "g-brief"
            document.set_textclass()
        return

    obsoletedby = { "Brieftext":       "Letter",
                    "Unterschrift":    "Signature",
                    "Strasse":         "Street",
                    "Zusatz":          "Addition",
                    "Ort":             "Town",
                    "Land":            "State",
                    "RetourAdresse":   "ReturnAddress",
                    "MeinZeichen":     "MyRef",
                    "IhrZeichen":      "YourRef",
                    "IhrSchreiben":    "YourMail",
                    "Telefon":         "Phone",
                    "BLZ":             "BankCode",
                    "Konto":           "BankAccount",
                    "Postvermerk":     "PostalComment",
                    "Adresse":         "Address",
                    "Datum":           "Date",
                    "Betreff":         "Reference",
                    "Anrede":          "Opening",
                    "Anlagen":         "Encl.",
                    "Verteiler":       "cc",
                    "Gruss":           "Closing"}
    i = 0
    while 1:
        i = find_token(document.body, "\\begin_layout", i)
        if i == -1:
            break

        layout = document.body[i][14:]
        if layout in obsoletedby:
            document.body[i] = "\\begin_layout " + obsoletedby[layout]

        i += 1
        
    document.textclass = "g-brief"
    document.set_textclass()


def revert_gbrief(document):
    " Revert g-brief to g-brief-en "
    if document.textclass == "g-brief":
        document.textclass = "g-brief-en"
        document.set_textclass()


def revert_html_options(document):
    " Remove html options "
    i = find_token(document.header, '\\html_use_mathml', 0)
    if i != -1:
        del document.header[i]
    i = find_token(document.header, '\\html_be_strict', 0)
    if i != -1:
        del document.header[i]


def revert_includeonly(document):
    i = 0
    while True:
        i = find_token(document.header, "\\begin_includeonly", i)
        if i == -1:
            return
        j = find_end_of(document.header, i, "\\begin_includeonly", "\\end_includeonly")
        if j == -1:
            # this should not happen
            break
        document.header[i : j + 1] = []


def revert_includeall(document):
    " Remove maintain_unincluded_children param "
    i = find_token(document.header, '\\maintain_unincluded_children', 0)
    if i != -1:
        del document.header[i]


def revert_multirow(document):
    " Revert multirow cells in tables to TeX-code"
    i = 0
    multirow = False
    while True:
      # cell type 3 is multirow begin cell
      i = find_token(document.body, '<cell multirow="3"', i)
      if i == -1:
          break
      # a multirow cell was found
      multirow = True
      # remove the multirow tag, set the valignment to top
      # and remove the bottom line
      document.body[i] = document.body[i].replace(' multirow="3" ', ' ')
      document.body[i] = document.body[i].replace('valignment="middle"', 'valignment="top"')
      document.body[i] = document.body[i].replace(' bottomline="true" ', ' ')
      # write ERT to create the multirow cell
      # use 2 rows and 2cm as default with because the multirow span
      # and the column width is only hardly accessible
      subst = [old_put_cmd_in_ert("\\multirow{2}{2cm}{")]
      document.body[i + 4:i + 4] = subst
      i = find_token(document.body, "</cell>", i)
      if i == -1:
           document.warning("Malformed LyX document: Could not find end of tabular cell.")
           break
      subst = [old_put_cmd_in_ert("}")]
      document.body[i - 3:i - 3] = subst
      # cell type 4 is multirow part cell
      i = find_token(document.body, '<cell multirow="4"', i)
      if i == -1:
          break
      # remove the multirow tag, set the valignment to top
      # and remove the top line
      document.body[i] = document.body[i].replace(' multirow="4" ', ' ')
      document.body[i] = document.body[i].replace('valignment="middle"', 'valignment="top"')
      document.body[i] = document.body[i].replace(' topline="true" ', ' ')
      i = i + 1
    if multirow == True:
        add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
        add_to_preamble(document, ["\\usepackage{multirow}"])


def convert_math_output(document):
    " Convert \html_use_mathml to \html_math_output "
    i = find_token(document.header, "\\html_use_mathml", 0)
    if i == -1:
        return
    rgx = re.compile(r'\\html_use_mathml\s+(\w+)')
    m = rgx.match(document.header[i])
    newval = "0" # MathML
    if m:
      val = m.group(1)
      if val != "true":
        newval = "2" # Images
    else:
      document.warning("Can't match " + document.header[i])
    document.header[i] = "\\html_math_output " + newval


def revert_math_output(document):
    " Revert \html_math_output to \html_use_mathml "
    i = find_token(document.header, "\\html_math_output", 0)
    if i == -1:
        return
    rgx = re.compile(r'\\html_math_output\s+(\d)')
    m = rgx.match(document.header[i])
    newval = "true"
    if m:
        val = m.group(1)
        if val == "1" or val == "2":
            newval = "false"
    else:
        document.warning("Unable to match " + document.header[i])
    document.header[i] = "\\html_use_mathml " + newval
                


def revert_inset_preview(document):
    " Dissolves the preview inset "
    i = 0
    j = 0
    k = 0
    while True:
      i = find_token(document.body, "\\begin_inset Preview", i)
      if i == -1:
          return
      j = find_end_of_inset(document.body, i)
      if j == -1:
          document.warning("Malformed LyX document: Could not find end of Preview inset.")
          return
      #If the layout is Standard we need to remove it, otherwise there
      #will be paragraph breaks that shouldn't be there.
      k = find_token(document.body, "\\begin_layout Standard", i)
      if k == i + 2:
          del document.body[i:i + 3]
          del document.body[j - 5:j - 2]
          i -= 6
      else:
          del document.body[i]
          del document.body[j - 1]
          i -= 2
                

def revert_equalspacing_xymatrix(document):
    " Revert a Formula with xymatrix@! to an ERT inset "
    i = 0
    j = 0
    has_preamble = False
    has_equal_spacing = False
    while True:
      found = -1
      i = find_token(document.body, "\\begin_inset Formula", i)
      if i == -1:
          break
      j = find_end_of_inset(document.body, i)
      if j == -1:
          document.warning("Malformed LyX document: Could not find end of Formula inset.")
          break
          
      for curline in range(i,j):
          found = document.body[curline].find("\\xymatrix@!")
          if found != -1:
              break
 
      if found != -1:
          has_equal_spacing = True
          content = [document.body[i][21:]]
          content += document.body[i + 1:j]
          subst = put_cmd_in_ert(content)
          document.body[i:j + 1] = subst
          i += len(subst)
      else:
          for curline in range(i,j):
              l = document.body[curline].find("\\xymatrix")
              if l != -1:
                  has_preamble = True;
                  break;
          i = j + 1
    if has_equal_spacing and not has_preamble:
        add_to_preamble(document, ['\\usepackage[all]{xy}'])


def revert_notefontcolor(document):
    " Reverts greyed-out note font color to preamble code "
    i = 0
    colorcode = ""
    while True:
      i = find_token(document.header, "\\notefontcolor", i)
      if i == -1:
          return
      colorcode = get_value(document.header, '\\notefontcolor', 0)
      del document.header[i]
      # the color code is in the form #rrggbb where every character denotes a hex number
      # convert the string to an int
      red = string.atoi(colorcode[1:3],16)
      # we want the output "0.5" for the value "127" therefore increment here
      if red != 0:
          red = red + 1
      redout = float(red) / 256
      green = string.atoi(colorcode[3:5],16)
      if green != 0:
          green = green + 1
      greenout = float(green) / 256
      blue = string.atoi(colorcode[5:7],16)
      if blue != 0:
          blue = blue + 1
      blueout = float(blue) / 256
      # write the preamble
      insert_to_preamble(0, document,
                           '% Commands inserted by lyx2lyx to set the font color\n'
                           '% for greyed-out notes\n'
                           + '\\@ifundefined{definecolor}{\\usepackage{color}}{}\n'
                           + '\\definecolor{note_fontcolor}{rgb}{'
                           + str(redout) + ', ' + str(greenout)
                           + ', ' + str(blueout) + '}\n'
                           + '\\renewenvironment{lyxgreyedout}\n'
                           + ' {\\textcolor{note_fontcolor}\\bgroup}{\\egroup}\n')


def revert_turkmen(document):
    "Set language Turkmen to English" 
    i = 0 
    if document.language == "turkmen": 
        document.language = "english" 
        i = find_token(document.header, "\\language", 0) 
        if i != -1: 
            document.header[i] = "\\language english" 
    j = 0 
    while True: 
        j = find_token(document.body, "\\lang turkmen", j) 
        if j == -1: 
            return 
        document.body[j] = document.body[j].replace("\\lang turkmen", "\\lang english") 
        j = j + 1 


def revert_fontcolor(document):
    " Reverts font color to preamble code "
    i = 0
    colorcode = ""
    while True:
      i = find_token(document.header, "\\fontcolor", i)
      if i == -1:
          return
      colorcode = get_value(document.header, '\\fontcolor', 0)
      del document.header[i]
      # don't clutter the preamble if backgroundcolor is not set
      if colorcode == "#000000":
          continue
      # the color code is in the form #rrggbb where every character denotes a hex number
      # convert the string to an int
      red = string.atoi(colorcode[1:3],16)
      # we want the output "0.5" for the value "127" therefore add here
      if red != 0:
          red = red + 1
      redout = float(red) / 256
      green = string.atoi(colorcode[3:5],16)
      if green != 0:
          green = green + 1
      greenout = float(green) / 256
      blue = string.atoi(colorcode[5:7],16)
      if blue != 0:
          blue = blue + 1
      blueout = float(blue) / 256
      # write the preamble
      insert_to_preamble(0, document,
                           '% Commands inserted by lyx2lyx to set the font color\n'
                           + '\\@ifundefined{definecolor}{\\usepackage{color}}{}\n'
                           + '\\definecolor{document_fontcolor}{rgb}{'
                           + str(redout) + ', ' + str(greenout)
                           + ', ' + str(blueout) + '}\n'
                           + '\\color{document_fontcolor}\n')

def revert_shadedboxcolor(document):
    " Reverts shaded box color to preamble code "
    i = 0
    colorcode = ""
    while True:
      i = find_token(document.header, "\\boxbgcolor", i)
      if i == -1:
          return
      colorcode = get_value(document.header, '\\boxbgcolor', 0)
      del document.header[i]
      # the color code is in the form #rrggbb where every character denotes a hex number
      # convert the string to an int
      red = string.atoi(colorcode[1:3],16)
      # we want the output "0.5" for the value "127" therefore increment here
      if red != 0:
          red = red + 1
      redout = float(red) / 256
      green = string.atoi(colorcode[3:5],16)
      if green != 0:
          green = green + 1
      greenout = float(green) / 256
      blue = string.atoi(colorcode[5:7],16)
      if blue != 0:
          blue = blue + 1
      blueout = float(blue) / 256
      # write the preamble
      insert_to_preamble(0, document,
                           '% Commands inserted by lyx2lyx to set the color\n'
                           '% of boxes with shaded background\n'
                           + '\\@ifundefined{definecolor}{\\usepackage{color}}{}\n'
                           + '\\definecolor{shadecolor}{rgb}{'
                           + str(redout) + ', ' + str(greenout)
                           + ', ' + str(blueout) + '}\n')


def revert_lyx_version(document):
    " Reverts LyX Version information from Inset Info "
    version = "LyX version"
    try:
        import lyx2lyx_version
        version = lyx2lyx_version.version
    except:
        pass

    i = 0
    while 1:
        i = find_token(document.body, '\\begin_inset Info', i)
        if i == -1:
            return
        j = find_end_of_inset(document.body, i + 1)
        if j == -1:
            # should not happen
            document.warning("Malformed LyX document: Could not find end of Info inset.")
        # We expect:
        # \begin_inset Info
        # type  "lyxinfo"
        # arg   "version"
        # \end_inset
        # but we shall try to be forgiving.
        arg = typ = ""
        for k in range(i, j):
            if document.body[k].startswith("arg"):
                arg = document.body[k][3:].strip().strip('"')
            if document.body[k].startswith("type"):
                typ = document.body[k][4:].strip().strip('"')
        if arg != "version" or typ != "lyxinfo":
            i = j + 1
            continue

        # We do not actually know the version of LyX used to produce the document.
        # But we can use our version, since we are reverting.
        s = [version]
        # Now we want to check if the line after "\end_inset" is empty. It normally
        # is, so we want to remove it, too.
        lastline = j + 1
        if document.body[j + 1].strip() == "":
            lastline = j + 2
        document.body[i: lastline] = s
        i = i + 1


def revert_math_scale(document):
  " Remove math scaling and LaTeX options "
  i = find_token(document.header, '\\html_math_img_scale', 0)
  if i != -1:
    del document.header[i]
  i = find_token(document.header, '\\html_latex_start', 0)
  if i != -1:
    del document.header[i]
  i = find_token(document.header, '\\html_latex_end', 0)
  if i != -1:
    del document.header[i]


def revert_pagesizes(document):
  i = 0
  " Revert page sizes to default "
  i = find_token(document.header, '\\papersize', 0)
  if i != -1:
    size = document.header[i][11:]
    if size == "a0paper" or size == "a1paper" or size == "a2paper" \
    or size == "a6paper" or size == "b0paper" or size == "b1paper" \
    or size == "b2paper" or size == "b6paper" or size == "b0j" \
    or size == "b1j" or size == "b2j" or size == "b3j" or size == "b4j" \
    or size == "b5j" or size == "b6j":
      del document.header[i]


def revert_DIN_C_pagesizes(document):
  i = 0
  " Revert DIN C page sizes to default "
  i = find_token(document.header, '\\papersize', 0)
  if i != -1:
    size = document.header[i][11:]
    if size == "c0paper" or size == "c1paper" or size == "c2paper" \
    or size == "c3paper" or size == "c4paper" or size == "c5paper" \
    or size == "c6paper":
      del document.header[i]


def convert_html_quotes(document):
  " Remove quotes around html_latex_start and html_latex_end "

  i = find_token(document.header, '\\html_latex_start', 0)
  if i != -1:
    line = document.header[i]
    l = re.compile(r'\\html_latex_start\s+"(.*)"')
    m = l.match(line)
    if m != None:
      document.header[i] = "\\html_latex_start " + m.group(1)
      
  i = find_token(document.header, '\\html_latex_end', 0)
  if i != -1:
    line = document.header[i]
    l = re.compile(r'\\html_latex_end\s+"(.*)"')
    m = l.match(line)
    if m != None:
      document.header[i] = "\\html_latex_end " + m.group(1)
      

def revert_html_quotes(document):
  " Remove quotes around html_latex_start and html_latex_end "
  
  i = find_token(document.header, '\\html_latex_start', 0)
  if i != -1:
    line = document.header[i]
    l = re.compile(r'\\html_latex_start\s+(.*)')
    m = l.match(line)
    document.header[i] = "\\html_latex_start \"" + m.group(1) + "\""
      
  i = find_token(document.header, '\\html_latex_end', 0)
  if i != -1:
    line = document.header[i]
    l = re.compile(r'\\html_latex_end\s+(.*)')
    m = l.match(line)
    document.header[i] = "\\html_latex_end \"" + m.group(1) + "\""


def revert_output_sync(document):
  " Remove forward search options "
  i = find_token(document.header, '\\output_sync_macro', 0)
  if i != -1:
    del document.header[i]
  i = find_token(document.header, '\\output_sync', 0)
  if i != -1:
    del document.header[i]


def convert_beamer_args(document):
  " Convert ERT arguments in Beamer to InsetArguments "

  if document.textclass != "beamer" and document.textclass != "article-beamer":
    return
  
  layouts = ("Block", "ExampleBlock", "AlertBlock")
  for layout in layouts:
    blay = 0
    while True:
      blay = find_token(document.body, '\\begin_layout ' + layout, blay)
      if blay == -1:
        break
      elay = find_end_of(document.body, blay, '\\begin_layout', '\\end_layout')
      if elay == -1:
        document.warning("Malformed LyX document: Can't find end of " + layout + " layout.")
        blay += 1
        continue
      bert = find_token(document.body, '\\begin_inset ERT', blay)
      if bert == -1:
        document.warning("Malformed Beamer LyX document: Can't find argument of " + layout + " layout.")
        blay = elay + 1
        continue
      eert = find_end_of_inset(document.body, bert)
      if eert == -1:
        document.warning("Malformed LyX document: Can't find end of ERT.")
        blay = elay + 1
        continue
      
      # So the ERT inset begins at line k and goes to line l. We now wrap it in 
      # an argument inset.
      # Do the end first, so as not to mess up the variables.
      document.body[eert + 1:eert + 1] = ['', '\\end_layout', '', '\\end_inset', '']
      document.body[bert:bert] = ['\\begin_inset OptArg', 'status open', '', 
          '\\begin_layout Plain Layout']
      blay = elay + 9


def revert_beamer_args(document):
  " Revert Beamer arguments to ERT "
  
  if document.textclass != "beamer" and document.textclass != "article-beamer":
    return
    
  layouts = ("Block", "ExampleBlock", "AlertBlock")
  for layout in layouts:
    blay = 0
    while True:
      blay = find_token(document.body, '\\begin_layout ' + layout, blay)
      if blay == -1:
        break
      elay = find_end_of(document.body, blay, '\\begin_layout', '\\end_layout')
      if elay == -1:
        document.warning("Malformed LyX document: Can't find end of " + layout + " layout.")
        blay += 1
        continue
      bopt = find_token(document.body, '\\begin_inset OptArg', blay)
      if bopt == -1:
        # it is legal not to have one of these
        blay = elay + 1
        continue
      eopt = find_end_of_inset(document.body, bopt)
      if eopt == -1:
        document.warning("Malformed LyX document: Can't find end of argument.")
        blay = elay + 1
        continue
      bplay = find_token(document.body, '\\begin_layout Plain Layout', blay)
      if bplay == -1:
        document.warning("Malformed LyX document: Can't find plain layout.")
        blay = elay + 1
        continue
      eplay = find_end_of(document.body, bplay, '\\begin_layout', '\\end_layout')
      if eplay == -1:
        document.warning("Malformed LyX document: Can't find end of plain layout.")
        blay = elay + 1
        continue
      # So the content of the argument inset goes from bplay + 1 to eplay - 1
      bcont = bplay + 1
      if bcont >= eplay:
        # Hmm.
        document.warning(str(bcont) + " " + str(eplay))
        blay = blay + 1
        continue
      # we convert the content of the argument into pure LaTeX...
      content = lyx2latex(document, document.body[bcont:eplay])
      strlist = put_cmd_in_ert(["{" + content + "}"])
      
      # now replace the optional argument with the ERT
      document.body[bopt:eopt + 1] = strlist
      blay = blay + 1


def revert_align_decimal(document):
  l = 0
  while True:
    l = document.body[l].find('alignment=decimal')
    if l == -1:
        break
    remove_option(document, l, 'decimal_point')
    document.body[l].replace('decimal', 'center')


def convert_optarg(document):
  " Convert \\begin_inset OptArg to \\begin_inset Argument "
  i = 0
  while 1:
    i = find_token(document.body, '\\begin_inset OptArg', i)
    if i == -1:
      return
    document.body[i] = "\\begin_inset Argument"
    i += 1


def revert_argument(document):
  " Convert \\begin_inset Argument to \\begin_inset OptArg "
  i = 0
  while 1:
    i = find_token(document.body, '\\begin_inset Argument', i)
    if i == -1:
      return
    document.body[i] = "\\begin_inset OptArg"
    i += 1


def revert_makebox(document):
  " Convert \\makebox to TeX code "
  i = 0
  while 1:
    # only revert frameless boxes without an inner box
    i = find_token(document.body, '\\begin_inset Box Frameless', i)
    if i == -1:
      # remove the option use_makebox
      revert_use_makebox(document)
      return
    z = find_end_of_inset(document.body, i)
    if z == -1:
      document.warning("Malformed LyX document: Can't find end of box inset.")
      return
    j = find_token(document.body, 'use_makebox 1', i)
    # assure we found the makebox of the current box
    if j < z and j != -1:
      y = find_token(document.body, "\\begin_layout", i)
      if y > z or y == -1:
        document.warning("Malformed LyX document: Can't find layout in box.")
        return
      # remove the \end_layout \end_inset pair
      document.body[z - 2:z + 1] = put_cmd_in_ert("}")
      # determine the alignment
      k = find_token(document.body, 'hor_pos', j - 4)
      align = document.body[k][9]
      # determine the width
      l = find_token(document.body, 'width "', j + 1)
      length = document.body[l][7:]
      # remove trailing '"'
      length = length[:-1]
      # latex_length returns "bool,length"
      length = latex_length(length).split(",")[1]
      subst = "\\makebox[" + length + "][" \
        + align + "]{"
      document.body[i:y + 1] = put_cmd_in_ert(subst)
    i += 1


def revert_use_makebox(document):
  " Deletes use_makebox option of boxes "
  h = 0
  while 1:
    # remove the option use_makebox
    h = find_token(document.body, 'use_makebox', 0)
    if h == -1:
      return
    del document.body[h]
    h += 1


def convert_use_makebox(document):
  " Adds use_makebox option for boxes "
  i = 0
  while 1:
    # remove the option use_makebox
    i = find_token(document.body, '\\begin_inset Box', i)
    if i == -1:
      return
    k = find_token(document.body, 'use_parbox', i)
    if k == -1:
      document.warning("Malformed LyX document: Can't find use_parbox statement in box.")
      return
    document.body.insert(k + 1, "use_makebox 0")
    i = k + 1


def revert_IEEEtran(document):
  " Convert IEEEtran layouts and styles to TeX code "
  if document.textclass != "IEEEtran":
    return
  revert_flex_inset(document, "IEEE membership", "\\IEEEmembership", 0)
  revert_flex_inset(document, "Lowercase", "\\MakeLowercase", 0)
  layouts = ("Special Paper Notice", "After Title Text", "Publication ID",
             "Page headings", "Biography without photo")
  latexcmd = {"Special Paper Notice": "\\IEEEspecialpapernotice",
              "After Title Text":     "\\IEEEaftertitletext",
              "Publication ID":       "\\IEEEpubid"}
  obsoletedby = {"Page headings":            "MarkBoth",
                 "Biography without photo":  "BiographyNoPhoto"}
  for layout in layouts:
    i = 0
    while True:
        i = find_token(document.body, '\\begin_layout ' + layout, i)
        if i == -1:
          break
        j = find_end_of(document.body, i, '\\begin_layout', '\\end_layout')
        if j == -1:
          document.warning("Malformed LyX document: Can't find end of " + layout + " layout.")
          i += 1
          continue
        if layout in obsoletedby:
          document.body[i] = "\\begin_layout " + obsoletedby[layout]
          i = j
        else:
          content = lyx2latex(document, document.body[i:j + 1])
          add_to_preamble(document, [latexcmd[layout] + "{" + content + "}"])
          del document.body[i:j + 1]


def convert_prettyref(document):
        " Converts prettyref references to neutral formatted refs "
        re_ref = re.compile("^\s*reference\s+\"(\w+):(\S+)\"")
        nm_ref = re.compile("^\s*name\s+\"(\w+):(\S+)\"")

        i = 0
        while True:
                i = find_token(document.body, "\\begin_inset CommandInset ref", i)
                if i == -1:
                        break
                j = find_end_of_inset(document.body, i)
                if j == -1:
                        document.warning("Malformed LyX document: No end of InsetRef!")
                        i += 1
                        continue
                k = find_token(document.body, "LatexCommand prettyref", i)
                if k != -1 and k < j:
                        document.body[k] = "LatexCommand formatted"
                i = j + 1
        document.header.insert(-1, "\\use_refstyle 0")
                
 
def revert_refstyle(document):
        " Reverts neutral formatted refs to prettyref "
        re_ref = re.compile("^reference\s+\"(\w+):(\S+)\"")
        nm_ref = re.compile("^\s*name\s+\"(\w+):(\S+)\"")

        i = 0
        while True:
                i = find_token(document.body, "\\begin_inset CommandInset ref", i)
                if i == -1:
                        break
                j = find_end_of_inset(document.body, i)
                if j == -1:
                        document.warning("Malformed LyX document: No end of InsetRef")
                        i += 1
                        continue
                k = find_token(document.body, "LatexCommand formatted", i)
                if k != -1 and k < j:
                        document.body[k] = "LatexCommand prettyref"
                i = j + 1
        i = find_token(document.header, "\\use_refstyle", 0)
        if i != -1:
                document.header.pop(i)
 

def revert_nameref(document):
  " Convert namerefs to regular references "
  cmds = ["Nameref", "nameref"]
  foundone = False
  rx = re.compile(r'reference "(.*)"')
  for cmd in cmds:
    i = 0
    oldcmd = "LatexCommand " + cmd
    while 1:
      # It seems better to look for this, as most of the reference
      # insets won't be ones we care about.
      i = find_token(document.body, oldcmd, i)
      if i == -1:
        break
      cmdloc = i
      i += 1
      # Make sure it is actually in an inset!
      # We could just check document.lines[i-1], but that relies
      # upon something that might easily change.
      # We'll look back a few lines.
      stins = cmdloc - 10
      if stins < 0:
        stins = 0
      stins = find_token(document.body, "\\begin_inset CommandInset ref", stins)
      if stins == -1 or stins > cmdloc:
        continue
      endins = find_end_of_inset(document.body, stins)
      if endins == -1:
        document.warning("Can't find end of inset at line " + stins + "!!")
        continue
      if endins < cmdloc:
        continue
      refline = find_token(document.body, "reference", stins)
      if refline == -1 or refline > endins:
        document.warning("Can't find reference for inset at line " + stinst + "!!")
        continue
      m = rx.match(document.body[refline])
      if not m:
        document.warning("Can't match reference line: " + document.body[ref])
        continue
      foundone = True
      ref = m.group(1)
      newcontent = ['\\begin_inset ERT', 'status collapsed', '', \
        '\\begin_layout Plain Layout', '', '\\backslash', \
        cmd + '{' + ref + '}', '\\end_layout', '', '\\end_inset']
      document.body[stins:endins + 1] = newcontent
  if foundone:
    add_to_preamble(document, "\usepackage{nameref}")


def remove_Nameref(document):
  " Convert Nameref commands to nameref commands "
  i = 0
  while 1:
    # It seems better to look for this, as most of the reference
    # insets won't be ones we care about.
    i = find_token(document.body, "LatexCommand Nameref" , i)
    if i == -1:
      break
    cmdloc = i
    i += 1
    
    # Make sure it is actually in an inset!
    # We could just check document.lines[i-1], but that relies
    # upon something that might easily change.
    # We'll look back a few lines.
    stins = cmdloc - 10
    if stins < 0:
      stins = 0
    stins = find_token(document.body, "\\begin_inset CommandInset ref", stins)
    if stins == -1 or stins > cmdloc:
      continue
    endins = find_end_of_inset(document.body, stins)
    if endins == -1:
      document.warning("Can't find end of inset at line " + stins + "!!")
      continue
    if endins < cmdloc:
      continue
    document.body[cmdloc] = "LatexCommand nameref"


def revert_mathrsfs(document):
    " Load mathrsfs if \mathrsfs us use in the document "
    i = 0
    end = len(document.body) - 1
    while True:
      j = document.body[i].find("\\mathscr{")
      if j != -1:
        add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
        add_to_preamble(document, ["\\usepackage{mathrsfs}"])
        break
      if i == end:
        break
      i += 1


def convert_flexnames(document):
    "Convert \\begin_inset Flex Custom:Style to \\begin_inset Flex Style and similarly for CharStyle and Element."
    
    i = 0
    rx = re.compile(r'^\\begin_inset Flex (?:Custom|CharStyle|Element):(.+)$')
    while True:
      i = find_token(document.body, "\\begin_inset Flex", i)
      if i == -1:
        return
      m = rx.match(document.body[i])
      if m:
        document.body[i] = "\\begin_inset Flex " + m.group(1)
      i += 1


flex_insets = [
  ["Alert", "CharStyle:Alert"],
  ["Code", "CharStyle:Code"],
  ["Concepts", "CharStyle:Concepts"],
  ["E-Mail", "CharStyle:E-Mail"],
  ["Emph", "CharStyle:Emph"],
  ["Expression", "CharStyle:Expression"],
  ["Initial", "CharStyle:Initial"],
  ["Institute", "CharStyle:Institute"],
  ["Meaning", "CharStyle:Meaning"],
  ["Noun", "CharStyle:Noun"],
  ["Strong", "CharStyle:Strong"],
  ["Structure", "CharStyle:Structure"],
  ["ArticleMode", "Custom:ArticleMode"],
  ["Endnote", "Custom:Endnote"],
  ["Glosse", "Custom:Glosse"],
  ["PresentationMode", "Custom:PresentationMode"],
  ["Tri-Glosse", "Custom:Tri-Glosse"]
]

flex_elements = [
  ["Abbrev", "Element:Abbrev"],
  ["CCC-Code", "Element:CCC-Code"],
  ["Citation-number", "Element:Citation-number"],
  ["City", "Element:City"],
  ["Code", "Element:Code"],
  ["CODEN", "Element:CODEN"],
  ["Country", "Element:Country"],
  ["Day", "Element:Day"],
  ["Directory", "Element:Directory"],
  ["Dscr", "Element:Dscr"],
  ["Email", "Element:Email"],
  ["Emph", "Element:Emph"],
  ["Filename", "Element:Filename"],
  ["Firstname", "Element:Firstname"],
  ["Fname", "Element:Fname"],
  ["GuiButton", "Element:GuiButton"],
  ["GuiMenu", "Element:GuiMenu"],
  ["GuiMenuItem", "Element:GuiMenuItem"],
  ["ISSN", "Element:ISSN"],
  ["Issue-day", "Element:Issue-day"],
  ["Issue-months", "Element:Issue-months"],
  ["Issue-number", "Element:Issue-number"],
  ["KeyCap", "Element:KeyCap"],
  ["KeyCombo", "Element:KeyCombo"],
  ["Keyword", "Element:Keyword"],
  ["Literal", "Element:Literal"],
  ["MenuChoice", "Element:MenuChoice"],
  ["Month", "Element:Month"],
  ["Orgdiv", "Element:Orgdiv"],
  ["Orgname", "Element:Orgname"],
  ["Postcode", "Element:Postcode"],
  ["SS-Code", "Element:SS-Code"],
  ["SS-Title", "Element:SS-Title"],
  ["State", "Element:State"],
  ["Street", "Element:Street"],
  ["Surname", "Element:Surname"],
  ["Volume", "Element:Volume"],
  ["Year", "Element:Year"]
]


def revert_flexnames(document):
  if document.backend == "latex":
    flexlist = flex_insets
  else:
    flexlist = flex_elements
  
  rx = re.compile(r'^\\begin_inset Flex\s+(.+)$')
  i = 0
  while True:
    i = find_token(document.body, "\\begin_inset Flex", i)
    if i == -1:
      return
    m = rx.match(document.body[i])
    if not m:
      document.warning("Illegal flex inset: " + document.body[i])
      i += 1
      continue
    
    style = m.group(1)
    for f in flexlist:
      if f[0] == style:
        document.body[i] = "\\begin_inset Flex " + f[1]
        break

    i += 1


def convert_mathdots(document):
    " Load mathdots automatically "
    while True:
      i = find_token(document.header, "\\use_esint" , 0)
      if i != -1:
        document.header.insert(i + 1, "\\use_mathdots 1")
      break


def revert_mathdots(document):
    " Load mathdots if used in the document "
    i = 0
    ddots = re.compile(r'\\begin_inset Formula .*\\ddots', re.DOTALL)
    vdots = re.compile(r'\\begin_inset Formula .*\\vdots', re.DOTALL)
    iddots = re.compile(r'\\begin_inset Formula .*\\iddots', re.DOTALL)
    mathdots = find_token(document.header, "\\use_mathdots" , 0)
    no = find_token(document.header, "\\use_mathdots 0" , 0)
    auto = find_token(document.header, "\\use_mathdots 1" , 0)
    yes = find_token(document.header, "\\use_mathdots 2" , 0)
    if mathdots != -1:
      del document.header[mathdots]
    while True:
      i = find_token(document.body, '\\begin_inset Formula', i)
      if i == -1:
        return
      j = find_end_of_inset(document.body, i)
      if j == -1:
        document.warning("Malformed LyX document: Can't find end of Formula inset.")
        return 
      k = ddots.search("\n".join(document.body[i:j]))
      l = vdots.search("\n".join(document.body[i:j]))
      m = iddots.search("\n".join(document.body[i:j]))
      if (yes == -1) and ((no != -1) or (not k and not l and not m) or (auto != -1 and not m)):
        i += 1
        continue
      # use \@ifundefined to catch also the "auto" case
      add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
      add_to_preamble(document, ["\\@ifundefined{iddots}{\\usepackage{mathdots}}\n"])
      return


def convert_rule(document):
    " Convert \\lyxline to CommandInset line "
    i = 0
    while True:
      i = find_token(document.body, "\\lyxline" , i)
      if i == -1:
        return
        
      j = find_token(document.body, "\\color" , i - 2)
      if j == i - 2:
        color = document.body[j] + '\n'
      else:
        color = ''
      k = find_token(document.body, "\\begin_layout Standard" , i - 4)
      # we need to handle the case that \lyxline is in a separate paragraph and that it is colored
      # the result is then an extra empty paragraph which we get by adding an empty ERT inset
      if k == i - 4 and j == i - 2 and document.body[i - 1] == '':
        layout = '\\begin_inset ERT\nstatus collapsed\n\n\\begin_layout Plain Layout\n\n\n\\end_layout\n\n\\end_inset\n' \
          + '\\end_layout\n\n' \
          + '\\begin_layout Standard\n'
      elif k == i - 2 and document.body[i - 1] == '':
        layout = ''
      else:
        layout = '\\end_layout\n\n' \
          + '\\begin_layout Standard\n'
      l = find_token(document.body, "\\begin_layout Standard" , i + 4)
      if l == i + 4 and document.body[i + 1] == '':
        layout2 = ''
      else:
        layout2 = '\\end_layout\n' \
          + '\n\\begin_layout Standard\n'
      subst = layout \
        + '\\noindent\n\n' \
        + color \
        + '\\begin_inset CommandInset line\n' \
        + 'LatexCommand rule\n' \
        + 'offset "0.5ex"\n' \
        + 'width "100line%"\n' \
        + 'height "1pt"\n' \
        + '\n\\end_inset\n\n\n' \
        + layout2
      document.body[i] = subst
      i += 1


def revert_rule(document):
    " Revert line insets to Tex code "
    i = 0
    while 1:
      i = find_token(document.body, "\\begin_inset CommandInset line" , i)
      if i == -1:
        return
      # find end of inset
      j = find_token(document.body, "\\end_inset" , i)
      # assure we found the end_inset of the current inset
      if j > i + 6 or j == -1:
        document.warning("Malformed LyX document: Can't find end of line inset.")
        return
      # determine the optional offset
      k = find_token(document.body, 'offset', i, j)
      if k != -1:
        offset = document.body[k][8:-1]
      else:
        offset = ""
      # determine the width
      l = find_token(document.body, 'width', i, j)
      if l != -1:
        width = document.body[l][7:-1]
      else:
        width = "100col%"
      # determine the height
      m = find_token(document.body, 'height', i, j)
      if m != -1:
        height = document.body[m][8:-1]
      else:
        height = "1pt"
      # output the \rule command
      if offset:
        subst = "\\rule[" + offset + "]{" + width + "}{" + height + "}"
      else:
        subst = "\\rule{" + width + "}{" + height + "}"
      document.body[i:j + 1] = put_cmd_in_ert(subst)
      i += 1


def revert_diagram(document):
  " Add the feyn package if \\Diagram is used in math "
  i = 0
  re_diagram = re.compile(r'\\begin_inset Formula .*\\Diagram', re.DOTALL)
  while True:
    i = find_token(document.body, '\\begin_inset Formula', i)
    if i == -1:
      return
    j = find_end_of_inset(document.body, i)
    if j == -1:
        document.warning("Malformed LyX document: Can't find end of Formula inset.")
        return 
    m = re_diagram.search("\n".join(document.body[i:j]))
    if not m:
      i += 1
      continue
    add_to_preamble(document, ["% this command was inserted by lyx2lyx"])
    add_to_preamble(document, "\\usepackage{feyn}")
    # only need to do it once!
    return


def convert_bibtex_clearpage(document):
  " insert a clear(double)page bibliographystyle if bibtotoc option is used "

  i = find_token(document.header, '\\papersides', 0)
  if i == -1:
    document.warning("Malformed LyX document: Can't find papersides definition.")
    return
  sides = int(document.header[i][12])

  j = 0
  while True:
    j = find_token(document.body, "\\begin_inset CommandInset bibtex", j)
    if j == -1:
      return

    k = find_end_of_inset(document.body, j)
    if k == -1:
      document.warning("Can't find end of Bibliography inset at line " + str(j))
      j += 1
      continue

    # only act if there is the option "bibtotoc"
    m = find_token(document.body, 'options', j, k)
    if m == -1:
      document.warning("Can't find options for bibliography inset at line " + str(j))
      j = k
      continue
    
    optline = document.body[m]
    idx = optline.find("bibtotoc")
    if idx == -1:
      j = k
      continue
    
    # so we want to insert a new page right before the paragraph that
    # this bibliography thing is in. we'll look for it backwards.
    lay = j - 1
    while lay >= 0:
      if document.body[lay].startswith("\\begin_layout"):
        break
      lay -= 1

    if lay < 0:
      document.warning("Can't find layout containing bibliography inset at line " + str(j))
      j = k
      continue

    subst1 = '\\begin_layout Standard\n' \
      + '\\begin_inset Newpage clearpage\n' \
      + '\\end_inset\n\n\n' \
      + '\\end_layout\n'
    subst2 = '\\begin_layout Standard\n' \
      + '\\begin_inset Newpage cleardoublepage\n' \
      + '\\end_inset\n\n\n' \
      + '\\end_layout\n'
    if sides == 1:
      document.body.insert(lay, subst1)
      document.warning(subst1)
    else:
      document.body.insert(lay, subst2)
      document.warning(subst2)

    j = k


##
# Conversion hub
#

supported_versions = ["2.0.0","2.0"]
convert = [[346, []],
           [347, []],
           [348, []],
           [349, []],
           [350, []],
           [351, []],
           [352, [convert_splitindex]],
           [353, []],
           [354, []],
           [355, []],
           [356, []],
           [357, []],
           [358, []],
           [359, [convert_nomencl_width]],
           [360, []],
           [361, []],
           [362, []],
           [363, []],
           [364, []],
           [365, []],
           [366, []],
           [367, []],
           [368, []],
           [369, [convert_author_id]],
           [370, []],
           [371, []],
           [372, []],
           [373, [merge_gbrief]],
           [374, []],
           [375, []],
           [376, []],
           [377, []],
           [378, []],
           [379, [convert_math_output]],
           [380, []],
           [381, []],
           [382, []],
           [383, []],
           [384, []],
           [385, []],
           [386, []],
           [387, []],
           [388, []],
           [389, [convert_html_quotes]],
           [390, []],
           [391, []],
           [392, []],
           [393, [convert_optarg]],
           [394, [convert_use_makebox]],
           [395, []],
           [396, []],
           [397, [remove_Nameref]],
           [398, []],
           [399, [convert_mathdots]],
           [400, [convert_rule]],
           [401, []],
           [402, [convert_bibtex_clearpage]],
           [403, [convert_flexnames]],
           [404, [convert_prettyref]]
]

revert =  [[403, [revert_refstyle]],
           [402, [revert_flexnames]],
           [401, []],
           [400, [revert_diagram]],
           [399, [revert_rule]],
           [398, [revert_mathdots]],
           [397, [revert_mathrsfs]],
           [396, []],
           [395, [revert_nameref]],
           [394, [revert_DIN_C_pagesizes]],
           [393, [revert_makebox]],
           [392, [revert_argument]],
           [391, [revert_beamer_args]],
           [390, [revert_align_decimal, revert_IEEEtran]],
           [389, [revert_output_sync]],
           [388, [revert_html_quotes]],
           [387, [revert_pagesizes]],
           [386, [revert_math_scale]],
           [385, [revert_lyx_version]],
           [384, [revert_shadedboxcolor]],
           [383, [revert_fontcolor]],
           [382, [revert_turkmen]],
           [381, [revert_notefontcolor]],
           [380, [revert_equalspacing_xymatrix]],
           [379, [revert_inset_preview]],
           [378, [revert_math_output]],
           [377, []],
           [376, [revert_multirow]],
           [375, [revert_includeall]],
           [374, [revert_includeonly]],
           [373, [revert_html_options]],
           [372, [revert_gbrief]],
           [371, [revert_fontenc]],
           [370, [revert_mhchem]],
           [369, [revert_suppress_date]],
           [368, [revert_author_id]],
           [367, [revert_hspace_glue_lengths]],
           [366, [revert_percent_vspace_lengths, revert_percent_hspace_lengths]],
           [365, [revert_percent_skip_lengths]],
           [364, [revert_paragraph_indentation]],
           [363, [revert_branch_filename]],
           [362, [revert_longtable_align]],
           [361, [revert_applemac]],
           [360, []],
           [359, [revert_nomencl_cwidth]],
           [358, [revert_nomencl_width]],
           [357, [revert_custom_processors]],
           [356, [revert_ulinelatex]],
           [355, []],
           [354, [revert_strikeout]],
           [353, [revert_printindexall]],
           [352, [revert_subindex]],
           [351, [revert_splitindex]],
           [350, [revert_backgroundcolor]],
           [349, [revert_outputformat]],
           [348, [revert_xetex]],
           [347, [revert_phantom, revert_hphantom, revert_vphantom]],
           [346, [revert_tabularvalign]],
           [345, [revert_swiss]]
          ]


if __name__ == "__main__":
    pass