Use unicode cmdline args consistently

Previously the commandline arguments were processed in an operating system
dependent encoding if running under python 2. Now they are converted to
unicode during the parsing, using the more modern argpase instead of optparse.
The individual conversion methods do no longer need to know anything about
commandline encoding. This fixes a bug similar to #10218 if running under
python 2 which I probably introduced during the python 3 conversion.

diff --git a/lib/lyx2lyx/LyX.py b/lib/lyx2lyx/LyX.py
index 6cbc6ffffad514898f8f4bba55ceb763735a5d3e..3ee9a68baf83e716678151e431d2a5176c6a8dac 100644 (file)
--- a/lib/lyx2lyx/LyX.py
+++ b/lib/lyx2lyx/LyX.py
@@ -204,10 +204,10 @@ def get_encoding(language, inputencoding, format, cjk_encoding):
 class LyX_base:
     """This class carries all the information of the LyX file."""
 
-    def __init__(self, end_format = 0, input = "", output = "", error = "",
-                 debug = default_debug__, try_hard = 0, cjk_encoding = '',
-                 final_version = "", systemlyxdir = '', language = "english",
-                 encoding = "auto"):
+    def __init__(self, end_format = 0, input = u'', output = u'', error = u'',
+                 debug = default_debug__, try_hard = 0, cjk_encoding = u'',
+                 final_version = u'', systemlyxdir = u'', language = u'english',
+                 encoding = u'auto'):
 
         """Arguments:
         end_format: final format that the file should be converted. (integer)
@@ -459,7 +459,7 @@ class LyX_base:
 
         # Since we do not know the encoding yet we need to read the input as
         # bytes in binary mode, and convert later to unicode.
-        if input and input != '-':
+        if input and input != u'-':
             self.dir = os.path.dirname(os.path.abspath(input))
             try:
                 gzip.open(input).readline()
@@ -469,7 +469,7 @@ class LyX_base:
                 self.input = open(input, 'rb')
                 self.compressed = False
         else:
-            self.dir = ''
+            self.dir = u''
             self.input = os.fdopen(sys.stdin.fileno(), 'rb')
             self.compressed = False
 
@@ -817,9 +817,9 @@ class LyX_base:
 class File(LyX_base):
     " This class reads existing LyX files."
 
-    def __init__(self, end_format = 0, input = "", output = "", error = "",
-                 debug = default_debug__, try_hard = 0, cjk_encoding = '',
-                 final_version = '', systemlyxdir = ''):
+    def __init__(self, end_format = 0, input = u'', output = u'', error = u'',
+                 debug = default_debug__, try_hard = 0, cjk_encoding = u'',
+                 final_version = u'', systemlyxdir = u''):
         LyX_base.__init__(self, end_format, input, output, error,
                           debug, try_hard, cjk_encoding, final_version,
                           systemlyxdir)

diff --git a/lib/lyx2lyx/lyx2lyx b/lib/lyx2lyx/lyx2lyx
index 04b23028b305f9672525b8f792ada64ffc498d9e..9d038305e16c9111fa3ed07b9016b23edbeaf55b 100755 (executable)
--- a/lib/lyx2lyx/lyx2lyx
+++ b/lib/lyx2lyx/lyx2lyx
@@ -19,58 +19,66 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 " Program used to convert between different versions of the lyx file format."
-import optparse
+import argparse
 import sys
 import LyX
 
+# Provide support for both python 2 and 3
+PY2 = sys.version_info[0] == 2
+if PY2:
+    # argparse returns strings in the commandline encoding, we need to convert.
+    # sys.getdefaultencoding() would not always be correct, see
+    # http://legacy.python.org/dev/peps/pep-0383/
+    def cmd_arg(arg):
+        return arg.decode(sys.getfilesystemencoding())
+else:
+    cmd_arg = str
+# End of code to support for both python 2 and 3
+
 def main():
     args = {}
-    args["usage"] = "usage: %prog [options] [file]"
-
-    args["version"] = """lyx2lyx, version %s
-Copyright (C) 2011 The LyX Team, José Matos and Dekel Tsur""" % LyX.version__
+    args["usage"] = "%(prog)s [options] [file]"
 
     args["description"] = """Convert old lyx file <file> to newer format,
     files can be compressed with gzip.  If there no file is specified then
     the standard input is assumed, in this case gziped files are not
     handled."""
 
-    parser = optparse.OptionParser(**args)
+    parser = argparse.ArgumentParser(**args)
 
     parser.set_defaults(debug=LyX.default_debug__, cjk_encoding = '')
-    parser.add_option("-d", "--debug", type="int",
+    parser.add_argument("-d", "--debug", type=int, dest="debug",
                       help="level=0..2 (O_ quiet, 10_verbose) default: 2")
-    parser.add_option("-q", "--quiet",
+    parser.add_argument("-q", "--quiet",
                       action="store_const", const=0, dest="debug")
-    parser.add_option("-v", "--verbose",
+    parser.add_argument("-v", "--verbose",
                       action="store_const", const=1, dest="debug")
-    parser.add_option("--noisy",
+    parser.add_argument("--noisy",
                       action="store_const", const=10, dest="debug")
-    parser.add_option("-c", "--encoding", dest="cjk_encoding",
+    parser.add_argument("-c", "--encoding", type=cmd_arg, dest="cjk_encoding",
                       help="files in format 413 and lower are read and"
                            " written in the format of CJK-LyX."
                            "If encoding is not given or 'auto' the encoding"
                            "is determined from the locale.")
-    parser.add_option("-e", "--err", dest="error",
+    parser.add_argument("-e", "--err", type=cmd_arg, dest="error",
                       help= "file name of the error file else goes to stderr")
-    parser.add_option("-o", "--output",
+    parser.add_argument("-o", "--output", type=cmd_arg, dest="output",
                       help= "name of the output file else goes to stdout")
-    parser.add_option("-t", "--to", dest= "end_format",
+    parser.add_argument("-t", "--to", type=cmd_arg, dest= "end_format",
                       help= "destination file format, default (latest)")
-    parser.add_option("-V", "--final_version", dest= "final_version",
+    parser.add_argument("-V", "--final_version", type=cmd_arg, dest= "final_version",
                       help= "destination version, default (latest)")
-    parser.add_option("-l", "--list", action="store_true",
+    parser.add_argument("-l", "--list", action="store_true",
                       help = "list all available formats and supported versions")
-    parser.add_option("-n", "--try-hard", action="store_true",
+    parser.add_argument("-n", "--try-hard", action="store_true",
                       help = "try hard (ignore any convertion errors)")
-    parser.add_option("-s", "--systemlyxdir", dest= "systemlyxdir",
+    parser.add_argument("-s", "--systemlyxdir", type=cmd_arg, dest= "systemlyxdir",
                       help= "LyX system directory for conversion from version 489 or older")
+    parser.add_argument('--version', action='version', version="""lyx2lyx, version %s
+Copyright (C) 2011 The LyX Team, José Matos and Dekel Tsur""" % LyX.version__)
+    parser.add_argument("input", nargs='?', type=cmd_arg, default=None)
 
-    (options, args) = parser.parse_args()
-    if args:
-        options.input = args[0]
-    else:
-        options.input = None
+    options = parser.parse_args()
 
     if options.list:
         sys.stderr.write(LyX.format_info())

diff --git a/lib/lyx2lyx/lyx_1_4.py b/lib/lyx2lyx/lyx_1_4.py
index c1a4591b54055574e14d8bd3ef72041e136a860c..29abc53bfac02a49713c70c4a9f5eb29910712f2 100644 (file)
--- a/lib/lyx2lyx/lyx_1_4.py
+++ b/lib/lyx2lyx/lyx_1_4.py
@@ -1893,8 +1893,7 @@ def convert_graphics(document):
             return
         i = i + 1
         filename = document.body[j].split()[1]
-        absname = os.path.normpath(os.path.join(document.dir, filename))
-        if document.input == stdin and not os.path.isabs(filename):
+        if document.dir == u'' and not os.path.isabs(filename):
             # We don't know the directory and cannot check the document.
             # We could use a heuristic and take the current directory,
             # and we could try to find out if documentname has an extension,
@@ -1905,6 +1904,7 @@ def convert_graphics(document):
          You may need to correct the document manually or run
          lyx2lyx again with the .lyx document as commandline argument.""" % filename)
             continue
+        absname = os.path.normpath(os.path.join(document.dir, filename))
         # This needs to be the same algorithm as in pre 233 insetgraphics
         if access(absname, F_OK):
             continue

diff --git a/lib/lyx2lyx/lyx_2_2.py b/lib/lyx2lyx/lyx_2_2.py
index 519c8b5d9c6783b952a0959ca4587ce8bfa31113..34490d8dfcc522a0580f7b50f311011febc405a9 100644 (file)
--- a/lib/lyx2lyx/lyx_2_2.py
+++ b/lib/lyx2lyx/lyx_2_2.py
@@ -39,10 +39,6 @@ from parser_tools import find_token, find_token_backwards, find_re, \
      find_end_of_inset, find_end_of_layout, find_nonempty_line, \
      get_containing_layout, get_value, check_token
 
-# Provide support for both python 2 and 3
-PY2 = sys.version_info[0] == 2
-# End of code to support for both python 2 and 3
-
 ####################################################################
 # Private helper functions
 
@@ -1135,11 +1131,11 @@ def convert_origin(document):
     if i == -1:
         document.warning("Malformed LyX document: No \\textclass!!")
         return
-    if document.dir == "":
-        origin = "stdin"
+    if document.dir == u'':
+        origin = u'stdin'
     else:
-        relpath = ''
-        if document.systemlyxdir and document.systemlyxdir != '':
+        relpath = u''
+        if document.systemlyxdir and document.systemlyxdir != u'':
             try:
                 if os.path.isabs(document.dir):
                     absdir = os.path.normpath(document.dir)
@@ -1150,16 +1146,14 @@ def convert_origin(document):
                 else:
                     abssys = os.path.normpath(os.path.abspath(document.systemlyxdir))
                 relpath = os.path.relpath(absdir, abssys)
-                if relpath.find('..') == 0:
-                    relpath = ''
+                if relpath.find(u'..') == 0:
+                    relpath = u''
             except:
-                relpath = ''
-        if relpath == '':
-            origin = document.dir.replace('\\', '/') + '/'
+                relpath = u''
+        if relpath == u'':
+            origin = document.dir.replace(u'\\', u'/') + u'/'
         else:
-            origin = os.path.join("/systemlyxdir", relpath).replace('\\', '/') + '/'
-        if os.name != 'nt' and PY2:
-            origin = unicode(origin, sys.getfilesystemencoding())
+            origin = os.path.join(u"/systemlyxdir", relpath).replace(u'\\', u'/') + u'/'
     document.header[i:i] = ["\\origin " + origin]