X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=lib%2Fscripts%2Flyxpak.py;h=2aaf2db42bc061c739fa3621c6e0d5c9c5723065;hb=0b56495e48437bec97117de8c6ac1825a8cf3fb1;hp=37829e09c2d5dfee21cf9080a8f1aea54ded64c8;hpb=5003431903c9e873ad61c80ceb60a504b8be8887;p=lyx.git diff --git a/lib/scripts/lyxpak.py b/lib/scripts/lyxpak.py index 37829e09c2..2aaf2db42b 100755 --- a/lib/scripts/lyxpak.py +++ b/lib/scripts/lyxpak.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python # -*- coding: utf-8 -*- # file lyxpak.py @@ -6,7 +5,7 @@ # Licence details can be found in the file COPYING. # author Enrico Forestieri -# author Richard Heck +# author Richard Kimberly Heck # Full author contact details are available in file CREDITS @@ -16,22 +15,38 @@ # a gzip compressed tar archive on *nix. This can be controlled by command # line options, however. -import os, re, string, sys -if sys.version_info < (2, 4, 0): - from sets import Set as set +from __future__ import print_function +import gzip, os, re, sys from getopt import getopt +from io import BytesIO +import subprocess + +# Provide support for both python 2 and 3 +if sys.version_info[0] != 2: + def unicode(arg, enc): + return arg + +# The path to the current python executable. sys.executable may fail, so in +# this case we revert to simply calling "python" from the path. +PYTHON_BIN = sys.executable if sys.executable else "python" + +running_on_windows = (os.name == 'nt') + +if running_on_windows: + from shutil import copyfile + from tempfile import NamedTemporaryFile # Pre-compiled regular expressions. -re_lyxfile = re.compile("\.lyx$") -re_input = re.compile(r'^(.*)\\(input|include){(\s*)(\S+)(\s*)}.*$') -re_ertinput = re.compile(r'^(input|include)({)(\s*)(\S+)(\s*)}.*$') -re_package = re.compile(r'^(.*)\\(usepackage){(\s*)(\S+)(\s*)}.*$') -re_class = re.compile(r'^(\\)(textclass)(\s+)(\S+)$') -re_norecur = re.compile(r'^(.*)\\(verbatiminput|lstinputlisting|includegraphics\[*.*\]*){(\s*)(\S+)(\s*)}.*$') -re_ertnorecur = re.compile(r'^(verbatiminput|lstinputlisting|includegraphics\[*.*\]*)({)(\s*)(\S+)(\s*)}.*$') -re_filename = re.compile(r'^(\s*)(filename)(\s+)(\S+)$') -re_options = re.compile(r'^(\s*)options(\s+)(\S+)$') -re_bibfiles = re.compile(r'^(\s*)bibfiles(\s+)(\S+)$') +re_lyxfile = re.compile(b"\.lyx$") +re_input = re.compile(b'^(.*)\\\\(input|include){(\\s*)(.+)(\\s*)}.*$') +re_ertinput = re.compile(b'^(input|include)({)(\\s*)(.+)(\\s*)}.*$') +re_package = re.compile(b'^(.*)\\\\(usepackage){(\\s*)(.+)(\\s*)}.*$') +re_class = re.compile(b'^(\\\\)(textclass)(\\s+)(.+)\\s*$') +re_norecur = re.compile(b'^(.*)\\\\(verbatiminput|lstinputlisting|includegraphics\\[*.*\\]*){(\\s*)(.+)(\\s*)}.*$') +re_ertnorecur = re.compile(b'^(verbatiminput|lstinputlisting|includegraphics\\[*.*\\]*)({)(\\s*)(.+)(\\s*)}.*$') +re_filename = re.compile(b'^(\\s*)(filename)(\\s+)(.+)\\s*$') +re_options = re.compile(b'^(\\s*)options(\\s+)(.+)\\s*$') +re_bibfiles = re.compile(b'^(\\s*)bibfiles(\\s+)(.+)\\s*$') def usage(prog_name): @@ -54,11 +69,17 @@ def error(message): sys.exit(1) -def run_cmd(cmd): - handle = os.popen(cmd, 'r') - cmd_stdout = handle.read() - cmd_status = handle.close() - return cmd_status, cmd_stdout +def tostr(message): + return message.decode(sys.getfilesystemencoding()) + + +def gzopen(file): + input = open(file.decode('utf-8'), 'rb') + magicnum = input.read(2) + input.close() + if magicnum == b"\x1f\x8b": + return gzip.open(file.decode('utf-8')) + return open(file.decode('utf-8'), 'rb') def find_exe(candidates, extlist, path): @@ -74,7 +95,7 @@ def find_exe(candidates, extlist, path): def abspath(name): " Resolve symlinks and returns the absolute normalized name." newname = os.path.normpath(os.path.abspath(name)) - if os.name != 'nt': + if not running_on_windows: newname = os.path.realpath(newname) return newname @@ -83,14 +104,34 @@ def gather_files(curfile, incfiles, lyx2lyx): " Recursively gather files." curdir = os.path.dirname(abspath(curfile)) is_lyxfile = re_lyxfile.search(curfile) + if is_lyxfile: - lyx2lyx_cmd = 'python "%s" "%s"' % (lyx2lyx, curfile) - l2l_status, l2l_stdout = run_cmd(lyx2lyx_cmd) - if l2l_status != None: - error('%s failed to convert "%s"' % (lyx2lyx, curfile)) + if running_on_windows: + # subprocess cannot cope with unicode arguments and we cannot be + # sure that curfile can be correctly converted to the current + # code page. So, we resort to running lyx2lyx on a copy. + tmp = NamedTemporaryFile(delete=False) + tmp.close() + copyfile(curfile.decode('utf-8'), tmp.name) + try: + l2l_stdout = subprocess.check_output([PYTHON_BIN, lyx2lyx, tmp.name]) + except subprocess.CalledProcessError: + error('%s failed to convert "%s"' % (lyx2lyx, tostr(curfile))) + os.unlink(tmp.name) + else: + try: + l2l_stdout = subprocess.check_output([PYTHON_BIN, lyx2lyx, curfile]) + except subprocess.CalledProcessError: + error('%s failed to convert "%s"' % (lyx2lyx, tostr(curfile))) + if l2l_stdout.startswith(b"\x1f\x8b"): + l2l_stdout = gzip.GzipFile("", "rb", 0, BytesIO(l2l_stdout)).read() + elif running_on_windows: + # For some unknown reason, there can be a spurious '\r' in the line + # separators, causing spurious empty lines when calling splitlines. + l2l_stdout = l2l_stdout.replace('\r\r\n', '\r\n') lines = l2l_stdout.splitlines() else: - input = open(curfile, 'rU') + input = gzopen(curfile) lines = input.readlines() input.close() @@ -99,7 +140,7 @@ def gather_files(curfile, incfiles, lyx2lyx): while i < len(lines): # Gather used files. recursive = True - extlist = [''] + extlist = [b''] match = re_filename.match(lines[i]) if not match: if maybe_in_ert: @@ -108,26 +149,26 @@ def gather_files(curfile, incfiles, lyx2lyx): match = re_input.match(lines[i]) if not match: match = re_package.match(lines[i]) - extlist = ['.sty'] + extlist = [b'.sty'] if not match: match = re_class.match(lines[i]) - extlist = ['.cls'] + extlist = [b'.cls'] if not match: if maybe_in_ert: match = re_ertnorecur.match(lines[i]) else: match = re_norecur.match(lines[i]) - extlist = ['', '.eps', '.pdf', '.png', '.jpg'] + extlist = [b'', b'.eps', b'.pdf', b'.png', b'.jpg'] recursive = False - maybe_in_ert = is_lyxfile and lines[i] == "\\backslash" + maybe_in_ert = is_lyxfile and lines[i] == b"\\backslash" if match: - file = match.group(4).strip('"') + file = match.group(4).strip(b'"') if not os.path.isabs(file): file = os.path.join(curdir, file) file_exists = False - if not os.path.isdir(file): + if not os.path.isdir(unicode(file, 'utf-8')): for ext in extlist: - if os.path.exists(file + ext): + if os.path.exists(unicode(file + ext, 'utf-8')): file = file + ext file_exists = True break @@ -145,10 +186,12 @@ def gather_files(curfile, incfiles, lyx2lyx): # Gather bibtex *.bst files. match = re_options.match(lines[i]) if match: - file = match.group(3).strip('"') + file = match.group(3).strip(b'"') + if file.startswith(b"bibtotoc,"): + file = file[9:] if not os.path.isabs(file): - file = os.path.join(curdir, file + '.bst') - if os.path.exists(file): + file = os.path.join(curdir, file + b'.bst') + if os.path.exists(unicode(file, 'utf-8')): incfiles.append(abspath(file)) i += 1 continue @@ -156,14 +199,14 @@ def gather_files(curfile, incfiles, lyx2lyx): # Gather bibtex *.bib files. match = re_bibfiles.match(lines[i]) if match: - bibfiles = match.group(3).strip('"').split(',') + bibfiles = match.group(3).strip(b'"').split(b',') j = 0 while j < len(bibfiles): if os.path.isabs(bibfiles[j]): - file = bibfiles[j] + file = bibfiles[j] + b'.bib' else: - file = os.path.join(curdir, bibfiles[j] + '.bib') - if os.path.exists(file): + file = os.path.join(curdir, bibfiles[j] + b'.bib') + if os.path.exists(unicode(file, 'utf-8')): incfiles.append(abspath(file)) j += 1 i += 1 @@ -174,7 +217,7 @@ def gather_files(curfile, incfiles, lyx2lyx): return 0 -def find_lyx2lyx(progloc): +def find_lyx2lyx(progloc, path): " Find a usable version of the lyx2lyx script. " # first we will see if the script is roughly where we are # i.e., we will assume we are in $SOMEDIR/scripts and look @@ -192,8 +235,9 @@ def find_lyx2lyx(progloc): lyx_exe, full_path = find_exe(["lyxc", "lyx"], extlist, path) if lyx_exe == None: error('Cannot find the LyX executable in the path.') - cmd_status, cmd_stdout = run_cmd("%s -version 2>&1" % lyx_exe) - if cmd_status != None: + try: + cmd_stdout = subprocess.check_output([lyx_exe, '-version'], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: error('Cannot query LyX about the lyx2lyx script.') re_msvc = re.compile(r'^(\s*)(Host type:)(\s+)(win32)$') re_sysdir = re.compile(r'^(\s*)(LyX files dir:)(\s+)(\S+)$') @@ -229,13 +273,13 @@ def main(args): if len(argv) != 1: error(usage(ourprog)) - makezip = (os.name == 'nt') + makezip = running_on_windows outdir = "" lyx2lyx = None for (opt, param) in options: if opt == "-h": - print usage(ourprog) + print(usage(ourprog)) sys.exit(0) elif opt == "-t": makezip = False @@ -245,37 +289,39 @@ def main(args): lyx2lyx = param elif opt == "-o": outdir = param - if not os.path.isdir(outdir): + if not os.path.isdir(unicode(outdir, 'utf-8')): error('Error: "%s" is not a directory.' % outdir) lyxfile = argv[0] - if not os.path.exists(lyxfile): - error('File "%s" not found.' % lyxfile) + if not running_on_windows: + lyxfile = unicode(lyxfile, sys.getfilesystemencoding()).encode('utf-8') + if not os.path.exists(unicode(lyxfile, 'utf-8')): + error('File "%s" not found.' % tostr(lyxfile)) # Check that it actually is a LyX document - input = open(lyxfile, 'rU') + input = gzopen(lyxfile) line = input.readline() input.close() - if not (line and line.startswith('#LyX')): - error('File "%s" is not a LyX document.' % lyxfile) + if not (line and line.startswith(b'#LyX')): + error('File "%s" is not a LyX document.' % tostr(lyxfile)) if makezip: import zipfile else: import tarfile - ar_ext = ".tar.gz" + ar_ext = b".tar.gz" if makezip: - ar_ext = ".zip" + ar_ext = b".zip" - ar_name = re_lyxfile.sub(ar_ext, abspath(lyxfile)) + ar_name = re_lyxfile.sub(ar_ext, abspath(lyxfile)).decode('utf-8') if outdir: ar_name = os.path.join(abspath(outdir), os.path.basename(ar_name)) - path = string.split(os.environ["PATH"], os.pathsep) + path = os.environ["PATH"].split(os.pathsep) if lyx2lyx == None: - lyx2lyx = find_lyx2lyx(ourprog) + lyx2lyx = find_lyx2lyx(ourprog, path) # Initialize the list with the specified LyX file and recursively # gather all required files (also from child documents). @@ -283,19 +329,19 @@ def main(args): gather_files(lyxfile, incfiles, lyx2lyx) # Find the topmost dir common to all files + path_sep = os.path.sep.encode('utf-8') if len(incfiles) > 1: topdir = os.path.commonprefix(incfiles) - # Check whether topdir is valid, as os.path.commonprefix() works on - # a character by character basis, rather than on path elements. - if not os.path.exists(topdir): - topdir = os.path.dirname(topdir) + os.path.sep + # As os.path.commonprefix() works on a character by character basis, + # rather than on path elements, we need to remove any trailing bytes. + topdir = topdir.rpartition(path_sep)[0] + path_sep else: - topdir = os.path.dirname(incfiles[0]) + os.path.sep + topdir = os.path.dirname(incfiles[0]) + path_sep # Remove the prefix common to all paths in the list i = 0 while i < len(incfiles): - incfiles[i] = string.replace(incfiles[i], topdir, '', 1) + incfiles[i] = incfiles[i].replace(topdir, b'', 1) i += 1 # Remove duplicates and sort the list @@ -303,26 +349,53 @@ def main(args): incfiles.sort() if topdir != '': - os.chdir(topdir) + os.chdir(unicode(topdir, 'utf-8')) # Create the archive try: if makezip: zip = zipfile.ZipFile(ar_name, "w", zipfile.ZIP_DEFLATED) for file in incfiles: - zip.write(file) + zip.write(file.decode('utf-8')) zip.close() else: tar = tarfile.open(ar_name, "w:gz") for file in incfiles: - tar.add(file) + tar.add(file.decode('utf-8')) tar.close() except: error('Failed to create LyX archive "%s"' % ar_name) - print 'LyX archive "%s" created successfully.' % ar_name + print('LyX archive "%s" created successfully.' % ar_name) return 0 if __name__ == "__main__": + if running_on_windows: + # This works around for Python 2. + # All arguments are retrieved in unicode format and converted to utf-8. + # In this way, when launched from the command line, lyxpak.py can deal + # with any non-ascii names. Unfortunately, this is not the case when + # launched by LyX, because LyX converts the arguments of the converters + # to the filesystem encoding. On Windows this corresponds to the current + # code page and not to the UTF-16 encoding used by NTFS, such that they + # are transliterated if not exactly encodable. As an example, α may + # become a, β may become ß, and so on. However, this is a problem only + # if the full path of the LyX document contains an unencodable character + # as all other paths are extracted from the document in utf-8 format. + from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int + from ctypes.wintypes import LPWSTR, LPCWSTR + GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) + CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(("CommandLineToArgvW", windll.shell32)) + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + # unicode_argv[0] is the Python interpreter, so skip that. + argv = [argv_unicode[i].encode('utf-8') for i in xrange(1, argc.value)] + # Also skip option arguments to the Python interpreter. + while len(argv) > 0: + if not argv[0].startswith("-"): + break + argv = argv[1:] + sys.argv = argv + main(sys.argv)