X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=lib%2Fscripts%2Flyxpak.py;h=5a7d69ba5dc131b13a11b9333c3cf5daba005d3d;hb=003d675c2f3bb66f147b99def57296545d4c170e;hp=bd0427307323a19aed51789bf68e95f478c23b7e;hpb=5577b5c6f0fd44017f549689d5b56759fddb69e6;p=lyx.git diff --git a/lib/scripts/lyxpak.py b/lib/scripts/lyxpak.py index bd04273073..5a7d69ba5d 100755 --- a/lib/scripts/lyxpak.py +++ b/lib/scripts/lyxpak.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python # -*- coding: utf-8 -*- # file lyxpak.py @@ -6,37 +5,63 @@ # Licence details can be found in the file COPYING. # author Enrico Forestieri +# author Richard Heck # Full author contact details are available in file CREDITS # This script creates a tar or zip archive with a lyx file and all included -# files (graphics and so on). A zip archive is created only if tar is not -# found in the path. The tar archive is then compressed with gzip or bzip2. +# files (graphics and so on). By default, the created archive is the standard +# type on a given platform, such that a zip archive is created on Windows and +# a gzip compressed tar archive on *nix. This can be controlled by command +# line options, however. -import os, re, string, sys -if sys.version_info < (2, 4, 0): - from sets import Set as set +from __future__ import print_function +import gzip, os, re, sys +from getopt import getopt +from io import BytesIO +import subprocess -# Replace with the actual path to the 1.5.x or 1.6.x lyx2lyx. -# If left undefined and the LyX executable is in the path, the script will -# try to locate lyx2lyx by querying LyX about the system dir. -# Example for *nix: -# lyx2lyx = /usr/share/lyx/lyx2lyx/lyx2lyx -lyx2lyx = None +# Provide support for both python 2 and 3 +if sys.version_info[0] != 2: + def unicode(arg, enc): + return arg + +# The path to the current python executable. sys.executable may fail, so in +# this case we revert to simply calling "python" from the path. +PYTHON_BIN = sys.executable if sys.executable else "python" + +running_on_windows = (os.name == 'nt') + +if running_on_windows: + from shutil import copyfile + from tempfile import NamedTemporaryFile # Pre-compiled regular expressions. -re_lyxfile = re.compile("\.lyx$") -re_input = re.compile(r'^(.*)\\(input|include){(\s*)(\S+)(\s*)}.*$') -re_package = re.compile(r'^(.*)\\(usepackage){(\s*)(\S+)(\s*)}.*$') -re_class = re.compile(r'^(\\)(textclass)(\s+)(\S+)$') -re_norecur = re.compile(r'^(.*)\\(verbatiminput|lstinputlisting|includegraphics\[*.*\]*){(\s*)(\S+)(\s*)}.*$') -re_filename = re.compile(r'^(\s*)(filename)(\s+)(\S+)$') -re_options = re.compile(r'^(\s*)options(\s+)(\S+)$') -re_bibfiles = re.compile(r'^(\s*)bibfiles(\s+)(\S+)$') +re_lyxfile = re.compile(b"\.lyx$") +re_input = re.compile(b'^(.*)\\\\(input|include){(\\s*)(.+)(\\s*)}.*$') +re_ertinput = re.compile(b'^(input|include)({)(\\s*)(.+)(\\s*)}.*$') +re_package = re.compile(b'^(.*)\\\\(usepackage){(\\s*)(.+)(\\s*)}.*$') +re_class = re.compile(b'^(\\\\)(textclass)(\\s+)(.+)\\s*$') +re_norecur = re.compile(b'^(.*)\\\\(verbatiminput|lstinputlisting|includegraphics\\[*.*\\]*){(\\s*)(.+)(\\s*)}.*$') +re_ertnorecur = re.compile(b'^(verbatiminput|lstinputlisting|includegraphics\\[*.*\\]*)({)(\\s*)(.+)(\\s*)}.*$') +re_filename = re.compile(b'^(\\s*)(filename)(\\s+)(.+)\\s*$') +re_options = re.compile(b'^(\\s*)options(\\s+)(.+)\\s*$') +re_bibfiles = re.compile(b'^(\\s*)bibfiles(\\s+)(.+)\\s*$') def usage(prog_name): - return "Usage: %s file.lyx [output_dir]\n" % prog_name + msg = ''' +Usage: %s [-t] [-z] [-l path] [-o output_dir] file.lyx +Options: +-l: Path to lyx2lyx script +-o: Directory for output +-t: Create gzipped tar file +-z: Create zip file +By default, we create file.zip on Windows and file.tar.gz on *nix, +with the file output to where file.lyx is, and we look for lyx2lyx +in the known locations, querying LyX itself if necessary. +''' + return msg % prog_name def error(message): @@ -44,11 +69,17 @@ def error(message): sys.exit(1) -def run_cmd(cmd): - handle = os.popen(cmd, 'r') - cmd_stdout = handle.read() - cmd_status = handle.close() - return cmd_status, cmd_stdout +def tostr(message): + return message.decode(sys.getfilesystemencoding()) + + +def gzopen(file): + input = open(file.decode('utf-8'), 'rb') + magicnum = input.read(2) + input.close() + if magicnum == b"\x1f\x8b": + return gzip.open(file.decode('utf-8')) + return open(file.decode('utf-8'), 'rb') def find_exe(candidates, extlist, path): @@ -64,58 +95,87 @@ def find_exe(candidates, extlist, path): def abspath(name): " Resolve symlinks and returns the absolute normalized name." newname = os.path.normpath(os.path.abspath(name)) - if os.name != 'nt': + if not running_on_windows: newname = os.path.realpath(newname) return newname -def gather_files(curfile, incfiles): +def gather_files(curfile, incfiles, lyx2lyx): " Recursively gather files." curdir = os.path.dirname(abspath(curfile)) is_lyxfile = re_lyxfile.search(curfile) + if is_lyxfile: - lyx2lyx_cmd = 'python "%s" "%s"' % (lyx2lyx, curfile) - l2l_status, l2l_stdout = run_cmd(lyx2lyx_cmd) - if l2l_status != None: - error('%s failed to convert "%s"' % (lyx2lyx, curfile)) + if running_on_windows: + # subprocess cannot cope with unicode arguments and we cannot be + # sure that curfile can be correctly converted to the current + # code page. So, we resort to running lyx2lyx on a copy. + tmp = NamedTemporaryFile(delete=False) + tmp.close() + copyfile(curfile.decode('utf-8'), tmp.name) + try: + l2l_stdout = subprocess.check_output([PYTHON_BIN, lyx2lyx, tmp.name]) + except subprocess.CalledProcessError: + error('%s failed to convert "%s"' % (lyx2lyx, tostr(curfile))) + os.unlink(tmp.name) + else: + try: + l2l_stdout = subprocess.check_output([PYTHON_BIN, lyx2lyx, curfile]) + except subprocess.CalledProcessError: + error('%s failed to convert "%s"' % (lyx2lyx, tostr(curfile))) + if l2l_stdout.startswith(b"\x1f\x8b"): + l2l_stdout = gzip.GzipFile("", "rb", 0, BytesIO(l2l_stdout)).read() + elif running_on_windows: + # For some unknown reason, there can be a spurious '\r' in the line + # separators, causing spurious empty lines when calling splitlines. + l2l_stdout = l2l_stdout.replace('\r\r\n', '\r\n') lines = l2l_stdout.splitlines() else: - input = open(curfile, 'rU') + input = gzopen(curfile) lines = input.readlines() input.close() + maybe_in_ert = False i = 0 while i < len(lines): # Gather used files. recursive = True - extlist = [''] + extlist = [b''] match = re_filename.match(lines[i]) if not match: - match = re_input.match(lines[i]) + if maybe_in_ert: + match = re_ertinput.match(lines[i]) + else: + match = re_input.match(lines[i]) if not match: match = re_package.match(lines[i]) - extlist = ['.sty'] + extlist = [b'.sty'] if not match: match = re_class.match(lines[i]) - extlist = ['.cls'] + extlist = [b'.cls'] if not match: - match = re_norecur.match(lines[i]) - extlist = ['', '.eps', '.pdf', '.png', '.jpg'] + if maybe_in_ert: + match = re_ertnorecur.match(lines[i]) + else: + match = re_norecur.match(lines[i]) + extlist = [b'', b'.eps', b'.pdf', b'.png', b'.jpg'] recursive = False + maybe_in_ert = is_lyxfile and lines[i] == b"\\backslash" if match: - file = match.group(4).strip('"') + file = match.group(4).strip(b'"') if not os.path.isabs(file): file = os.path.join(curdir, file) file_exists = False - for ext in extlist: - if os.path.exists(file + ext): - file = file + ext - file_exists = True - break - if file_exists: + if not os.path.isdir(unicode(file, 'utf-8')): + for ext in extlist: + if os.path.exists(unicode(file + ext, 'utf-8')): + file = file + ext + file_exists = True + break + if file_exists and not abspath(file) in incfiles: incfiles.append(abspath(file)) if recursive: - gather_files(file, incfiles) + gather_files(file, incfiles, lyx2lyx) i += 1 continue @@ -126,10 +186,12 @@ def gather_files(curfile, incfiles): # Gather bibtex *.bst files. match = re_options.match(lines[i]) if match: - file = match.group(3).strip('"') + file = match.group(3).strip(b'"') + if file.startswith(b"bibtotoc,"): + file = file[9:] if not os.path.isabs(file): - file = os.path.join(curdir, file + '.bst') - if os.path.exists(file): + file = os.path.join(curdir, file + b'.bst') + if os.path.exists(unicode(file, 'utf-8')): incfiles.append(abspath(file)) i += 1 continue @@ -137,14 +199,14 @@ def gather_files(curfile, incfiles): # Gather bibtex *.bib files. match = re_bibfiles.match(lines[i]) if match: - bibfiles = match.group(3).strip('"').split(',') + bibfiles = match.group(3).strip(b'"').split(b',') j = 0 while j < len(bibfiles): if os.path.isabs(bibfiles[j]): - file = bibfiles[j] + file = bibfiles[j] + b'.bib' else: - file = os.path.join(curdir, bibfiles[j] + '.bib') - if os.path.exists(file): + file = os.path.join(curdir, bibfiles[j] + b'.bib') + if os.path.exists(unicode(file, 'utf-8')): incfiles.append(abspath(file)) j += 1 i += 1 @@ -155,135 +217,185 @@ def gather_files(curfile, incfiles): return 0 -def main(argv): +def find_lyx2lyx(progloc, path): + " Find a usable version of the lyx2lyx script. " + # first we will see if the script is roughly where we are + # i.e., we will assume we are in $SOMEDIR/scripts and look + # for $SOMEDIR/lyx2lyx/lyx2lyx. + ourpath = os.path.dirname(abspath(progloc)) + (upone, discard) = os.path.split(ourpath) + tryit = os.path.join(upone, "lyx2lyx", "lyx2lyx") + if os.access(tryit, os.X_OK): + return tryit + + # now we will try to query LyX itself to find the path. + extlist = [''] + if "PATHEXT" in os.environ: + extlist = extlist + os.environ["PATHEXT"].split(os.pathsep) + lyx_exe, full_path = find_exe(["lyxc", "lyx"], extlist, path) + if lyx_exe == None: + error('Cannot find the LyX executable in the path.') + try: + cmd_stdout = subprocess.check_output([lyx_exe, '-version'], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + error('Cannot query LyX about the lyx2lyx script.') + re_msvc = re.compile(r'^(\s*)(Host type:)(\s+)(win32)$') + re_sysdir = re.compile(r'^(\s*)(LyX files dir:)(\s+)(\S+)$') + lines = cmd_stdout.splitlines() + for line in lines: + match = re_msvc.match(line) + if match: + # The LyX executable was built with MSVC, so the + # "LyX files dir:" line is unusable + basedir = os.path.dirname(os.path.dirname(full_path)) + tryit = os.path.join(basedir, 'Resources', 'lyx2lyx', 'lyx2lyx') + break + match = re_sysdir.match(line) + if match: + tryit = os.path.join(match.group(4), 'lyx2lyx', 'lyx2lyx') + break + + if not os.access(tryit, os.X_OK): + error('Unable to find the lyx2lyx script.') + return tryit + + +def main(args): + + ourprog = args[0] - if len(argv) < 2 and len(argv) > 3: - error(usage(argv[0])) + try: + (options, argv) = getopt(args[1:], "htzl:o:") + except: + error(usage(ourprog)) - lyxfile = argv[1] - if not os.path.exists(lyxfile): - error('File "%s" not found.' % lyxfile) + # we expect the filename to be left + if len(argv) != 1: + error(usage(ourprog)) + makezip = running_on_windows outdir = "" - if len(argv) == 3: - outdir = argv[2] - if not os.path.isdir(outdir): - error('Error: "%s" is not a directory.' % outdir) + lyx2lyx = None + + for (opt, param) in options: + if opt == "-h": + print(usage(ourprog)) + sys.exit(0) + elif opt == "-t": + makezip = False + elif opt == "-z": + makezip = True + elif opt == "-l": + lyx2lyx = param + elif opt == "-o": + outdir = param + if not os.path.isdir(unicode(outdir, 'utf-8')): + error('Error: "%s" is not a directory.' % outdir) + + lyxfile = argv[0] + if not running_on_windows: + lyxfile = unicode(lyxfile, sys.getfilesystemencoding()).encode('utf-8') + if not os.path.exists(unicode(lyxfile, 'utf-8')): + error('File "%s" not found.' % tostr(lyxfile)) # Check that it actually is a LyX document - input = open(lyxfile, 'rU') + input = gzopen(lyxfile) line = input.readline() input.close() - if not (line and line.startswith('#LyX')): - error('File "%s" is not a LyX document.' % lyxfile) + if not (line and line.startswith(b'#LyX')): + error('File "%s" is not a LyX document.' % tostr(lyxfile)) - # Either tar or zip must be available - extlist = [''] - if os.environ.has_key("PATHEXT"): - extlist = extlist + os.environ["PATHEXT"].split(os.pathsep) - path = string.split(os.environ["PATH"], os.pathsep) - archiver, full_path = find_exe(["tar", "zip"], extlist, path) - - if archiver == "tar": - ar_cmd = "tar cf" - ar_name = re_lyxfile.sub(".tar", abspath(lyxfile)) - # Archive will be compressed if either gzip or bzip2 are available - compress, full_path = find_exe(["gzip", "bzip2"], extlist, path) - if compress == "gzip": - ext = ".gz" - elif compress == "bzip2": - ext = ".bz2" - elif archiver == "zip": - ar_cmd = "zip" - ar_name = re_lyxfile.sub(".zip", abspath(lyxfile)) - compress = None + if makezip: + import zipfile else: - error("Unable to find either tar or zip.") + import tarfile + + ar_ext = b".tar.gz" + if makezip: + ar_ext = b".zip" + ar_name = re_lyxfile.sub(ar_ext, abspath(lyxfile)).decode('utf-8') if outdir: ar_name = os.path.join(abspath(outdir), os.path.basename(ar_name)) - # Try to find the location of the lyx2lyx script - global lyx2lyx + path = os.environ["PATH"].split(os.pathsep) + if lyx2lyx == None: - # first we will see if the script is roughly where we are - # i.e., we will assume we are in $SOMEDIR/scripts and look - # for $SOMEDIR/lyx2lyx/lyx2lyx. - ourpath = os.path.dirname(abspath(argv[0])) - (upone, discard) = os.path.split(ourpath) - tryit = os.path.join(upone, "lyx2lyx", "lyx2lyx") - if os.path.exists(tryit): - lyx2lyx = tryit - else: - lyx_exe, full_path = find_exe(["lyxc", "lyx"], extlist, path) - if lyx_exe == None: - error('Cannot find the LyX executable in the path.') - cmd_status, cmd_stdout = run_cmd("%s -version 2>&1" % lyx_exe) - if cmd_status != None: - error('Cannot query LyX about the lyx2lyx script.') - re_msvc = re.compile(r'^(\s*)(Host type:)(\s+)(win32)$') - re_sysdir = re.compile(r'^(\s*)(LyX files dir:)(\s+)(\S+)$') - lines = cmd_stdout.splitlines() - for line in lines: - match = re_msvc.match(line) - if match: - # The LyX executable was built with MSVC, so the - # "LyX files dir:" line is unusable - basedir = os.path.dirname(os.path.dirname(full_path)) - lyx2lyx = os.path.join(basedir, 'Resources', 'lyx2lyx', 'lyx2lyx') - break - match = re_sysdir.match(line) - if match: - lyx2lyx = os.path.join(match.group(4), 'lyx2lyx', 'lyx2lyx') - break - if not os.access(lyx2lyx, os.X_OK): - error('Unable to find the lyx2lyx script.') + lyx2lyx = find_lyx2lyx(ourprog, path) # Initialize the list with the specified LyX file and recursively # gather all required files (also from child documents). incfiles = [abspath(lyxfile)] - gather_files(lyxfile, incfiles) + gather_files(lyxfile, incfiles, lyx2lyx) # Find the topmost dir common to all files + path_sep = os.path.sep.encode('utf-8') if len(incfiles) > 1: topdir = os.path.commonprefix(incfiles) + # As os.path.commonprefix() works on a character by character basis, + # rather than on path elements, we need to remove any trailing bytes. + topdir = topdir.rpartition(path_sep)[0] + path_sep else: - topdir = os.path.dirname(incfiles[0]) + os.path.sep + topdir = os.path.dirname(incfiles[0]) + path_sep # Remove the prefix common to all paths in the list i = 0 while i < len(incfiles): - incfiles[i] = string.replace(incfiles[i], topdir, '') + incfiles[i] = incfiles[i].replace(topdir, b'', 1) i += 1 # Remove duplicates and sort the list incfiles = list(set(incfiles)) incfiles.sort() - # Build the archive command - ar_cmd = '%s "%s"' % (ar_cmd, ar_name) - for file in incfiles: - #print file - ar_cmd = ar_cmd + ' "' + file + '"' + if topdir != '': + os.chdir(unicode(topdir, 'utf-8')) # Create the archive - if topdir != '': - os.chdir(topdir) - cmd_status, cmd_stdout = run_cmd(ar_cmd) - if cmd_status != None: + try: + if makezip: + zip = zipfile.ZipFile(ar_name, "w", zipfile.ZIP_DEFLATED) + for file in incfiles: + zip.write(file.decode('utf-8')) + zip.close() + else: + tar = tarfile.open(ar_name, "w:gz") + for file in incfiles: + tar.add(file.decode('utf-8')) + tar.close() + except: error('Failed to create LyX archive "%s"' % ar_name) - # If possible, compress the archive - if compress != None: - compress_cmd = '%s "%s"' % (compress, ar_name) - cmd_status, cmd_stdout = run_cmd(compress_cmd) - if cmd_status != None: - error('Failed to compress LyX archive "%s"' % ar_name) - ar_name = ar_name + ext - - print 'LyX archive "%s" created successfully.' % ar_name + print('LyX archive "%s" created successfully.' % ar_name) return 0 if __name__ == "__main__": + if running_on_windows: + # This works around for Python 2. + # All arguments are retrieved in unicode format and converted to utf-8. + # In this way, when launched from the command line, lyxpak.py can deal + # with any non-ascii names. Unfortunately, this is not the case when + # launched by LyX, because LyX converts the arguments of the converters + # to the filesystem encoding. On Windows this corresponds to the current + # code page and not to the UTF-16 encoding used by NTFS, such that they + # are transliterated if not exactly encodable. As an example, α may + # become a, β may become ß, and so on. However, this is a problem only + # if the full path of the LyX document contains an unencodable character + # as all other paths are extracted from the document in utf-8 format. + from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int + from ctypes.wintypes import LPWSTR, LPCWSTR + GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) + CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(("CommandLineToArgvW", windll.shell32)) + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + # unicode_argv[0] is the Python interpreter, so skip that. + argv = [argv_unicode[i].encode('utf-8') for i in xrange(1, argc.value)] + # Also skip option arguments to the Python interpreter. + while len(argv) > 0: + if not argv[0].startswith("-"): + break + argv = argv[1:] + sys.argv = argv + main(sys.argv)