X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=po%2Flyx_pot.py;h=d16a1393821cc2c27d4027f8aa0f20f07aa24c56;hb=6012beb90eb88011d1213c9ae38c4a77d711737e;hp=25cb1fd649ba84461aac679cb6a249092954305c;hpb=8995a8387a03361b6a7ff44b447a90a6187b1a14;p=lyx.git diff --git a/po/lyx_pot.py b/po/lyx_pot.py index 25cb1fd649..d16a139382 100755 --- a/po/lyx_pot.py +++ b/po/lyx_pot.py @@ -16,9 +16,10 @@ # This script will extract translatable strings from input files and write # to output in gettext .pot format. # +from __future__ import print_function + import sys, os, re, getopt -if sys.version_info < (2, 4, 0): - from sets import Set as set +import io def relativePath(path, base): '''return relative path from top source dir''' @@ -26,7 +27,7 @@ def relativePath(path, base): path1 = os.path.normpath(os.path.realpath(path)).split(os.sep) path2 = os.path.normpath(os.path.realpath(base)).split(os.sep) if path1[:len(path2)] != path2: - print "Path %s is not under top source directory" % path + print("Path %s is not under top source directory" % path) path3 = os.path.join(*path1[len(path2):]); # replace all \ by / such that we get the same comments on Windows and *nix path3 = path3.replace('\\', '/') @@ -37,13 +38,13 @@ def writeString(outfile, infile, basefile, lineno, string): string = string.replace('\\', '\\\\').replace('"', '') if string == "": return - print >> outfile, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(infile, basefile), lineno, string) + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(infile, basefile), lineno, string), file=outfile) def ui_l10n(input_files, output, base): '''Generate pot file from lib/ui/*''' - output = open(output, 'w') + output = io.open(output, 'w', encoding='utf_8', newline='\n') Submenu = re.compile(r'^[^#]*Submenu\s+"([^"]*)"', re.IGNORECASE) Popupmenu = re.compile(r'^[^#]*PopupMenu\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE) IconPalette = re.compile(r'^[^#]*IconPalette\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE) @@ -51,7 +52,7 @@ def ui_l10n(input_files, output, base): Item = re.compile(r'[^#]*Item\s+"([^"]*)"', re.IGNORECASE) TableInsert = re.compile(r'[^#]*TableInsert\s+"([^"]*)"', re.IGNORECASE) for src in input_files: - input = open(src) + input = io.open(src, encoding='utf_8') for lineno, line in enumerate(input.readlines()): if Submenu.match(line): (string,) = Submenu.match(line).groups() @@ -70,38 +71,46 @@ def ui_l10n(input_files, output, base): continue string = string.replace('"', '') if string != "": - print >> output, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(src, base), lineno+1, string) + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(src, base), lineno+1, string), file=output) input.close() output.close() def layouts_l10n(input_files, output, base, layouttranslations): - '''Generate pot file from lib/layouts/*.{layout,inc,module}''' - Style = re.compile(r'^\s*Style\s+(.*)\s*$', re.IGNORECASE) + '''Generate pot file from lib/layouts/*.{layout,inc,module} and lib/citeengines/*.citeengine''' + ClassDescription = re.compile(r'^\s*#\s*\\Declare(LaTeX|DocBook)Class.*\{(.*)\}$', re.IGNORECASE) + ClassCategory = re.compile(r'^\s*#\s*\\DeclareCategory\{(.*)\}$', re.IGNORECASE) + Style = re.compile(r'^\s*Style\s+(.*\S)\s*$', re.IGNORECASE) # match LabelString, EndLabelString, LabelStringAppendix and maybe others but no comments - LabelString = re.compile(r'^[^#]*LabelString\S*\s+(.*)\s*$', re.IGNORECASE) - GuiName = re.compile(r'^\s*GuiName\s+(.*)\s*$', re.IGNORECASE) - ListName = re.compile(r'^\s*ListName\s+(.*)\s*$', re.IGNORECASE) - CategoryName = re.compile(r'^\s*Category\s+(.*)\s*$', re.IGNORECASE) + LabelString = re.compile(r'^[^#]*LabelString\S*\s+(.*\S)\s*$', re.IGNORECASE) + MenuString = re.compile(r'^[^#]*MenuString\S*\s+(.*\S)\s*$', re.IGNORECASE) + OutlinerName = re.compile(r'^[^#]*OutlinerName\s+(\S+|\"[^\"]*\")\s+\"([^\"]*)\"', re.IGNORECASE) + Tooltip = re.compile(r'^\s*Tooltip\S*\s+(.*\S)\s*$', re.IGNORECASE) + GuiName = re.compile(r'^\s*GuiName\s+(.*\S)\s*$', re.IGNORECASE) + ListName = re.compile(r'^\s*ListName\s+(.*\S)\s*$', re.IGNORECASE) + CategoryName = re.compile(r'^\s*Category\s+(.*\S)\s*$', re.IGNORECASE) NameRE = re.compile(r'^\s*#\s*\\DeclareLyXModule.*{(.*)}$', re.IGNORECASE) + CiteNameRE = re.compile(r'^\s*#\s*\\DeclareLyXCiteEngine.*\{(.*)\}$', re.IGNORECASE) InsetLayout = re.compile(r'^InsetLayout\s+\"?(.*)\"?\s*$', re.IGNORECASE) FlexCheck = re.compile(r'^Flex:(.*)', re.IGNORECASE) - DescBegin = re.compile(r'^\s*#DescriptionBegin\s*$', re.IGNORECASE) + CaptionCheck = re.compile(r'^Caption:(.*)', re.IGNORECASE) + DescBegin = re.compile(r'^\s*#\s*DescriptionBegin\s*$', re.IGNORECASE) DescEnd = re.compile(r'^\s*#\s*DescriptionEnd\s*$', re.IGNORECASE) - Category = re.compile(r'^\s*#\s*Category:\s+(.*)\s*$', re.IGNORECASE) + Category = re.compile(r'^\s*#\s*Category:\s+(.*\S)\s*$', re.IGNORECASE) I18nPreamble = re.compile(r'^\s*((Lang)|(Babel))Preamble\s*$', re.IGNORECASE) EndI18nPreamble = re.compile(r'^\s*End((Lang)|(Babel))Preamble\s*$', re.IGNORECASE) I18nString = re.compile(r'_\(([^\)]+)\)') CounterFormat = re.compile(r'^\s*PrettyFormat\s+"?(.*)"?\s*$', re.IGNORECASE) CiteFormat = re.compile(r'^\s*CiteFormat', re.IGNORECASE) - KeyVal = re.compile(r'^\s*_\w+\s+(.*)\s*$') + # Note: preceding and trailing space in the val below matters + KeyVal = re.compile(r'^\s*_\w+\s(.*\S)*$') Float = re.compile(r'^\s*Float\s*$', re.IGNORECASE) - UsesFloatPkg = re.compile(r'^\s*UsesFloatPkg\s+(.*)\s*$', re.IGNORECASE) - IsPredefined = re.compile(r'^\s*IsPredefined\s+(.*)\s*$', re.IGNORECASE) + UsesFloatPkg = re.compile(r'^\s*UsesFloatPkg\s+(.*\S)\s*$', re.IGNORECASE) + IsPredefined = re.compile(r'^\s*IsPredefined\s+(.*\S)\s*$', re.IGNORECASE) End = re.compile(r'^\s*End', re.IGNORECASE) Comment = re.compile(r'^(.*)#') - Translation = re.compile(r'^\s*Translation\s+(.*)\s*$', re.IGNORECASE) + Translation = re.compile(r'^\s*Translation\s+(.*\S)\s*$', re.IGNORECASE) KeyValPair = re.compile(r'\s*"(.*)"\s+"(.*)"') oldlanguages = [] @@ -119,7 +128,7 @@ def layouts_l10n(input_files, output, base, layouttranslations): # read old translations if available try: - input = open(output) + input = io.open(output, encoding='utf_8') lang = '' for line in input.readlines(): res = Comment.search(line) @@ -141,25 +150,28 @@ def layouts_l10n(input_files, output, base, layouttranslations): continue res = KeyValPair.search(line) if res and lang != '': - key = res.group(1).decode('utf-8') - val = res.group(2).decode('utf-8') + key = res.group(1) + val = res.group(2) key = key.replace('\\"', '"').replace('\\\\', '\\') val = val.replace('\\"', '"').replace('\\\\', '\\') oldtrans[lang][key] = val keyset.add(key) continue - print "Error: Unable to handle line:" - print line + print("Error: Unable to handle line:") + print(line) except IOError: - print "Warning: Unable to open %s for reading." % output - print " Old translations will be lost." + print("Warning: Unable to open %s for reading." % output) + print(" Old translations will be lost.") # walon is not a known document language # FIXME: Do not hardcode, read from lib/languages! if 'wa' in languages: languages.remove('wa') - out = open(output, 'w') + if layouttranslations: + out = io.open(output, 'w', encoding='utf_8') + else: + out = io.open(output, 'w', encoding='utf_8', newline='\n') for src in input_files: readingDescription = False readingI18nPreamble = False @@ -172,8 +184,20 @@ def layouts_l10n(input_files, output, base, layouttranslations): descStartLine = -1 descLines = [] lineno = 0 - for line in open(src).readlines(): + for line in io.open(src, encoding='utf_8').readlines(): lineno += 1 + res = ClassDescription.search(line) + if res != None: + string = res.group(2) + if not layouttranslations: + writeString(out, src, base, lineno + 1, string) + continue + res = ClassCategory.search(line) + if res != None: + string = res.group(1) + if not layouttranslations: + writeString(out, src, base, lineno + 1, string) + continue if readingDescription: res = DescEnd.search(line) if res != None: @@ -207,6 +231,12 @@ def layouts_l10n(input_files, output, base, layouttranslations): readingI18nPreamble = True continue res = NameRE.search(line) + if res != None: + string = res.group(1) + if not layouttranslations: + writeString(out, src, base, lineno + 1, string) + continue + res = CiteNameRE.search(line) if res != None: string = res.group(1) if not layouttranslations: @@ -222,6 +252,24 @@ def layouts_l10n(input_files, output, base, layouttranslations): writeString(out, src, base, lineno, string) continue res = LabelString.search(line) + if res != None: + string = res.group(1) + if not layouttranslations: + writeString(out, src, base, lineno, string) + continue + res = MenuString.search(line) + if res != None: + string = res.group(1) + if not layouttranslations: + writeString(out, src, base, lineno, string) + continue + res = OutlinerName.search(line) + if res != None: + string = res.group(2) + if not layouttranslations: + writeString(out, src, base, lineno, string) + continue + res = Tooltip.search(line) if res != None: string = res.group(1) if not layouttranslations: @@ -259,6 +307,10 @@ def layouts_l10n(input_files, output, base, layouttranslations): #if not layouttranslations: # writeString(out, src, base, lineno, string) m = FlexCheck.search(string) + if m: + if not layouttranslations: + writeString(out, src, base, lineno, m.group(1)) + m = CaptionCheck.search(string) if m: if not layouttranslations: writeString(out, src, base, lineno, m.group(1)) @@ -301,11 +353,14 @@ def layouts_l10n(input_files, output, base, layouttranslations): continue res = End.search(line) if res != None: - # If a float is predefined by the package and it does not need - # the float package then it uses the standard babel translations. - # This is even true for MarginFigure, MarginTable (both from + # We have four combinations of the flags usesFloatPkg and isPredefined: + # usesFloatPkg and isPredefined: might use standard babel translations + # usesFloatPkg and not isPredefined: does not use standard babel translations + # not usesFloatPkg and isPredefined: uses standard babel translations + # not usesFloatPkg and not isPredefined: not supported by LyX + # The third combination is even true for MarginFigure, MarginTable (both from # tufte-book.layout) and Planotable, Plate (both from aguplus.inc). - if layouttranslations and readingFloat and usesFloatPkg and not isPredefined: + if layouttranslations and readingFloat and usesFloatPkg: if floatname != '': keyset.add(floatname) if listname != '': @@ -338,16 +393,16 @@ def layouts_l10n(input_files, output, base, layouttranslations): ContextRe = re.compile(r'(.*)(\[\[.*\]\])') - print >> out, '''# This file has been automatically generated by po/lyx_pot.py. + print(u'''# This file has been automatically generated by po/lyx_pot.py. # PLEASE MODIFY ONLY THE LAGUAGES HAVING NO .po FILE! If you want to regenerate # this file from the translations, run `make ../lib/layouttranslations' in po. # Python polib library is needed for building the output file. # # This file should remain fixed during minor LyX releases. -# For more comments see README.localization file.''' +# For more comments see README.localization file.''', file=out) for lang in languages: - print >> out, '\nTranslation %s' % lang - if lang in oldtrans.keys(): + print(u'\nTranslation %s' % lang, file=out) + if lang in list(oldtrans.keys()): trans = oldtrans[lang] else: trans = dict() @@ -366,11 +421,13 @@ def layouts_l10n(input_files, output, base, layouttranslations): if val != key: trans[key] = val for key in keys: - if key in trans.keys(): + if key in list(trans.keys()): val = trans[key].replace('\\', '\\\\').replace('"', '\\"') + res = ContextRe.search(val) + if res != None: + val = res.group(1) key = key.replace('\\', '\\\\').replace('"', '\\"') - print >> out, '\t"%s" "%s"' % \ - (key.encode('utf-8'), val.encode('utf-8')) + print(u'\t"%s" "%s"' % (key, val), file=out) # also print untranslated entries to help translators elif not lang in oldlanguages: key = key.replace('\\', '\\\\').replace('"', '\\"') @@ -379,20 +436,19 @@ def layouts_l10n(input_files, output, base, layouttranslations): val = res.group(1) else: val = key - print >> out, '\t"%s" "%s"' % \ - (key.encode('utf-8'), val.encode('utf-8')) - print >> out, 'End' + print(u'\t"%s" "%s"' % (key, val), file=out) + print(u'End', file=out) out.close() def qt4_l10n(input_files, output, base): '''Generate pot file from src/frontends/qt4/ui/*.ui''' - output = open(output, 'w') + output = io.open(output, 'w', encoding='utf_8', newline='\n') pat = re.compile(r'\s*(.*)') prop = re.compile(r'\s* @@ -409,35 +465,55 @@ def qt4_l10n(input_files, output, base): string = string.replace('<', '<').replace('>', '>') string = string.replace('\\', '\\\\').replace('"', r'\"') string = string.replace(' ', r'\n') - print >> output, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(src, base), lineno+1, string) + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(src, base), lineno+1, string), file=output) input.close() output.close() def languages_l10n(input_files, output, base): '''Generate pot file from lib/languages''' - out = open(output, 'w') + out = io.open(output, 'w', encoding='utf_8', newline='\n') + GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE) + + for src in input_files: + descStartLine = -1 + descLines = [] + lineno = 0 + for line in io.open(src, encoding='utf_8').readlines(): + lineno += 1 + res = GuiName.search(line) + if res != None: + string = res.group(1) + writeString(out, src, base, lineno, string) + continue + + out.close() + + +def latexfonts_l10n(input_files, output, base): + '''Generate pot file from lib/latexfonts''' + out = io.open(output, 'w', encoding='utf_8', newline='\n') GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE) - + for src in input_files: descStartLine = -1 descLines = [] lineno = 0 - for line in open(src).readlines(): + for line in io.open(src, encoding='utf_8').readlines(): lineno += 1 res = GuiName.search(line) if res != None: string = res.group(1) writeString(out, src, base, lineno, string) continue - + out.close() def external_l10n(input_files, output, base): - '''Generate pot file from lib/external_templates''' - output = open(output, 'w') + '''Generate pot file from lib/xtemplates''' + output = io.open(output, 'w', encoding='utf_8', newline='\n') Template = re.compile(r'^Template\s+(.*)', re.IGNORECASE) GuiName = re.compile(r'\s*GuiName\s+(.*)', re.IGNORECASE) HelpTextStart = re.compile(r'\s*HelpText\s', re.IGNORECASE) @@ -445,7 +521,7 @@ def external_l10n(input_files, output, base): HelpTextEnd = re.compile(r'\s*HelpTextEnd\s', re.IGNORECASE) i = -1 for src in input_files: - input = open(src) + input = io.open(src, encoding='utf_8') inHelp = False hadHelp = False prev_help_string = '' @@ -457,20 +533,26 @@ def external_l10n(input_files, output, base): elif inHelp: if HelpTextEnd.match(line): if hadHelp: - print >> output, '\nmsgstr ""\n' + print(u'\nmsgstr ""\n', file=output) inHelp = False hadHelp = False prev_help_string = '' elif HelpTextSection.match(line): (help_string,) = HelpTextSection.match(line).groups() help_string = help_string.replace('"', '') + help_string = help_string.replace('\\', '_backsl_') + help_string = help_string.replace('_backsl_', '\\\\') if help_string != "" and prev_help_string == '': - print >> output, '#: %s:%d\nmsgid ""\n"%s\\n"' % \ - (relativePath(src, base), lineno+1, help_string) + print(u'#: %s:%d\nmsgid ""\n"%s\\n"' % \ + (relativePath(src, base), lineno+1, help_string), file=output) hadHelp = True elif help_string != "": - print >> output, '"%s\\n"' % help_string + print(u'"%s\\n"' % help_string, file=output) prev_help_string = help_string + else: + # Empty line + print(u'"\\n"', file=output) + prev_help_string = 'xxxx' elif HelpTextStart.match(line): inHelp = True prev_help_string = '' @@ -478,18 +560,18 @@ def external_l10n(input_files, output, base): continue string = string.replace('"', '') if string != "" and not inHelp: - print >> output, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(src, base), lineno+1, string) + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(src, base), lineno+1, string), file=output) input.close() output.close() def formats_l10n(input_files, output, base): '''Generate pot file from configure.py''' - output = open(output, 'w') + output = io.open(output, 'w', encoding='utf_8', newline='\n') GuiName = re.compile(r'.*\\Format\s+\S+\s+\S+\s+"([^"]*)"\s+(\S*)\s+.*', re.IGNORECASE) GuiName2 = re.compile(r'.*\\Format\s+\S+\s+\S+\s+([^"]\S+)\s+(\S*)\s+.*', re.IGNORECASE) - input = open(input_files[0]) + input = io.open(input_files[0], encoding='utf_8') for lineno, line in enumerate(input.readlines()): label = "" labelsc = "" @@ -505,31 +587,33 @@ def formats_l10n(input_files, output, base): if shortcut != "": labelsc = label + "|" + shortcut if label != "": - print >> output, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(input_files[0], base), lineno+1, label) + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(input_files[0], base), lineno+1, label), file=output) if labelsc != "": - print >> output, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(input_files[0], base), lineno+1, labelsc) + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(input_files[0], base), lineno+1, labelsc), file=output) input.close() output.close() def encodings_l10n(input_files, output, base): '''Generate pot file from lib/encodings''' - output = open(output, 'w') + output = io.open(output, 'w', encoding='utf_8', newline='\n') # assuming only one encodings file # Encoding utf8 utf8 "Unicode (utf8)" UTF-8 variable inputenc - reg = re.compile('Encoding [\w-]+\s+[\w-]+\s+"([\w \-\(\)]+)"\s+[\w-]+\s+(fixed|variable)\s+\w+.*') - input = open(input_files[0]) + reg = re.compile('Encoding [\w-]+\s+[\w-]+\s+"([\w \-\(\)^"]*)"\s+["\w-]+\s+(fixed|variable|variableunsafe)\s+\w+.*') + input = io.open(input_files[0], encoding='utf_8') for lineno, line in enumerate(input.readlines()): if not line.startswith('Encoding'): continue if reg.match(line): - print >> output, '#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ - (relativePath(input_files[0], base), lineno+1, reg.match(line).groups()[0]) + guiname = reg.match(line).groups()[0] + if guiname != "": + print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \ + (relativePath(input_files[0], base), lineno+1, guiname), file=output) else: - print "Error: Unable to handle line:" - print line + print("Error: Unable to handle line:") + print(line) # No need to abort if the parsing fails # sys.exit(1) input.close() @@ -553,8 +637,9 @@ where layouttranslations: create lib/layouttranslations from po/*.po and lib/layouts/* qt4: qt4 ui files languages: file lib/languages + latexfonts: file lib/latexfonts encodings: file lib/encodings - external: external templates file + external: external templates files formats: formats predefined in lib/configure.py ''' @@ -568,7 +653,7 @@ if __name__ == '__main__': ['help', 'type=', 'output=', 'base=', 'src_file=']) for (opt, value) in optlist: if opt in ['-h', '--help']: - print Usage + print(Usage) sys.exit(0) elif opt in ['-o', '--output']: output = value @@ -577,16 +662,25 @@ if __name__ == '__main__': elif opt in ['-t', '--type']: input_type = value elif opt in ['-s', '--src_file']: - input_files = [f.strip() for f in open(value)] + input_files = [f.strip() for f in io.open(value, encoding='utf_8')] - if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'encodings', 'external', 'formats'] or output is None: - print 'Wrong input type or output filename.' + if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'latexfonts', 'encodings', 'external', 'formats'] or output is None: + print('Wrong input type or output filename.') sys.exit(1) input_files += args + # Ensure a unique sorting of input files and ignore the order in which they + # are given on the command line. This is important to avoid huge + # pseudo-diffs in the generated .pot file which would then end up in the + # .po files as well. We had this situation for years with people using + # different build systems to remerge .po files. + input_files.sort() + if input_type == 'ui': ui_l10n(input_files, output, base) + elif input_type == 'latexfonts': + latexfonts_l10n(input_files, output, base) elif input_type == 'layouts': layouts_l10n(input_files, output, base, False) elif input_type == 'layouttranslations':