2 # -*- coding: utf-8 -*-
5 # This file is part of LyX, the document processor.
6 # Licence details can be found in the file COPYING.
10 # Full author contact details are available in file CREDITS
14 # to get usage message
16 # This script will extract translatable strings from input files and write
17 # to output in gettext .pot format.
19 from __future__ import print_function
21 import sys, os, re, getopt
24 def relativePath(path, base):
25 '''return relative path from top source dir'''
26 # full pathname of path
27 path1 = os.path.normpath(os.path.realpath(path)).split(os.sep)
28 path2 = os.path.normpath(os.path.realpath(base)).split(os.sep)
29 if path1[:len(path2)] != path2:
30 print("Path %s is not under top source directory" % path)
31 path3 = os.path.join(*path1[len(path2):]);
32 # replace all \ by / such that we get the same comments on Windows and *nix
33 path3 = path3.replace('\\', '/')
37 def writeString(outfile, infile, basefile, lineno, string):
38 string = string.replace('\\', '\\\\').replace('"', '')
41 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
42 (relativePath(infile, basefile), lineno, string), file=outfile)
45 def ui_l10n(input_files, output, base):
46 '''Generate pot file from lib/ui/*'''
47 output = io.open(output, 'w', encoding='utf_8', newline='\n')
48 Submenu = re.compile(r'^[^#]*Submenu\s+"([^"]*)"', re.IGNORECASE)
49 Popupmenu = re.compile(r'^[^#]*PopupMenu\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
50 IconPalette = re.compile(r'^[^#]*IconPalette\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
51 Toolbar = re.compile(r'^[^#]*Toolbar\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
52 Item = re.compile(r'[^#]*Item\s+"([^"]*)"', re.IGNORECASE)
53 TableInsert = re.compile(r'[^#]*TableInsert\s+"([^"]*)"', re.IGNORECASE)
54 for src in input_files:
55 input = io.open(src, encoding='utf_8')
56 for lineno, line in enumerate(input.readlines()):
57 if Submenu.match(line):
58 (string,) = Submenu.match(line).groups()
59 string = string.replace('_', ' ')
60 elif Popupmenu.match(line):
61 (string,) = Popupmenu.match(line).groups()
62 elif IconPalette.match(line):
63 (string,) = IconPalette.match(line).groups()
64 elif Toolbar.match(line):
65 (string,) = Toolbar.match(line).groups()
66 elif Item.match(line):
67 (string,) = Item.match(line).groups()
68 elif TableInsert.match(line):
69 (string,) = TableInsert.match(line).groups()
72 string = string.replace('"', '')
74 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
75 (relativePath(src, base), lineno+1, string), file=output)
80 def layouts_l10n(input_files, output, base, layouttranslations):
81 '''Generate pot file from lib/layouts/*.{layout,inc,module}'''
82 ClassDescription = re.compile(r'^\s*#\s*\\Declare(LaTeX|DocBook)Class.*\{(.*)\}$', re.IGNORECASE)
83 ClassCategory = re.compile(r'^\s*#\s*\\DeclareCategory\{(.*)\}$', re.IGNORECASE)
84 Style = re.compile(r'^\s*Style\s+(.*\S)\s*$', re.IGNORECASE)
85 # match LabelString, EndLabelString, LabelStringAppendix and maybe others but no comments
86 LabelString = re.compile(r'^[^#]*LabelString\S*\s+(.*\S)\s*$', re.IGNORECASE)
87 MenuString = re.compile(r'^[^#]*MenuString\S*\s+(.*\S)\s*$', re.IGNORECASE)
88 OutlinerName = re.compile(r'^[^#]*OutlinerName\s+(\S+|\"[^\"]*\")\s+(\S+|\"[^\"]*\")\s*$', re.IGNORECASE)
89 Tooltip = re.compile(r'^\s*Tooltip\S*\s+(.*\S)\s*$', re.IGNORECASE)
90 GuiName = re.compile(r'^\s*GuiName\s+(.*\S)\s*$', re.IGNORECASE)
91 ListName = re.compile(r'^\s*ListName\s+(.*\S)\s*$', re.IGNORECASE)
92 CategoryName = re.compile(r'^\s*Category\s+(.*\S)\s*$', re.IGNORECASE)
93 NameRE = re.compile(r'^\s*#\s*\\DeclareLyXModule.*{(.*)}$', re.IGNORECASE)
94 CiteNameRE = re.compile(r'^\s*#\s*\\DeclareLyXCiteEngine.*{(.*)}$', re.IGNORECASE)
95 InsetLayout = re.compile(r'^InsetLayout\s+\"?(.*)\"?\s*$', re.IGNORECASE)
96 FlexCheck = re.compile(r'^Flex:(.*)', re.IGNORECASE)
97 CaptionCheck = re.compile(r'^Caption:(.*)', re.IGNORECASE)
98 DescBegin = re.compile(r'^\s*#DescriptionBegin\s*$', re.IGNORECASE)
99 DescEnd = re.compile(r'^\s*#\s*DescriptionEnd\s*$', re.IGNORECASE)
100 Category = re.compile(r'^\s*#\s*Category:\s+(.*\S)\s*$', re.IGNORECASE)
101 I18nPreamble = re.compile(r'^\s*((Lang)|(Babel))Preamble\s*$', re.IGNORECASE)
102 EndI18nPreamble = re.compile(r'^\s*End((Lang)|(Babel))Preamble\s*$', re.IGNORECASE)
103 I18nString = re.compile(r'_\(([^\)]+)\)')
104 CounterFormat = re.compile(r'^\s*PrettyFormat\s+"?(.*)"?\s*$', re.IGNORECASE)
105 CiteFormat = re.compile(r'^\s*CiteFormat', re.IGNORECASE)
106 KeyVal = re.compile(r'^\s*_\w+\s+(.*\S)\s*$')
107 Float = re.compile(r'^\s*Float\s*$', re.IGNORECASE)
108 UsesFloatPkg = re.compile(r'^\s*UsesFloatPkg\s+(.*\S)\s*$', re.IGNORECASE)
109 IsPredefined = re.compile(r'^\s*IsPredefined\s+(.*\S)\s*$', re.IGNORECASE)
110 End = re.compile(r'^\s*End', re.IGNORECASE)
111 Comment = re.compile(r'^(.*)#')
112 Translation = re.compile(r'^\s*Translation\s+(.*\S)\s*$', re.IGNORECASE)
113 KeyValPair = re.compile(r'\s*"(.*)"\s+"(.*)"')
119 if layouttranslations:
120 linguas_file = os.path.join(base, 'po/LINGUAS')
121 for line in open(linguas_file).readlines():
122 res = Comment.search(line)
125 if line.strip() != '':
126 languages.extend(line.split())
128 # read old translations if available
130 input = io.open(output, encoding='utf_8')
132 for line in input.readlines():
133 res = Comment.search(line)
136 if line.strip() == '':
138 res = Translation.search(line)
141 if lang not in languages:
142 oldlanguages.append(lang)
143 languages.append(lang)
144 oldtrans[lang] = dict()
146 res = End.search(line)
150 res = KeyValPair.search(line)
151 if res and lang != '':
154 key = key.replace('\\"', '"').replace('\\\\', '\\')
155 val = val.replace('\\"', '"').replace('\\\\', '\\')
156 oldtrans[lang][key] = val
159 print("Error: Unable to handle line:")
162 print("Warning: Unable to open %s for reading." % output)
163 print(" Old translations will be lost.")
165 # walon is not a known document language
166 # FIXME: Do not hardcode, read from lib/languages!
167 if 'wa' in languages:
168 languages.remove('wa')
170 if layouttranslations:
171 out = io.open(output, 'w', encoding='utf_8')
173 out = io.open(output, 'w', encoding='utf_8', newline='\n')
174 for src in input_files:
175 readingDescription = False
176 readingI18nPreamble = False
178 readingCiteFormats = False
186 for line in io.open(src, encoding='utf_8').readlines():
188 res = ClassDescription.search(line)
190 string = res.group(2)
191 if not layouttranslations:
192 writeString(out, src, base, lineno + 1, string)
194 res = ClassCategory.search(line)
196 string = res.group(1)
197 if not layouttranslations:
198 writeString(out, src, base, lineno + 1, string)
200 if readingDescription:
201 res = DescEnd.search(line)
203 readingDescription = False
204 desc = " ".join(descLines)
205 if not layouttranslations:
206 writeString(out, src, base, lineno + 1, desc)
208 descLines.append(line[1:].strip())
210 res = DescBegin.search(line)
212 readingDescription = True
213 descStartLine = lineno
215 if readingI18nPreamble:
216 res = EndI18nPreamble.search(line)
218 readingI18nPreamble = False
220 res = I18nString.search(line)
222 string = res.group(1)
223 if layouttranslations:
226 writeString(out, src, base, lineno, string)
228 res = I18nPreamble.search(line)
230 readingI18nPreamble = True
232 res = NameRE.search(line)
234 string = res.group(1)
235 if not layouttranslations:
236 writeString(out, src, base, lineno + 1, string)
238 res = CiteNameRE.search(line)
240 string = res.group(1)
241 if not layouttranslations:
242 writeString(out, src, base, lineno + 1, string)
244 res = Style.search(line)
246 string = res.group(1)
247 string = string.replace('_', ' ')
248 # Style means something else inside a float definition
250 if not layouttranslations:
251 writeString(out, src, base, lineno, string)
253 res = LabelString.search(line)
255 string = res.group(1)
256 if not layouttranslations:
257 writeString(out, src, base, lineno, string)
259 res = MenuString.search(line)
261 string = res.group(1)
262 if not layouttranslations:
263 writeString(out, src, base, lineno, string)
265 res = OutlinerName.search(line)
267 string = res.group(2)
268 if not layouttranslations:
269 writeString(out, src, base, lineno, string)
271 res = Tooltip.search(line)
273 string = res.group(1)
274 if not layouttranslations:
275 writeString(out, src, base, lineno, string)
277 res = GuiName.search(line)
279 string = res.group(1)
280 if layouttranslations:
281 # gui name must only be added for floats
285 writeString(out, src, base, lineno, string)
287 res = CategoryName.search(line)
289 string = res.group(1)
290 if not layouttranslations:
291 writeString(out, src, base, lineno, string)
293 res = ListName.search(line)
295 string = res.group(1)
296 if layouttranslations:
297 listname = string.strip('"')
299 writeString(out, src, base, lineno, string)
301 res = InsetLayout.search(line)
303 string = res.group(1)
304 string = string.replace('_', ' ')
305 #Flex:xxx is not used in translation
306 #if not layouttranslations:
307 # writeString(out, src, base, lineno, string)
308 m = FlexCheck.search(string)
310 if not layouttranslations:
311 writeString(out, src, base, lineno, m.group(1))
312 m = CaptionCheck.search(string)
314 if not layouttranslations:
315 writeString(out, src, base, lineno, m.group(1))
317 res = Category.search(line)
319 string = res.group(1)
320 if not layouttranslations:
321 writeString(out, src, base, lineno, string)
323 res = CounterFormat.search(line)
325 string = res.group(1)
326 if not layouttranslations:
327 writeString(out, src, base, lineno, string)
329 res = Float.search(line)
333 res = IsPredefined.search(line)
335 string = res.group(1).lower()
341 res = UsesFloatPkg.search(line)
343 string = res.group(1).lower()
349 res = CiteFormat.search(line)
351 readingCiteFormats = True
353 res = End.search(line)
355 # We have four combinations of the flags usesFloatPkg and isPredefined:
356 # usesFloatPkg and isPredefined: might use standard babel translations
357 # usesFloatPkg and not isPredefined: does not use standard babel translations
358 # not usesFloatPkg and isPredefined: uses standard babel translations
359 # not usesFloatPkg and not isPredefined: not supported by LyX
360 # The third combination is even true for MarginFigure, MarginTable (both from
361 # tufte-book.layout) and Planotable, Plate (both from aguplus.inc).
362 if layouttranslations and readingFloat and usesFloatPkg:
364 keyset.add(floatname)
371 readingCiteFormats = False
374 if readingCiteFormats:
375 res = KeyVal.search(line)
378 if not layouttranslations:
379 writeString(out, src, base, lineno, val)
381 if layouttranslations:
382 # Extract translations of layout files
385 # Sort languages and key to minimize the diff between different runs
386 # with changed translations
393 ContextRe = re.compile(r'(.*)(\[\[.*\]\])')
395 print(u'''# This file has been automatically generated by po/lyx_pot.py.
396 # PLEASE MODIFY ONLY THE LAGUAGES HAVING NO .po FILE! If you want to regenerate
397 # this file from the translations, run `make ../lib/layouttranslations' in po.
398 # Python polib library is needed for building the output file.
400 # This file should remain fixed during minor LyX releases.
401 # For more comments see README.localization file.''', file=out)
402 for lang in languages:
403 print(u'\nTranslation %s' % lang, file=out)
404 if lang in list(oldtrans.keys()):
405 trans = oldtrans[lang]
408 if not lang in oldlanguages:
409 poname = os.path.join(base, 'po/' + lang + '.po')
410 po = polib.pofile(poname)
411 # Iterate through po entries and not keys for speed reasons.
412 # FIXME: The code is still too slow
414 if not entry.translated():
416 if entry.msgid in keys:
419 # some translators keep untranslated entries
423 if key in list(trans.keys()):
424 val = trans[key].replace('\\', '\\\\').replace('"', '\\"')
425 res = ContextRe.search(val)
428 key = key.replace('\\', '\\\\').replace('"', '\\"')
429 print(u'\t"%s" "%s"' % (key, val), file=out)
430 # also print untranslated entries to help translators
431 elif not lang in oldlanguages:
432 key = key.replace('\\', '\\\\').replace('"', '\\"')
433 res = ContextRe.search(key)
438 print(u'\t"%s" "%s"' % (key, val), file=out)
439 print(u'End', file=out)
444 def qt4_l10n(input_files, output, base):
445 '''Generate pot file from src/frontends/qt4/ui/*.ui'''
446 output = io.open(output, 'w', encoding='utf_8', newline='\n')
447 pat = re.compile(r'\s*<string>(.*)</string>')
448 prop = re.compile(r'\s*<property.*name.*=.*shortcut')
449 for src in input_files:
450 input = io.open(src, encoding='utf_8')
452 for lineno, line in enumerate(input.readlines()):
453 # skip the line after <property name=shortcut>
460 # get lines that match <string>...</string>
462 (string,) = pat.match(line).groups()
463 string = string.replace('&', '&').replace('"', '"')
464 string = string.replace('<', '<').replace('>', '>')
465 string = string.replace('\\', '\\\\').replace('"', r'\"')
466 string = string.replace('
', r'\n')
467 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
468 (relativePath(src, base), lineno+1, string), file=output)
473 def languages_l10n(input_files, output, base):
474 '''Generate pot file from lib/languages'''
475 out = io.open(output, 'w', encoding='utf_8', newline='\n')
476 GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
478 for src in input_files:
482 for line in io.open(src, encoding='utf_8').readlines():
484 res = GuiName.search(line)
486 string = res.group(1)
487 writeString(out, src, base, lineno, string)
493 def latexfonts_l10n(input_files, output, base):
494 '''Generate pot file from lib/latexfonts'''
495 out = io.open(output, 'w', encoding='utf_8', newline='\n')
496 GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
498 for src in input_files:
502 for line in io.open(src, encoding='utf_8').readlines():
504 res = GuiName.search(line)
506 string = res.group(1)
507 writeString(out, src, base, lineno, string)
513 def external_l10n(input_files, output, base):
514 '''Generate pot file from lib/external_templates'''
515 output = io.open(output, 'w', encoding='utf_8', newline='\n')
516 Template = re.compile(r'^Template\s+(.*)', re.IGNORECASE)
517 GuiName = re.compile(r'\s*GuiName\s+(.*)', re.IGNORECASE)
518 HelpTextStart = re.compile(r'\s*HelpText\s', re.IGNORECASE)
519 HelpTextSection = re.compile(r'\s*(\S.*)\s*$')
520 HelpTextEnd = re.compile(r'\s*HelpTextEnd\s', re.IGNORECASE)
522 for src in input_files:
523 input = io.open(src, encoding='utf_8')
526 prev_help_string = ''
527 for lineno, line in enumerate(input.readlines()):
528 if Template.match(line):
529 (string,) = Template.match(line).groups()
530 elif GuiName.match(line):
531 (string,) = GuiName.match(line).groups()
533 if HelpTextEnd.match(line):
535 print(u'\nmsgstr ""\n', file=output)
538 prev_help_string = ''
539 elif HelpTextSection.match(line):
540 (help_string,) = HelpTextSection.match(line).groups()
541 help_string = help_string.replace('"', '')
542 if help_string != "" and prev_help_string == '':
543 print(u'#: %s:%d\nmsgid ""\n"%s\\n"' % \
544 (relativePath(src, base), lineno+1, help_string), file=output)
546 elif help_string != "":
547 print(u'"%s\\n"' % help_string, file=output)
548 prev_help_string = help_string
549 elif HelpTextStart.match(line):
551 prev_help_string = ''
554 string = string.replace('"', '')
555 if string != "" and not inHelp:
556 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
557 (relativePath(src, base), lineno+1, string), file=output)
562 def formats_l10n(input_files, output, base):
563 '''Generate pot file from configure.py'''
564 output = io.open(output, 'w', encoding='utf_8', newline='\n')
565 GuiName = re.compile(r'.*\\Format\s+\S+\s+\S+\s+"([^"]*)"\s+(\S*)\s+.*', re.IGNORECASE)
566 GuiName2 = re.compile(r'.*\\Format\s+\S+\s+\S+\s+([^"]\S+)\s+(\S*)\s+.*', re.IGNORECASE)
567 input = io.open(input_files[0], encoding='utf_8')
568 for lineno, line in enumerate(input.readlines()):
571 if GuiName.match(line):
572 label = GuiName.match(line).group(1)
573 shortcut = GuiName.match(line).group(2).replace('"', '')
574 elif GuiName2.match(line):
575 label = GuiName2.match(line).group(1)
576 shortcut = GuiName2.match(line).group(2).replace('"', '')
579 label = label.replace('\\', '\\\\').replace('"', '')
581 labelsc = label + "|" + shortcut
583 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
584 (relativePath(input_files[0], base), lineno+1, label), file=output)
586 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
587 (relativePath(input_files[0], base), lineno+1, labelsc), file=output)
592 def encodings_l10n(input_files, output, base):
593 '''Generate pot file from lib/encodings'''
594 output = io.open(output, 'w', encoding='utf_8', newline='\n')
595 # assuming only one encodings file
596 # Encoding utf8 utf8 "Unicode (utf8)" UTF-8 variable inputenc
597 reg = re.compile('Encoding [\w-]+\s+[\w-]+\s+"([\w \-\(\)]+)"\s+[\w-]+\s+(fixed|variable|variableunsafe)\s+\w+.*')
598 input = io.open(input_files[0], encoding='utf_8')
599 for lineno, line in enumerate(input.readlines()):
600 if not line.startswith('Encoding'):
603 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
604 (relativePath(input_files[0], base), lineno+1, reg.match(line).groups()[0]), file=output)
606 print("Error: Unable to handle line:")
608 # No need to abort if the parsing fails
616 lyx_pot.py [-b|--base top_src_dir] [-o|--output output_file] [-h|--help] [-s|src_file filename] -t|--type input_type input_files
620 path to the top source directory. default to '.'
622 output pot file, default to './lyx.pot'
624 filename that contains a list of input files in each line
627 layouts: lib/layouts/*
628 layouttranslations: create lib/layouttranslations from po/*.po and lib/layouts/*
630 languages: file lib/languages
631 latexfonts: file lib/latexfonts
632 encodings: file lib/encodings
633 external: external templates file
634 formats: formats predefined in lib/configure.py
637 if __name__ == '__main__':
643 optlist, args = getopt.getopt(sys.argv[1:], 'ht:o:b:s:',
644 ['help', 'type=', 'output=', 'base=', 'src_file='])
645 for (opt, value) in optlist:
646 if opt in ['-h', '--help']:
649 elif opt in ['-o', '--output']:
651 elif opt in ['-b', '--base']:
653 elif opt in ['-t', '--type']:
655 elif opt in ['-s', '--src_file']:
656 input_files = [f.strip() for f in io.open(value, encoding='utf_8')]
658 if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'latexfonts', 'encodings', 'external', 'formats'] or output is None:
659 print('Wrong input type or output filename.')
664 # Ensure a unique sorting of input files and ignore the order in which they
665 # are given on the command line. This is important to avoid huge
666 # pseudo-diffs in the generated .pot file which would then end up in the
667 # .po files as well. We had this situation for years with people using
668 # different build systems to remerge .po files.
671 if input_type == 'ui':
672 ui_l10n(input_files, output, base)
673 elif input_type == 'latexfonts':
674 latexfonts_l10n(input_files, output, base)
675 elif input_type == 'layouts':
676 layouts_l10n(input_files, output, base, False)
677 elif input_type == 'layouttranslations':
678 layouts_l10n(input_files, output, base, True)
679 elif input_type == 'qt4':
680 qt4_l10n(input_files, output, base)
681 elif input_type == 'external':
682 external_l10n(input_files, output, base)
683 elif input_type == 'formats':
684 formats_l10n(input_files, output, base)
685 elif input_type == 'encodings':
686 encodings_l10n(input_files, output, base)
688 languages_l10n(input_files, output, base)