2 # -*- coding: utf-8 -*-
5 # This file is part of LyX, the document processor.
6 # Licence details can be found in the file COPYING.
10 # Full author contact details are available in file CREDITS
14 # to get usage message
16 # This script will extract translatable strings from input files and write
17 # to output in gettext .pot format.
19 from __future__ import print_function
21 import sys, os, re, getopt
24 def relativePath(path, base):
25 '''return relative path from top source dir'''
26 # full pathname of path
27 path1 = os.path.normpath(os.path.realpath(path)).split(os.sep)
28 path2 = os.path.normpath(os.path.realpath(base)).split(os.sep)
29 if path1[:len(path2)] != path2:
30 print("Path %s is not under top source directory" % path)
31 path3 = os.path.join(*path1[len(path2):]);
32 # replace all \ by / such that we get the same comments on Windows and *nix
33 path3 = path3.replace('\\', '/')
37 def writeString(outfile, infile, basefile, lineno, string):
38 string = string.replace('\\', '\\\\').replace('"', '')
41 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
42 (relativePath(infile, basefile), lineno, string), file=outfile)
45 def ui_l10n(input_files, output, base):
46 '''Generate pot file from lib/ui/*'''
47 output = io.open(output, 'w', encoding='utf_8')
48 Submenu = re.compile(r'^[^#]*Submenu\s+"([^"]*)"', re.IGNORECASE)
49 Popupmenu = re.compile(r'^[^#]*PopupMenu\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
50 IconPalette = re.compile(r'^[^#]*IconPalette\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
51 Toolbar = re.compile(r'^[^#]*Toolbar\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
52 Item = re.compile(r'[^#]*Item\s+"([^"]*)"', re.IGNORECASE)
53 TableInsert = re.compile(r'[^#]*TableInsert\s+"([^"]*)"', re.IGNORECASE)
54 for src in input_files:
55 input = io.open(src, encoding='utf_8')
56 for lineno, line in enumerate(input.readlines()):
57 if Submenu.match(line):
58 (string,) = Submenu.match(line).groups()
59 string = string.replace('_', ' ')
60 elif Popupmenu.match(line):
61 (string,) = Popupmenu.match(line).groups()
62 elif IconPalette.match(line):
63 (string,) = IconPalette.match(line).groups()
64 elif Toolbar.match(line):
65 (string,) = Toolbar.match(line).groups()
66 elif Item.match(line):
67 (string,) = Item.match(line).groups()
68 elif TableInsert.match(line):
69 (string,) = TableInsert.match(line).groups()
72 string = string.replace('"', '')
74 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
75 (relativePath(src, base), lineno+1, string), file=output)
80 def layouts_l10n(input_files, output, base, layouttranslations):
81 '''Generate pot file from lib/layouts/*.{layout,inc,module}'''
82 ClassDescription = re.compile(r'^\s*#\s*\\Declare(LaTeX|DocBook)Class.*\{(.*)\}$', re.IGNORECASE)
83 ClassCategory = re.compile(r'^\s*#\s*\\DeclareCategory\{(.*)\}$', re.IGNORECASE)
84 Style = re.compile(r'^\s*Style\s+(.*\S)\s*$', re.IGNORECASE)
85 # match LabelString, EndLabelString, LabelStringAppendix and maybe others but no comments
86 LabelString = re.compile(r'^[^#]*LabelString\S*\s+(.*\S)\s*$', re.IGNORECASE)
87 MenuString = re.compile(r'^[^#]*MenuString\S*\s+(.*\S)\s*$', re.IGNORECASE)
88 OutlinerName = re.compile(r'^[^#]*OutlinerName\s+(\S+|\"[^\"]*\")\s+(\S+|\"[^\"]*\")\s*$', re.IGNORECASE)
89 Tooltip = re.compile(r'^\s*Tooltip\S*\s+(.*\S)\s*$', re.IGNORECASE)
90 GuiName = re.compile(r'^\s*GuiName\s+(.*\S)\s*$', re.IGNORECASE)
91 ListName = re.compile(r'^\s*ListName\s+(.*\S)\s*$', re.IGNORECASE)
92 CategoryName = re.compile(r'^\s*Category\s+(.*\S)\s*$', re.IGNORECASE)
93 NameRE = re.compile(r'^\s*#\s*\\DeclareLyXModule.*{(.*)}$', re.IGNORECASE)
94 InsetLayout = re.compile(r'^InsetLayout\s+\"?(.*)\"?\s*$', re.IGNORECASE)
95 FlexCheck = re.compile(r'^Flex:(.*)', re.IGNORECASE)
96 CaptionCheck = re.compile(r'^Caption:(.*)', re.IGNORECASE)
97 DescBegin = re.compile(r'^\s*#DescriptionBegin\s*$', re.IGNORECASE)
98 DescEnd = re.compile(r'^\s*#\s*DescriptionEnd\s*$', re.IGNORECASE)
99 Category = re.compile(r'^\s*#\s*Category:\s+(.*\S)\s*$', re.IGNORECASE)
100 I18nPreamble = re.compile(r'^\s*((Lang)|(Babel))Preamble\s*$', re.IGNORECASE)
101 EndI18nPreamble = re.compile(r'^\s*End((Lang)|(Babel))Preamble\s*$', re.IGNORECASE)
102 I18nString = re.compile(r'_\(([^\)]+)\)')
103 CounterFormat = re.compile(r'^\s*PrettyFormat\s+"?(.*)"?\s*$', re.IGNORECASE)
104 CiteFormat = re.compile(r'^\s*CiteFormat', re.IGNORECASE)
105 KeyVal = re.compile(r'^\s*_\w+\s+(.*\S)\s*$')
106 Float = re.compile(r'^\s*Float\s*$', re.IGNORECASE)
107 UsesFloatPkg = re.compile(r'^\s*UsesFloatPkg\s+(.*\S)\s*$', re.IGNORECASE)
108 IsPredefined = re.compile(r'^\s*IsPredefined\s+(.*\S)\s*$', re.IGNORECASE)
109 End = re.compile(r'^\s*End', re.IGNORECASE)
110 Comment = re.compile(r'^(.*)#')
111 Translation = re.compile(r'^\s*Translation\s+(.*\S)\s*$', re.IGNORECASE)
112 KeyValPair = re.compile(r'\s*"(.*)"\s+"(.*)"')
118 if layouttranslations:
119 linguas_file = os.path.join(base, 'po/LINGUAS')
120 for line in open(linguas_file).readlines():
121 res = Comment.search(line)
124 if line.strip() != '':
125 languages.extend(line.split())
127 # read old translations if available
129 input = io.open(output, encoding='utf_8')
131 for line in input.readlines():
132 res = Comment.search(line)
135 if line.strip() == '':
137 res = Translation.search(line)
140 if lang not in languages:
141 oldlanguages.append(lang)
142 languages.append(lang)
143 oldtrans[lang] = dict()
145 res = End.search(line)
149 res = KeyValPair.search(line)
150 if res and lang != '':
153 key = key.replace('\\"', '"').replace('\\\\', '\\')
154 val = val.replace('\\"', '"').replace('\\\\', '\\')
155 oldtrans[lang][key] = val
158 print("Error: Unable to handle line:")
161 print("Warning: Unable to open %s for reading." % output)
162 print(" Old translations will be lost.")
164 # walon is not a known document language
165 # FIXME: Do not hardcode, read from lib/languages!
166 if 'wa' in languages:
167 languages.remove('wa')
169 out = io.open(output, 'w', encoding='utf_8')
170 for src in input_files:
171 readingDescription = False
172 readingI18nPreamble = False
174 readingCiteFormats = False
182 for line in io.open(src, encoding='utf_8').readlines():
184 res = ClassDescription.search(line)
186 string = res.group(2)
187 if not layouttranslations:
188 writeString(out, src, base, lineno + 1, string)
190 res = ClassCategory.search(line)
192 string = res.group(1)
193 if not layouttranslations:
194 writeString(out, src, base, lineno + 1, string)
196 if readingDescription:
197 res = DescEnd.search(line)
199 readingDescription = False
200 desc = " ".join(descLines)
201 if not layouttranslations:
202 writeString(out, src, base, lineno + 1, desc)
204 descLines.append(line[1:].strip())
206 res = DescBegin.search(line)
208 readingDescription = True
209 descStartLine = lineno
211 if readingI18nPreamble:
212 res = EndI18nPreamble.search(line)
214 readingI18nPreamble = False
216 res = I18nString.search(line)
218 string = res.group(1)
219 if layouttranslations:
222 writeString(out, src, base, lineno, string)
224 res = I18nPreamble.search(line)
226 readingI18nPreamble = True
228 res = NameRE.search(line)
230 string = res.group(1)
231 if not layouttranslations:
232 writeString(out, src, base, lineno + 1, string)
234 res = Style.search(line)
236 string = res.group(1)
237 string = string.replace('_', ' ')
238 # Style means something else inside a float definition
240 if not layouttranslations:
241 writeString(out, src, base, lineno, string)
243 res = LabelString.search(line)
245 string = res.group(1)
246 if not layouttranslations:
247 writeString(out, src, base, lineno, string)
249 res = MenuString.search(line)
251 string = res.group(1)
252 if not layouttranslations:
253 writeString(out, src, base, lineno, string)
255 res = OutlinerName.search(line)
257 string = res.group(2)
258 if not layouttranslations:
259 writeString(out, src, base, lineno, string)
261 res = Tooltip.search(line)
263 string = res.group(1)
264 if not layouttranslations:
265 writeString(out, src, base, lineno, string)
267 res = GuiName.search(line)
269 string = res.group(1)
270 if layouttranslations:
271 # gui name must only be added for floats
275 writeString(out, src, base, lineno, string)
277 res = CategoryName.search(line)
279 string = res.group(1)
280 if not layouttranslations:
281 writeString(out, src, base, lineno, string)
283 res = ListName.search(line)
285 string = res.group(1)
286 if layouttranslations:
287 listname = string.strip('"')
289 writeString(out, src, base, lineno, string)
291 res = InsetLayout.search(line)
293 string = res.group(1)
294 string = string.replace('_', ' ')
295 #Flex:xxx is not used in translation
296 #if not layouttranslations:
297 # writeString(out, src, base, lineno, string)
298 m = FlexCheck.search(string)
300 if not layouttranslations:
301 writeString(out, src, base, lineno, m.group(1))
302 m = CaptionCheck.search(string)
304 if not layouttranslations:
305 writeString(out, src, base, lineno, m.group(1))
307 res = Category.search(line)
309 string = res.group(1)
310 if not layouttranslations:
311 writeString(out, src, base, lineno, string)
313 res = CounterFormat.search(line)
315 string = res.group(1)
316 if not layouttranslations:
317 writeString(out, src, base, lineno, string)
319 res = Float.search(line)
323 res = IsPredefined.search(line)
325 string = res.group(1).lower()
331 res = UsesFloatPkg.search(line)
333 string = res.group(1).lower()
339 res = CiteFormat.search(line)
341 readingCiteFormats = True
343 res = End.search(line)
345 # We have four combinations of the flags usesFloatPkg and isPredefined:
346 # usesFloatPkg and isPredefined: might use standard babel translations
347 # usesFloatPkg and not isPredefined: does not use standard babel translations
348 # not usesFloatPkg and isPredefined: uses standard babel translations
349 # not usesFloatPkg and not isPredefined: not supported by LyX
350 # The third combination is even true for MarginFigure, MarginTable (both from
351 # tufte-book.layout) and Planotable, Plate (both from aguplus.inc).
352 if layouttranslations and readingFloat and usesFloatPkg:
354 keyset.add(floatname)
361 readingCiteFormats = False
364 if readingCiteFormats:
365 res = KeyVal.search(line)
368 if not layouttranslations:
369 writeString(out, src, base, lineno, val)
371 if layouttranslations:
372 # Extract translations of layout files
375 # Sort languages and key to minimize the diff between different runs
376 # with changed translations
383 ContextRe = re.compile(r'(.*)(\[\[.*\]\])')
385 print(u'''# This file has been automatically generated by po/lyx_pot.py.
386 # PLEASE MODIFY ONLY THE LAGUAGES HAVING NO .po FILE! If you want to regenerate
387 # this file from the translations, run `make ../lib/layouttranslations' in po.
388 # Python polib library is needed for building the output file.
390 # This file should remain fixed during minor LyX releases.
391 # For more comments see README.localization file.''', file=out)
392 for lang in languages:
393 print(u'\nTranslation %s' % lang, file=out)
394 if lang in list(oldtrans.keys()):
395 trans = oldtrans[lang]
398 if not lang in oldlanguages:
399 poname = os.path.join(base, 'po/' + lang + '.po')
400 po = polib.pofile(poname)
401 # Iterate through po entries and not keys for speed reasons.
402 # FIXME: The code is still too slow
404 if not entry.translated():
406 if entry.msgid in keys:
409 # some translators keep untranslated entries
413 if key in list(trans.keys()):
414 val = trans[key].replace('\\', '\\\\').replace('"', '\\"')
415 res = ContextRe.search(val)
418 key = key.replace('\\', '\\\\').replace('"', '\\"')
419 print(u'\t"%s" "%s"' % (key, val), file=out)
420 # also print untranslated entries to help translators
421 elif not lang in oldlanguages:
422 key = key.replace('\\', '\\\\').replace('"', '\\"')
423 res = ContextRe.search(key)
428 print(u'\t"%s" "%s"' % (key, val), file=out)
429 print(u'End', file=out)
434 def qt4_l10n(input_files, output, base):
435 '''Generate pot file from src/frontends/qt4/ui/*.ui'''
436 output = io.open(output, 'w', encoding='utf_8')
437 pat = re.compile(r'\s*<string>(.*)</string>')
438 prop = re.compile(r'\s*<property.*name.*=.*shortcut')
439 for src in input_files:
440 input = io.open(src, encoding='utf_8')
442 for lineno, line in enumerate(input.readlines()):
443 # skip the line after <property name=shortcut>
450 # get lines that match <string>...</string>
452 (string,) = pat.match(line).groups()
453 string = string.replace('&', '&').replace('"', '"')
454 string = string.replace('<', '<').replace('>', '>')
455 string = string.replace('\\', '\\\\').replace('"', r'\"')
456 string = string.replace('
', r'\n')
457 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
458 (relativePath(src, base), lineno+1, string), file=output)
463 def languages_l10n(input_files, output, base):
464 '''Generate pot file from lib/languages'''
465 out = io.open(output, 'w', encoding='utf_8')
466 GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
468 for src in input_files:
472 for line in io.open(src, encoding='utf_8').readlines():
474 res = GuiName.search(line)
476 string = res.group(1)
477 writeString(out, src, base, lineno, string)
483 def latexfonts_l10n(input_files, output, base):
484 '''Generate pot file from lib/latexfonts'''
485 out = io.open(output, 'w', encoding='utf_8')
486 GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
488 for src in input_files:
492 for line in io.open(src, encoding='utf_8').readlines():
494 res = GuiName.search(line)
496 string = res.group(1)
497 writeString(out, src, base, lineno, string)
503 def external_l10n(input_files, output, base):
504 '''Generate pot file from lib/external_templates'''
505 output = io.open(output, 'w', encoding='utf_8')
506 Template = re.compile(r'^Template\s+(.*)', re.IGNORECASE)
507 GuiName = re.compile(r'\s*GuiName\s+(.*)', re.IGNORECASE)
508 HelpTextStart = re.compile(r'\s*HelpText\s', re.IGNORECASE)
509 HelpTextSection = re.compile(r'\s*(\S.*)\s*$')
510 HelpTextEnd = re.compile(r'\s*HelpTextEnd\s', re.IGNORECASE)
512 for src in input_files:
513 input = io.open(src, encoding='utf_8')
516 prev_help_string = ''
517 for lineno, line in enumerate(input.readlines()):
518 if Template.match(line):
519 (string,) = Template.match(line).groups()
520 elif GuiName.match(line):
521 (string,) = GuiName.match(line).groups()
523 if HelpTextEnd.match(line):
525 print(u'\nmsgstr ""\n', file=output)
528 prev_help_string = ''
529 elif HelpTextSection.match(line):
530 (help_string,) = HelpTextSection.match(line).groups()
531 help_string = help_string.replace('"', '')
532 if help_string != "" and prev_help_string == '':
533 print(u'#: %s:%d\nmsgid ""\n"%s\\n"' % \
534 (relativePath(src, base), lineno+1, help_string), file=output)
536 elif help_string != "":
537 print(u'"%s\\n"' % help_string, file=output)
538 prev_help_string = help_string
539 elif HelpTextStart.match(line):
541 prev_help_string = ''
544 string = string.replace('"', '')
545 if string != "" and not inHelp:
546 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
547 (relativePath(src, base), lineno+1, string), file=output)
552 def formats_l10n(input_files, output, base):
553 '''Generate pot file from configure.py'''
554 output = io.open(output, 'w', encoding='utf_8')
555 GuiName = re.compile(r'.*\\Format\s+\S+\s+\S+\s+"([^"]*)"\s+(\S*)\s+.*', re.IGNORECASE)
556 GuiName2 = re.compile(r'.*\\Format\s+\S+\s+\S+\s+([^"]\S+)\s+(\S*)\s+.*', re.IGNORECASE)
557 input = io.open(input_files[0], encoding='utf_8')
558 for lineno, line in enumerate(input.readlines()):
561 if GuiName.match(line):
562 label = GuiName.match(line).group(1)
563 shortcut = GuiName.match(line).group(2).replace('"', '')
564 elif GuiName2.match(line):
565 label = GuiName2.match(line).group(1)
566 shortcut = GuiName2.match(line).group(2).replace('"', '')
569 label = label.replace('\\', '\\\\').replace('"', '')
571 labelsc = label + "|" + shortcut
573 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
574 (relativePath(input_files[0], base), lineno+1, label), file=output)
576 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
577 (relativePath(input_files[0], base), lineno+1, labelsc), file=output)
582 def encodings_l10n(input_files, output, base):
583 '''Generate pot file from lib/encodings'''
584 output = io.open(output, 'w', encoding='utf_8')
585 # assuming only one encodings file
586 # Encoding utf8 utf8 "Unicode (utf8)" UTF-8 variable inputenc
587 reg = re.compile('Encoding [\w-]+\s+[\w-]+\s+"([\w \-\(\)]+)"\s+[\w-]+\s+(fixed|variable|variableunsafe)\s+\w+.*')
588 input = io.open(input_files[0], encoding='utf_8')
589 for lineno, line in enumerate(input.readlines()):
590 if not line.startswith('Encoding'):
593 print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
594 (relativePath(input_files[0], base), lineno+1, reg.match(line).groups()[0]), file=output)
596 print("Error: Unable to handle line:")
598 # No need to abort if the parsing fails
606 lyx_pot.py [-b|--base top_src_dir] [-o|--output output_file] [-h|--help] [-s|src_file filename] -t|--type input_type input_files
610 path to the top source directory. default to '.'
612 output pot file, default to './lyx.pot'
614 filename that contains a list of input files in each line
617 layouts: lib/layouts/*
618 layouttranslations: create lib/layouttranslations from po/*.po and lib/layouts/*
620 languages: file lib/languages
621 latexfonts: file lib/latexfonts
622 encodings: file lib/encodings
623 external: external templates file
624 formats: formats predefined in lib/configure.py
627 if __name__ == '__main__':
633 optlist, args = getopt.getopt(sys.argv[1:], 'ht:o:b:s:',
634 ['help', 'type=', 'output=', 'base=', 'src_file='])
635 for (opt, value) in optlist:
636 if opt in ['-h', '--help']:
639 elif opt in ['-o', '--output']:
641 elif opt in ['-b', '--base']:
643 elif opt in ['-t', '--type']:
645 elif opt in ['-s', '--src_file']:
646 input_files = [f.strip() for f in io.open(value, encoding='utf_8')]
648 if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'latexfonts', 'encodings', 'external', 'formats'] or output is None:
649 print('Wrong input type or output filename.')
654 if input_type == 'ui':
655 ui_l10n(input_files, output, base)
656 elif input_type == 'latexfonts':
657 latexfonts_l10n(input_files, output, base)
658 elif input_type == 'layouts':
659 layouts_l10n(input_files, output, base, False)
660 elif input_type == 'layouttranslations':
661 layouts_l10n(input_files, output, base, True)
662 elif input_type == 'qt4':
663 qt4_l10n(input_files, output, base)
664 elif input_type == 'external':
665 external_l10n(input_files, output, base)
666 elif input_type == 'formats':
667 formats_l10n(input_files, output, base)
668 elif input_type == 'encodings':
669 encodings_l10n(input_files, output, base)
671 languages_l10n(input_files, output, base)