Now it produces the same output if running under python3 (tested with 3.4.2)
or python2 (tested with 2.7.9). python3 always uses unicode strings
internally, so we have to specify the file encoding on opening a file, such
that strings can be converted from and to the file encoding on reading and
writing. Using the io module for file io ensures that the behaviour is the
same for python2 and python3. For python2 we also have to mark string literals
as unicode strings by using the u prefix (which is a noop in python3).
Many thanks to José for review and pointing out all the details.
from __future__ import print_function
import sys, os, re, getopt
from __future__ import print_function
import sys, os, re, getopt
def relativePath(path, base):
'''return relative path from top source dir'''
def relativePath(path, base):
'''return relative path from top source dir'''
string = string.replace('\\', '\\\\').replace('"', '')
if string == "":
return
string = string.replace('\\', '\\\\').replace('"', '')
if string == "":
return
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(infile, basefile), lineno, string), file=outfile)
def ui_l10n(input_files, output, base):
'''Generate pot file from lib/ui/*'''
(relativePath(infile, basefile), lineno, string), file=outfile)
def ui_l10n(input_files, output, base):
'''Generate pot file from lib/ui/*'''
- output = open(output, 'w')
+ output = io.open(output, 'w', encoding='utf_8')
Submenu = re.compile(r'^[^#]*Submenu\s+"([^"]*)"', re.IGNORECASE)
Popupmenu = re.compile(r'^[^#]*PopupMenu\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
IconPalette = re.compile(r'^[^#]*IconPalette\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
Submenu = re.compile(r'^[^#]*Submenu\s+"([^"]*)"', re.IGNORECASE)
Popupmenu = re.compile(r'^[^#]*PopupMenu\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
IconPalette = re.compile(r'^[^#]*IconPalette\s+"[^"]+"\s+"([^"]*)"', re.IGNORECASE)
Item = re.compile(r'[^#]*Item\s+"([^"]*)"', re.IGNORECASE)
TableInsert = re.compile(r'[^#]*TableInsert\s+"([^"]*)"', re.IGNORECASE)
for src in input_files:
Item = re.compile(r'[^#]*Item\s+"([^"]*)"', re.IGNORECASE)
TableInsert = re.compile(r'[^#]*TableInsert\s+"([^"]*)"', re.IGNORECASE)
for src in input_files:
+ input = io.open(src, encoding='utf_8')
for lineno, line in enumerate(input.readlines()):
if Submenu.match(line):
(string,) = Submenu.match(line).groups()
for lineno, line in enumerate(input.readlines()):
if Submenu.match(line):
(string,) = Submenu.match(line).groups()
continue
string = string.replace('"', '')
if string != "":
continue
string = string.replace('"', '')
if string != "":
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(src, base), lineno+1, string), file=output)
input.close()
output.close()
(relativePath(src, base), lineno+1, string), file=output)
input.close()
output.close()
# read old translations if available
try:
# read old translations if available
try:
+ input = io.open(output, encoding='utf_8')
lang = ''
for line in input.readlines():
res = Comment.search(line)
lang = ''
for line in input.readlines():
res = Comment.search(line)
continue
res = KeyValPair.search(line)
if res and lang != '':
continue
res = KeyValPair.search(line)
if res and lang != '':
- key = res.group(1).decode('utf-8')
- val = res.group(2).decode('utf-8')
+ key = res.group(1)
+ val = res.group(2)
key = key.replace('\\"', '"').replace('\\\\', '\\')
val = val.replace('\\"', '"').replace('\\\\', '\\')
oldtrans[lang][key] = val
key = key.replace('\\"', '"').replace('\\\\', '\\')
val = val.replace('\\"', '"').replace('\\\\', '\\')
oldtrans[lang][key] = val
if 'wa' in languages:
languages.remove('wa')
if 'wa' in languages:
languages.remove('wa')
- out = open(output, 'w')
+ out = io.open(output, 'w', encoding='utf_8')
for src in input_files:
readingDescription = False
readingI18nPreamble = False
for src in input_files:
readingDescription = False
readingI18nPreamble = False
descStartLine = -1
descLines = []
lineno = 0
descStartLine = -1
descLines = []
lineno = 0
- for line in open(src).readlines():
+ for line in io.open(src, encoding='utf_8').readlines():
lineno += 1
res = ClassDescription.search(line)
if res != None:
lineno += 1
res = ClassDescription.search(line)
if res != None:
ContextRe = re.compile(r'(.*)(\[\[.*\]\])')
ContextRe = re.compile(r'(.*)(\[\[.*\]\])')
- print('''# This file has been automatically generated by po/lyx_pot.py.
+ print(u'''# This file has been automatically generated by po/lyx_pot.py.
# PLEASE MODIFY ONLY THE LAGUAGES HAVING NO .po FILE! If you want to regenerate
# this file from the translations, run `make ../lib/layouttranslations' in po.
# Python polib library is needed for building the output file.
# PLEASE MODIFY ONLY THE LAGUAGES HAVING NO .po FILE! If you want to regenerate
# this file from the translations, run `make ../lib/layouttranslations' in po.
# Python polib library is needed for building the output file.
# This file should remain fixed during minor LyX releases.
# For more comments see README.localization file.''', file=out)
for lang in languages:
# This file should remain fixed during minor LyX releases.
# For more comments see README.localization file.''', file=out)
for lang in languages:
- print('\nTranslation %s' % lang, file=out)
+ print(u'\nTranslation %s' % lang, file=out)
if lang in list(oldtrans.keys()):
trans = oldtrans[lang]
else:
if lang in list(oldtrans.keys()):
trans = oldtrans[lang]
else:
if res != None:
val = res.group(1)
key = key.replace('\\', '\\\\').replace('"', '\\"')
if res != None:
val = res.group(1)
key = key.replace('\\', '\\\\').replace('"', '\\"')
- print('\t"%s" "%s"' % \
- (key.encode('utf-8'), val.encode('utf-8')), file=out)
+ print(u'\t"%s" "%s"' % (key, val), file=out)
# also print untranslated entries to help translators
elif not lang in oldlanguages:
key = key.replace('\\', '\\\\').replace('"', '\\"')
# also print untranslated entries to help translators
elif not lang in oldlanguages:
key = key.replace('\\', '\\\\').replace('"', '\\"')
val = res.group(1)
else:
val = key
val = res.group(1)
else:
val = key
- print('\t"%s" "%s"' % \
- (key.encode('utf-8'), val.encode('utf-8')), file=out)
- print('End', file=out)
+ print(u'\t"%s" "%s"' % (key, val), file=out)
+ print(u'End', file=out)
out.close()
def qt4_l10n(input_files, output, base):
'''Generate pot file from src/frontends/qt4/ui/*.ui'''
out.close()
def qt4_l10n(input_files, output, base):
'''Generate pot file from src/frontends/qt4/ui/*.ui'''
- output = open(output, 'w')
+ output = io.open(output, 'w', encoding='utf_8')
pat = re.compile(r'\s*<string>(.*)</string>')
prop = re.compile(r'\s*<property.*name.*=.*shortcut')
for src in input_files:
pat = re.compile(r'\s*<string>(.*)</string>')
prop = re.compile(r'\s*<property.*name.*=.*shortcut')
for src in input_files:
+ input = io.open(src, encoding='utf_8')
skipNextLine = False
for lineno, line in enumerate(input.readlines()):
# skip the line after <property name=shortcut>
skipNextLine = False
for lineno, line in enumerate(input.readlines()):
# skip the line after <property name=shortcut>
string = string.replace('<', '<').replace('>', '>')
string = string.replace('\\', '\\\\').replace('"', r'\"')
string = string.replace('
', r'\n')
string = string.replace('<', '<').replace('>', '>')
string = string.replace('\\', '\\\\').replace('"', r'\"')
string = string.replace('
', r'\n')
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(src, base), lineno+1, string), file=output)
input.close()
output.close()
(relativePath(src, base), lineno+1, string), file=output)
input.close()
output.close()
def languages_l10n(input_files, output, base):
'''Generate pot file from lib/languages'''
def languages_l10n(input_files, output, base):
'''Generate pot file from lib/languages'''
- out = open(output, 'w')
+ out = io.open(output, 'w', encoding='utf_8')
GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
for src in input_files:
descStartLine = -1
descLines = []
lineno = 0
for src in input_files:
descStartLine = -1
descLines = []
lineno = 0
- for line in open(src).readlines():
+ for line in io.open(src, encoding='utf_8').readlines():
lineno += 1
res = GuiName.search(line)
if res != None:
string = res.group(1)
writeString(out, src, base, lineno, string)
continue
lineno += 1
res = GuiName.search(line)
if res != None:
string = res.group(1)
writeString(out, src, base, lineno, string)
continue
out.close()
def latexfonts_l10n(input_files, output, base):
'''Generate pot file from lib/latexfonts'''
out.close()
def latexfonts_l10n(input_files, output, base):
'''Generate pot file from lib/latexfonts'''
- out = open(output, 'w')
+ out = io.open(output, 'w', encoding='utf_8')
GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
GuiName = re.compile(r'^[^#]*GuiName\s+(.*)', re.IGNORECASE)
for src in input_files:
descStartLine = -1
descLines = []
lineno = 0
for src in input_files:
descStartLine = -1
descLines = []
lineno = 0
- for line in open(src).readlines():
+ for line in io.open(src, encoding='utf_8').readlines():
lineno += 1
res = GuiName.search(line)
if res != None:
string = res.group(1)
writeString(out, src, base, lineno, string)
continue
lineno += 1
res = GuiName.search(line)
if res != None:
string = res.group(1)
writeString(out, src, base, lineno, string)
continue
out.close()
def external_l10n(input_files, output, base):
'''Generate pot file from lib/external_templates'''
out.close()
def external_l10n(input_files, output, base):
'''Generate pot file from lib/external_templates'''
- output = open(output, 'w')
+ output = io.open(output, 'w', encoding='utf_8')
Template = re.compile(r'^Template\s+(.*)', re.IGNORECASE)
GuiName = re.compile(r'\s*GuiName\s+(.*)', re.IGNORECASE)
HelpTextStart = re.compile(r'\s*HelpText\s', re.IGNORECASE)
Template = re.compile(r'^Template\s+(.*)', re.IGNORECASE)
GuiName = re.compile(r'\s*GuiName\s+(.*)', re.IGNORECASE)
HelpTextStart = re.compile(r'\s*HelpText\s', re.IGNORECASE)
HelpTextEnd = re.compile(r'\s*HelpTextEnd\s', re.IGNORECASE)
i = -1
for src in input_files:
HelpTextEnd = re.compile(r'\s*HelpTextEnd\s', re.IGNORECASE)
i = -1
for src in input_files:
+ input = io.open(src, encoding='utf_8')
inHelp = False
hadHelp = False
prev_help_string = ''
inHelp = False
hadHelp = False
prev_help_string = ''
elif inHelp:
if HelpTextEnd.match(line):
if hadHelp:
elif inHelp:
if HelpTextEnd.match(line):
if hadHelp:
- print('\nmsgstr ""\n', file=output)
+ print(u'\nmsgstr ""\n', file=output)
inHelp = False
hadHelp = False
prev_help_string = ''
inHelp = False
hadHelp = False
prev_help_string = ''
(help_string,) = HelpTextSection.match(line).groups()
help_string = help_string.replace('"', '')
if help_string != "" and prev_help_string == '':
(help_string,) = HelpTextSection.match(line).groups()
help_string = help_string.replace('"', '')
if help_string != "" and prev_help_string == '':
- print('#: %s:%d\nmsgid ""\n"%s\\n"' % \
+ print(u'#: %s:%d\nmsgid ""\n"%s\\n"' % \
(relativePath(src, base), lineno+1, help_string), file=output)
hadHelp = True
elif help_string != "":
(relativePath(src, base), lineno+1, help_string), file=output)
hadHelp = True
elif help_string != "":
- print('"%s\\n"' % help_string, file=output)
+ print(u'"%s\\n"' % help_string, file=output)
prev_help_string = help_string
elif HelpTextStart.match(line):
inHelp = True
prev_help_string = help_string
elif HelpTextStart.match(line):
inHelp = True
continue
string = string.replace('"', '')
if string != "" and not inHelp:
continue
string = string.replace('"', '')
if string != "" and not inHelp:
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(src, base), lineno+1, string), file=output)
input.close()
output.close()
(relativePath(src, base), lineno+1, string), file=output)
input.close()
output.close()
def formats_l10n(input_files, output, base):
'''Generate pot file from configure.py'''
def formats_l10n(input_files, output, base):
'''Generate pot file from configure.py'''
- output = open(output, 'w')
+ output = io.open(output, 'w', encoding='utf_8')
GuiName = re.compile(r'.*\\Format\s+\S+\s+\S+\s+"([^"]*)"\s+(\S*)\s+.*', re.IGNORECASE)
GuiName2 = re.compile(r'.*\\Format\s+\S+\s+\S+\s+([^"]\S+)\s+(\S*)\s+.*', re.IGNORECASE)
GuiName = re.compile(r'.*\\Format\s+\S+\s+\S+\s+"([^"]*)"\s+(\S*)\s+.*', re.IGNORECASE)
GuiName2 = re.compile(r'.*\\Format\s+\S+\s+\S+\s+([^"]\S+)\s+(\S*)\s+.*', re.IGNORECASE)
- input = open(input_files[0])
+ input = io.open(input_files[0], encoding='utf_8')
for lineno, line in enumerate(input.readlines()):
label = ""
labelsc = ""
for lineno, line in enumerate(input.readlines()):
label = ""
labelsc = ""
if shortcut != "":
labelsc = label + "|" + shortcut
if label != "":
if shortcut != "":
labelsc = label + "|" + shortcut
if label != "":
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(input_files[0], base), lineno+1, label), file=output)
if labelsc != "":
(relativePath(input_files[0], base), lineno+1, label), file=output)
if labelsc != "":
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(input_files[0], base), lineno+1, labelsc), file=output)
input.close()
output.close()
(relativePath(input_files[0], base), lineno+1, labelsc), file=output)
input.close()
output.close()
def encodings_l10n(input_files, output, base):
'''Generate pot file from lib/encodings'''
def encodings_l10n(input_files, output, base):
'''Generate pot file from lib/encodings'''
- output = open(output, 'w')
+ output = io.open(output, 'w', encoding='utf_8')
# assuming only one encodings file
# Encoding utf8 utf8 "Unicode (utf8)" UTF-8 variable inputenc
reg = re.compile('Encoding [\w-]+\s+[\w-]+\s+"([\w \-\(\)]+)"\s+[\w-]+\s+(fixed|variable|variableunsafe)\s+\w+.*')
# assuming only one encodings file
# Encoding utf8 utf8 "Unicode (utf8)" UTF-8 variable inputenc
reg = re.compile('Encoding [\w-]+\s+[\w-]+\s+"([\w \-\(\)]+)"\s+[\w-]+\s+(fixed|variable|variableunsafe)\s+\w+.*')
- input = open(input_files[0])
+ input = io.open(input_files[0], encoding='utf_8')
for lineno, line in enumerate(input.readlines()):
if not line.startswith('Encoding'):
continue
if reg.match(line):
for lineno, line in enumerate(input.readlines()):
if not line.startswith('Encoding'):
continue
if reg.match(line):
- print('#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
+ print(u'#: %s:%d\nmsgid "%s"\nmsgstr ""\n' % \
(relativePath(input_files[0], base), lineno+1, reg.match(line).groups()[0]), file=output)
else:
print("Error: Unable to handle line:")
(relativePath(input_files[0], base), lineno+1, reg.match(line).groups()[0]), file=output)
else:
print("Error: Unable to handle line:")
elif opt in ['-t', '--type']:
input_type = value
elif opt in ['-s', '--src_file']:
elif opt in ['-t', '--type']:
input_type = value
elif opt in ['-s', '--src_file']:
- input_files = [f.strip() for f in open(value)]
+ input_files = [f.strip() for f in io.open(value, encoding='utf_8')]
if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'latexfonts', 'encodings', 'external', 'formats'] or output is None:
print('Wrong input type or output filename.')
if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'latexfonts', 'encodings', 'external', 'formats'] or output is None:
print('Wrong input type or output filename.')