-#!@PYTHON@
+#!/usr/bin/env python
"""
USAGE: translations-status.py BUILDSCRIPT-DIR LOCALEDIR
Reads template files translations.template.html.in
and for each LANG in LANGUAGES LANG/translations.template.html.in
-
Writes translations.html.in and for each LANG in LANGUAGES
translations.LANG.html.in
+ Writes out/translations-status.txt
+ Updates word counts in TRANSLATION
"""
import sys
import re
import string
import os
-import gettext
-import subprocess
+
+import langdefs
+import buildlib
def progress (str):
sys.stderr.write (str + '\n')
progress ("translations-status.py")
-buildscript_dir = sys.argv[1]
-localedir = sys.argv[2]
-
_doc = lambda s: s
-sys.path.append (buildscript_dir)
-import langdefs
-
# load gettext messages catalogs
-translation = {}
-for l in langdefs.LANGUAGES:
- if l.enabled and l.code != 'en':
- translation[l.code] = gettext.translation('lilypond-doc', localedir, [l.code]).gettext
-
-def read_pipe (command):
- child = subprocess.Popen (command,
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE,
- shell = True)
- (output, error) = child.communicate ()
- code = str (child.wait ())
- if not child.stdout or child.stdout.close ():
- print "pipe failed: %(command)s" % locals ()
- if code != '0':
- error = code + ' ' + error
- return (output, error)
+translation = langdefs.translation
+
+language_re = re.compile (r'^@documentlanguage (.+)', re.M)
comments_re = re.compile (r'^@ignore\n(.|\n)*?\n@end ignore$|@c .*?$', re.M)
space_re = re.compile (r'\s+', re.M)
lilypond_re = re.compile (r'@lilypond({.*?}|(.|\n)*?\n@end lilypond$)', re.M)
node_re = re.compile ('^@node .*?$', re.M)
-title_re = re.compile ('^@(top|chapter|(?:sub){0,2}section|(?:unnumbered|appendix)(?:(?:sub){0,2}sec)?) (.*?)$', re.M)
+title_re = re.compile ('^@(top|chapter|(?:sub){0,2}section|' + \
+'(?:unnumbered|appendix)(?:(?:sub){0,2}sec)?) (.*?)$', re.M)
include_re = re.compile ('^@include (.*?)$', re.M)
-committish_re = re.compile ('GIT [Cc]ommittish: ([a-f0-9]+)')
translators_re = re.compile (r'^@c\s+Translators\s*:\s*(.*?)$', re.M | re.I)
-checkers_re = re.compile (r'^@c\s+Translation\s*checkers\s*:\s*(.*?)$', re.M | re.I)
+checkers_re = re.compile (r'^@c\s+Translation\s*checkers\s*:\s*(.*?)$',
+ re.M | re.I)
status_re = re.compile (r'^@c\s+Translation\s*status\s*:\s*(.*?)$', re.M | re.I)
post_gdp_re = re.compile ('post.GDP', re.I)
-untranslated_node_str = 'UNTRANSLATED NODE: IGNORE ME'
+untranslated_node_str = '@untranslated'
skeleton_str = '-- SKELETON FILE --'
-diff_cmd = 'git diff --no-color %(committish)s HEAD -- %(original)s | cat'
-
+section_titles_string = _doc ('Section titles')
+last_updated_string = _doc (' <p><i>Last updated %s</i></p>\n')
+detailed_status_heads = [_doc ('Translators'), _doc ('Translation checkers'),
+ _doc ('Translated'), _doc ('Up to date'),
+ _doc ('Other info')]
format_table = {
- 'not translated': {'color':'d0f0f8', 'short':_doc ('no'), 'long':_doc ('not translated')},
- 'partially translated': {'color':'dfef77', 'short':_doc ('partially (%(p)d %%)'),
+ 'not translated': {'color':'d0f0f8', 'short':_doc ('no'), 'abbr':'NT',
+ 'long':_doc ('not translated')},
+ 'partially translated': {'color':'dfef77',
+ 'short':_doc ('partially (%(p)d %%)'),
+ 'abbr':'%(p)d%%',
'long':_doc ('partially translated (%(p)d %%)')},
- 'fully translated': {'color':'1fff1f', 'short':_doc ('yes'), 'long': _doc ('translated')},
- 'up to date': {'short':_doc ('yes'), 'long':_doc ('up to date')},
- 'outdated': {'short':_doc ('partially (%(p)d %%)'), 'long':_doc ('partially up-to-date (%(p)d %%)')},
- 'N/A': {'short':_doc ('N/A'), 'long':'', 'color':'d587ff' },
+ 'fully translated': {'color':'1fff1f', 'short':_doc ('yes'), 'abbr':'FT',
+ 'long': _doc ('translated')},
+ 'up to date': {'short':_doc ('yes'), 'long':_doc ('up to date'),
+ 'abbr':'100%%', 'vague':_doc ('up to date')},
+ 'outdated': {'short':_doc ('partially'), 'abbr':'%(p)d%%',
+ 'vague':_doc ('partially up to date')},
+ 'N/A': {'short':_doc ('N/A'), 'abbr':'N/A', 'color':'d587ff', 'vague':''},
'pre-GDP':_doc ('pre-GDP'),
'post-GDP':_doc ('post-GDP')
}
def __increase_last_index (self):
type = self.__data[-1][1]
if type == 'l':
- self.__data[-1][0] = self.__data[-1][0].translate (appendix_number_trans)
+ self.__data[-1][0] = \
+ self.__data[-1][0].translate (appendix_number_trans)
elif type == 'n':
self.__data[-1][0] += 1
def percentage_color (percent):
p = percent / 100.0
if p < 0.33:
- c = [hex (int (3 * p * b + (1 - 3 * p) * a))[2:] for (a, b) in [(0xff, 0xff), (0x5c, 0xa6), (0x5c, 0x4c)]]
+ c = [hex (int (3 * p * b + (1 - 3 * p) * a))[2:]
+ for (a, b) in [(0xff, 0xff), (0x5c, 0xa6), (0x5c, 0x4c)]]
elif p < 0.67:
- c = [hex (int ((3 * p - 1) * b + (2 - 3 * p) * a))[2:] for (a, b) in [(0xff, 0xff), (0xa6, 0xff), (0x4c, 0x3d)]]
+ c = [hex (int ((3 * p - 1) * b + (2 - 3 * p) * a))[2:]
+ for (a, b) in [(0xff, 0xff), (0xa6, 0xff), (0x4c, 0x3d)]]
else:
- c = [hex (int ((3 * p - 2) * b + 3 * (1 - p) * a))[2:] for (a, b) in [(0xff, 0x1f), (0xff, 0xff), (0x3d, 0x1f)]]
+ c = [hex (int ((3 * p - 2) * b + 3 * (1 - p) * a))[2:]
+ for (a, b) in [(0xff, 0x1f), (0xff, 0xff), (0x3d, 0x1f)]]
return ''.join (c)
+
+def update_word_count (text, filename, word_count):
+ return re.sub (r'(?m)^(\d+) *' + filename,
+ str (word_count).ljust (6) + filename,
+ text)
+
+po_msgid_re = re.compile (r'^msgid "(.*?)"(?:\n"(.*?)")*', re.M)
+
+def po_word_count (po_content):
+ s = ' '.join ([''.join (t) for t in po_msgid_re.findall (po_content)])
+ return len (space_re.split (s))
+
+sgml_tag_re = re.compile (r'<.*?>', re.S)
+
+def sgml_word_count (sgml_doc):
+ s = sgml_tag_re.sub ('', sgml_doc)
+ return len (space_re.split (s))
+
def tely_word_count (tely_doc):
'''
Calculate word count of a Texinfo document node by node.
self.title = 'Untitled'
self.level = ('u', 1)
- included_files = [os.path.join (os.path.dirname (filename), t) for t in include_re.findall (self.contents)]
+ m = language_re.search (self.contents)
+ if m:
+ self.language = m.group (1)
+
+ included_files = [os.path.join (os.path.dirname (filename), t)
+ for t in include_re.findall (self.contents)]
self.included_files = [p for p in included_files if os.path.exists (p)]
def print_title (self, section_number):
TelyDocument.__init__ (self, filename)
self.masterdocument = masterdocument
+ if not hasattr (self, 'language') \
+ and hasattr (parent_translation, 'language'):
+ self.language = parent_translation.language
+ if hasattr (self, 'language'):
+ self.translation = translation[self.language]
+ else:
+ self.translation = lambda x: x
+ self.title = self.translation (self.title)
## record authoring information
m = translators_re.search (self.contents)
## calculate translation percentage
master_total_word_count = sum (masterdocument.word_count)
- translation_word_count = sum ([masterdocument.word_count[k] * self.translated_nodes[k]
- for k in range (min (len (masterdocument.word_count), len (self.translated_nodes)))])
- self.translation_percentage = 100 * translation_word_count / master_total_word_count
+ translation_word_count = \
+ sum ([masterdocument.word_count[k] * self.translated_nodes[k]
+ for k in range (min (len (masterdocument.word_count),
+ len (self.translated_nodes)))])
+ self.translation_percentage = \
+ 100 * translation_word_count / master_total_word_count
## calculate how much the file is outdated
- m = committish_re.search (self.contents)
- if not m:
- sys.stderr.write ('error: ' + filename + \
- ": no 'GIT committish: <hash>' found.\nPlease check " + \
- 'the whole file against the original in English, then ' + \
- 'fill in HEAD committish in the header.\n')
- sys.exit (1)
- (diff_string, error) = read_pipe (diff_cmd % {'committish':m.group (1), 'original':masterdocument.filename})
+ (diff_string, error) = \
+ buildlib.check_translated_doc (masterdocument.filename, self.filename, self.contents)
if error:
sys.stderr.write ('warning: %s: %s' % (self.filename, error))
self.uptodate_percentage = None
else:
diff = diff_string.splitlines ()
- insertions = sum ([len (l) - 1 for l in diff if l.startswith ('+') and not l.startswith ('+++')])
- deletions = sum ([len (l) - 1 for l in diff if l.startswith ('-') and not l.startswith ('---')])
- outdateness_percentage = 50.0 * (deletions + insertions) / (masterdocument.size + 0.5 * (deletions - insertions))
+ insertions = sum ([len (l) - 1 for l in diff
+ if l.startswith ('+')
+ and not l.startswith ('+++')])
+ deletions = sum ([len (l) - 1 for l in diff
+ if l.startswith ('-')
+ and not l.startswith ('---')])
+ outdateness_percentage = 50.0 * (deletions + insertions) / \
+ (masterdocument.size + 0.5 * (deletions - insertions))
self.uptodate_percentage = 100 - int (outdateness_percentage)
if self.uptodate_percentage > 100:
alternative = 50
- progress ("%s: strange uptodateness percentage %d %%, setting to %d %%" \
- % (self.filename, self.uptodate_percentage, alternative))
+ progress ("%s: strange uptodateness percentage %d %%, \
+setting to %d %%" % (self.filename, self.uptodate_percentage, alternative))
self.uptodate_percentage = alternative
elif self.uptodate_percentage < 1:
alternative = 1
- progress ("%s: strange uptodateness percentage %d %%, setting to %d %%" \
- % (self.filename, self.uptodate_percentage, alternative))
+ progress ("%s: strange uptodateness percentage %d %%, \
+setting to %d %%" % (self.filename, self.uptodate_percentage, alternative))
self.uptodate_percentage = alternative
- def completeness (self, formats=['long']):
+ def completeness (self, formats=['long'], translated=False):
+ if translated:
+ translation = self.translation
+ else:
+ translation = lambda x: x
+
if isinstance (formats, str):
formats = [formats]
p = self.translation_percentage
status = 'fully translated'
else:
status = 'partially translated'
- return dict ([(f, format_table[status][f] % locals()) for f in formats])
+ return dict ([(f, translation (format_table[status][f]) % locals())
+ for f in formats])
+
+ def uptodateness (self, formats=['long'], translated=False):
+ if translated:
+ translation = self.translation
+ else:
+ translation = lambda x: x
- def uptodateness (self, formats=['long']):
if isinstance (formats, str):
formats = [formats]
p = self.uptodate_percentage
if f == 'color' and p != None:
l['color'] = percentage_color (p)
else:
- l[f] = format_table[status][f] % locals ()
+ l[f] = translation (format_table[status][f]) % locals ()
return l
- def gdp_status (self, translation=lambda s: s):
+ def gdp_status (self):
if self.post_gdp:
- return translation (format-table['post-GDP'])
+ return self.translation (format_table['post-GDP'])
else:
- return translation (format-table['pre-GDP'])
+ return self.translation (format_table['pre-GDP'])
def short_html_status (self):
s = ' <td>'
if self.partially_translated:
s += '<br>\n '.join (self.translators) + '<br>\n'
if self.checkers:
- s += ' <small>' + '<br>\n '.join (self.checkers) + '</small><br>\n'
+ s += ' <small>' + \
+ '<br>\n '.join (self.checkers) + '</small><br>\n'
- c = self.completeness (['long', 'color'])
- s += ' <span style="background-color: #%(color)s">%(long)s</span><br>\n' % c
+ c = self.completeness (['color', 'long'])
+ s += ' <span style="background-color: #%(color)s">\
+%(long)s</span><br>\n' % c
if self.partially_translated:
- u = self.uptodateness (['long', 'color'])
- s += ' <span style="background-color: #%(color)s">%(long)s</span><br>\n' % u
+ u = self.uptodateness (['vague', 'color'])
+ s += ' <span style="background-color: #%(color)s">\
+%(vague)s</span><br>\n' % u
s += ' </td>\n'
return s
- def html_status (self):
- # TODO
- return ''
+ def text_status (self):
+ s = self.completeness ('abbr')['abbr'] + ' '
+
+ if self.partially_translated:
+ s += self.uptodateness ('abbr')['abbr'] + ' '
+ return s
+
+ def html_status (self, numbering=SectionNumber ()):
+ if self.title == 'Untitled':
+ return ''
+
+ if self.level[1] == 0: # if self is a master document
+ s = '''<table align="center" border="2">
+ <tr align="center">
+ <th>%s</th>''' % self.print_title (numbering)
+ s += ''.join ([' <th>%s</th>\n' % self.translation (h)
+ for h in detailed_status_heads])
+ s += ' </tr>\n'
+ s += ' <tr align="left">\n <td>%s<br>(%d)</td>\n' \
+ % (self.translation (section_titles_string),
+ sum (self.masterdocument.word_count))
+
+ else:
+ s = ' <tr align="left">\n <td>%s<br>(%d)</td>\n' \
+ % (self.print_title (numbering),
+ sum (self.masterdocument.word_count))
+
+ if self.partially_translated:
+ s += ' <td>' + '<br>\n '.join (self.translators) + '</td>\n'
+ s += ' <td>' + '<br>\n '.join (self.checkers) + '</td>\n'
+ else:
+ s += ' <td></td>\n' * 2
+
+ c = self.completeness (['color', 'short'], translated=True)
+ s += ' <td><span style="background-color: #%(color)s">\
+%(short)s</span></td>\n' % {'color': c['color'],
+ 'short': c['short']}
+
+ if self.partially_translated:
+ u = self.uptodateness (['short', 'color'], translated=True)
+ s += ' <td><span style="background-color: #%(color)s">\
+%(short)s</span></td>\n' % {'color': u['color'],
+ 'short': u['short']}
+ else:
+ s += ' <td></td>\n'
+
+ s += ' <td>' + self.gdp_status () + '</td>\n </tr>\n'
+ s += ''.join ([i.translations[self.language].html_status (numbering)
+ for i in self.masterdocument.includes
+ if self.language in i.translations])
+
+ if self.level[1] == 0: # if self is a master document
+ s += '</table>\n<p></p>\n'
+ return s
class MasterTelyDocument (TelyDocument):
- def __init__ (self, filename, parent_translations=dict ([(lang, None) for lang in langdefs.LANGDICT.keys()])):
- #print "init MasterTelyDocument %s" % filename
+ def __init__ (self,
+ filename,
+ parent_translations=dict ([(lang, None)
+ for lang in langdefs.LANGDICT])):
TelyDocument.__init__ (self, filename)
self.size = len (self.contents)
self.word_count = tely_word_count (self.contents)
- translations = dict ([(lang, os.path.join (lang, filename)) for lang in langdefs.LANGDICT.keys()])
- #print translations
- self.translations = dict ([(lang, TranslatedTelyDocument (translations[lang], self, parent_translations.get (lang)))
- for lang in langdefs.LANGDICT.keys() if os.path.exists (translations[lang])])
+ translations = dict ([(lang, os.path.join (lang, filename))
+ for lang in langdefs.LANGDICT])
+ self.translations = \
+ dict ([(lang,
+ TranslatedTelyDocument (translations[lang],
+ self, parent_translations.get (lang)))
+ for lang in langdefs.LANGDICT
+ if os.path.exists (translations[lang])])
if self.translations:
- self.includes = [MasterTelyDocument (f, self.translations) for f in self.included_files]
+ self.includes = [MasterTelyDocument (f, self.translations)
+ for f in self.included_files]
else:
self.includes = []
- # TODO
- def print_wc_priority (self):
- return
+ def update_word_counts (self, s):
+ s = update_word_count (s, self.filename, sum (self.word_count))
+ for i in self.includes:
+ s = i.update_word_counts (s)
+ return s
def html_status (self, numbering=SectionNumber ()):
if self.title == 'Untitled' or not self.translations:
s = '''<table align="center" border="2">
<tr align="center">
<th>%s</th>''' % self.print_title (numbering)
- s += ''.join ([' <th>%s</th>\n' % l for l in self.translations.keys ()])
+ s += ''.join ([' <th>%s</th>\n' % l for l in self.translations])
s += ' </tr>\n'
s += ' <tr align="left">\n <td>Section titles<br>(%d)</td>\n' \
% sum (self.word_count)
- else:
+ else: # if self is an included file
s = ' <tr align="left">\n <td>%s<br>(%d)</td>\n' \
% (self.print_title (numbering), sum (self.word_count))
- s += ''.join ([t.short_html_status () for t in self.translations.values ()])
+ s += ''.join ([t.short_html_status ()
+ for t in self.translations.values ()])
s += ' </tr>\n'
s += ''.join ([i.html_status (numbering) for i in self.includes])
- if self.level[1] == 0:
+ if self.level[1] == 0: # if self is a master document
s += '</table>\n<p></p>\n'
return s
+ def text_status (self, numbering=SectionNumber (), colspec=[48,12]):
+ if self.title == 'Untitled' or not self.translations:
+ return ''
+
+ s = ''
+ if self.level[1] == 0: # if self is a master document
+ s += (self.print_title (numbering) + ' ').ljust (colspec[0])
+ s += ''.join (['%s'.ljust (colspec[1]) % l
+ for l in self.translations])
+ s += '\n'
+ s += ('Section titles (%d)' % \
+ sum (self.word_count)).ljust (colspec[0])
+
+ else:
+ s = '%s (%d) ' \
+ % (self.print_title (numbering), sum (self.word_count))
+ s = s.ljust (colspec[0])
+
+ s += ''.join ([t.text_status ().ljust(colspec[1])
+ for t in self.translations.values ()])
+ s += '\n\n'
+ s += ''.join ([i.text_status (numbering) for i in self.includes])
+
+ if self.level[1] == 0:
+ s += '\n'
+ return s
+
+
+update_category_word_counts_re = re.compile (r'(?ms)^-(\d+)-(.*?\n)\d+ *total')
+
+counts_re = re.compile (r'(?m)^(\d+) ')
+
+def update_category_word_counts_sub (m):
+ return '-' + m.group (1) + '-' + m.group (2) + \
+ str (sum ([int (c)
+ for c in counts_re.findall (m.group (2))])).ljust (6) + \
+ 'total'
+
+
progress ("Reading documents...")
-tely_files = read_pipe ("find -maxdepth 2 -name '*.tely'")[0].splitlines ()
-master_docs = [MasterTelyDocument (os.path.normpath (filename)) for filename in tely_files]
+tely_files = \
+ buildlib.read_pipe ("find -maxdepth 2 -name '*.tely'")[0].splitlines ()
+tely_files.sort ()
+master_docs = [MasterTelyDocument (os.path.normpath (filename))
+ for filename in tely_files]
master_docs = [doc for doc in master_docs if doc.translations]
main_status_page = open ('translations.template.html.in').read ()
-## TODO
-#per_lang_status_pages = dict ([(l, open (os.path.join (l, 'translations.template.html')). read ())
-# for l in langdefs.LANGDICT.keys ()
-# if langdefs.LANGDICT[l].enabled])
+enabled_languages = [l for l in langdefs.LANGDICT
+ if langdefs.LANGDICT[l].enabled
+ and l != 'en']
+lang_status_pages = \
+ dict ([(l, open (os.path.join (l, 'translations.template.html.in')). read ())
+ for l in enabled_languages])
progress ("Generating status pages...")
-main_status_html = ' <p><i>Last updated %s</i></p>\n' % read_pipe ('LANG= date -u')[0]
+date_time = buildlib.read_pipe ('LANG= date -u')[0]
+
+main_status_html = last_updated_string % date_time
main_status_html += '\n'.join ([doc.html_status () for doc in master_docs])
html_re = re.compile ('<html>', re.I)
end_body_re = re.compile ('</body>', re.I)
-main_status_page = html_re.sub ('''<html>
+html_header = '''<html>
<!-- This page is automatically generated by translation-status.py from
-translations.template.html.in; DO NOT EDIT !-->''', main_status_page)
+translations.template.html.in; DO NOT EDIT !-->'''
-main_status_page = end_body_re.sub (main_status_html + '\n</body>', main_status_page)
+main_status_page = html_re.sub (html_header, main_status_page)
+
+main_status_page = end_body_re.sub (main_status_html + '\n</body>',
+ main_status_page)
open ('translations.html.in', 'w').write (main_status_page)
+
+for l in enabled_languages:
+ date_time = buildlib.read_pipe ('LANG=%s date -u' % l)[0]
+ lang_status_pages[l] = translation[l] (last_updated_string) % date_time + lang_status_pages[l]
+ lang_status_page = html_re.sub (html_header, lang_status_pages[l])
+ html_status = '\n'.join ([doc.translations[l].html_status ()
+ for doc in master_docs
+ if l in doc.translations])
+ lang_status_page = end_body_re.sub (html_status + '\n</body>',
+ lang_status_page)
+ open (os.path.join (l, 'translations.html.in'), 'w').write (lang_status_page)
+
+main_status_txt = '''Documentation translations status
+Generated %s
+NT = not translated
+FT = fully translated
+
+''' % date_time
+
+main_status_txt += '\n'.join ([doc.text_status () for doc in master_docs])
+
+status_txt_file = 'out/translations-status.txt'
+progress ("Writing %s..." % status_txt_file)
+open (status_txt_file, 'w').write (main_status_txt)
+
+translation_instructions_file = 'TRANSLATION'
+progress ("Updating %s..." % translation_instructions_file)
+translation_instructions = open (translation_instructions_file).read ()
+
+for doc in master_docs:
+ translation_instructions = doc.update_word_counts (translation_instructions)
+
+for html_file in re.findall (r'(?m)^\d+ *(\S+?\.html\S*?)(?: |$)',
+ translation_instructions):
+ word_count = sgml_word_count (open (html_file).read ())
+ translation_instructions = update_word_count (translation_instructions,
+ html_file,
+ word_count)
+
+for po_file in re.findall (r'(?m)^\d+ *(\S+?\.po\S*?)(?: |$)',
+ translation_instructions):
+ word_count = po_word_count (open (po_file).read ())
+ translation_instructions = update_word_count (translation_instructions,
+ po_file,
+ word_count)
+
+translation_instructions = \
+ update_category_word_counts_re.sub (update_category_word_counts_sub,
+ translation_instructions)
+
+open (translation_instructions_file, 'w').write (translation_instructions)
projects / lilypond.git / blobdiff