X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=buildscripts%2Ftranslations-status.py;h=a3b6799f21f8b43777ed3e69229610395114062f;hb=3ead9c2be44a4f5bb6d32a776188931f7995048e;hp=2335d5d5e16c7dfd36af4560fedc6b33aac3eb8f;hpb=e80a3d7fa7d74a57f2a4003b0cc8ce3aab9b5b30;p=lilypond.git diff --git a/buildscripts/translations-status.py b/buildscripts/translations-status.py old mode 100644 new mode 100755 index 2335d5d5e1..a3b6799f21 --- a/buildscripts/translations-status.py +++ b/buildscripts/translations-status.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!/usr/bin/env python """ USAGE: translations-status.py BUILDSCRIPT-DIR LOCALEDIR @@ -7,17 +7,18 @@ USAGE: translations-status.py BUILDSCRIPT-DIR LOCALEDIR Reads template files translations.template.html.in and for each LANG in LANGUAGES LANG/translations.template.html.in - Writes translations.html.in and for each LANG in LANGUAGES translations.LANG.html.in + Writes out/translations-status.txt + Updates word counts in TRANSLATION """ import sys import re import string import os -import gettext -import subprocess + +import langdefs def progress (str): sys.stderr.write (str + '\n') @@ -25,59 +26,50 @@ def progress (str): progress ("translations-status.py") buildscript_dir = sys.argv[1] -localedir = sys.argv[2] _doc = lambda s: s sys.path.append (buildscript_dir) -import langdefs +import buildlib # load gettext messages catalogs -translation = {} -for l in langdefs.LANGUAGES: - if l.enabled and l.code != 'en': - translation[l.code] = gettext.translation('lilypond-doc', localedir, [l.code]).gettext - -def read_pipe (command): - child = subprocess.Popen (command, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - shell = True) - (output, error) = child.communicate () - code = str (child.wait ()) - if not child.stdout or child.stdout.close (): - print "pipe failed: %(command)s" % locals () - if code != '0': - error = code + ' ' + error - return (output, error) +translation = langdefs.translation + +language_re = re.compile (r'^@documentlanguage (.+)', re.M) comments_re = re.compile (r'^@ignore\n(.|\n)*?\n@end ignore$|@c .*?$', re.M) space_re = re.compile (r'\s+', re.M) lilypond_re = re.compile (r'@lilypond({.*?}|(.|\n)*?\n@end lilypond$)', re.M) node_re = re.compile ('^@node .*?$', re.M) -title_re = re.compile ('^@(top|chapter|(?:sub){0,2}section|(?:unnumbered|appendix)(?:(?:sub){0,2}sec)?) (.*?)$', re.M) +title_re = re.compile ('^@(top|chapter|(?:sub){0,2}section|' + \ +'(?:unnumbered|appendix)(?:(?:sub){0,2}sec)?) (.*?)$', re.M) include_re = re.compile ('^@include (.*?)$', re.M) -committish_re = re.compile ('GIT [Cc]ommittish: ([a-f0-9]+)') translators_re = re.compile (r'^@c\s+Translators\s*:\s*(.*?)$', re.M | re.I) -checkers_re = re.compile (r'^@c\s+Translation\s*checkers\s*:\s*(.*?)$', re.M | re.I) +checkers_re = re.compile (r'^@c\s+Translation\s*checkers\s*:\s*(.*?)$', + re.M | re.I) status_re = re.compile (r'^@c\s+Translation\s*status\s*:\s*(.*?)$', re.M | re.I) post_gdp_re = re.compile ('post.GDP', re.I) untranslated_node_str = 'UNTRANSLATED NODE: IGNORE ME' skeleton_str = '-- SKELETON FILE --' -diff_cmd = 'git diff --no-color %(committish)s HEAD -- %(original)s | cat' - +section_titles_string = _doc ('Section titles') +last_updated_string = _doc ('

Last updated %s

\n') +detailed_status_heads = [_doc ('Translators'), _doc ('Translation checkers'), + _doc ('Translated'), _doc ('Up to date'), + _doc ('Other info')] format_table = { 'not translated': {'color':'d0f0f8', 'short':_doc ('no'), 'abbr':'NT', 'long':_doc ('not translated')}, - 'partially translated': {'color':'dfef77', 'short':_doc ('partially (%(p)d %%)'), - 'abbr':'%(p)d%%', 'long':_doc ('partially translated (%(p)d %%)')}, + 'partially translated': {'color':'dfef77', + 'short':_doc ('partially (%(p)d %%)'), + 'abbr':'%(p)d%%', + 'long':_doc ('partially translated (%(p)d %%)')}, 'fully translated': {'color':'1fff1f', 'short':_doc ('yes'), 'abbr':'FT', 'long': _doc ('translated')}, - 'up to date': {'short':_doc ('yes'), 'long':_doc ('up to date'), 'abbr':'100%%', - 'vague':_doc ('up to date')}, - 'outdated': {'short':_doc ('partially (%(p)d %%)'), 'abbr':'%(p)d%%', + 'up to date': {'short':_doc ('yes'), 'long':_doc ('up to date'), + 'abbr':'100%%', 'vague':_doc ('up to date')}, + 'outdated': {'short':_doc ('partially'), 'abbr':'%(p)d%%', 'vague':_doc ('partially up to date')}, 'N/A': {'short':_doc ('N/A'), 'abbr':'N/A', 'color':'d587ff', 'vague':''}, 'pre-GDP':_doc ('pre-GDP'), @@ -106,7 +98,8 @@ class SectionNumber (object): def __increase_last_index (self): type = self.__data[-1][1] if type == 'l': - self.__data[-1][0] = self.__data[-1][0].translate (appendix_number_trans) + self.__data[-1][0] = \ + self.__data[-1][0].translate (appendix_number_trans) elif type == 'n': self.__data[-1][0] += 1 @@ -138,13 +131,34 @@ class SectionNumber (object): def percentage_color (percent): p = percent / 100.0 if p < 0.33: - c = [hex (int (3 * p * b + (1 - 3 * p) * a))[2:] for (a, b) in [(0xff, 0xff), (0x5c, 0xa6), (0x5c, 0x4c)]] + c = [hex (int (3 * p * b + (1 - 3 * p) * a))[2:] + for (a, b) in [(0xff, 0xff), (0x5c, 0xa6), (0x5c, 0x4c)]] elif p < 0.67: - c = [hex (int ((3 * p - 1) * b + (2 - 3 * p) * a))[2:] for (a, b) in [(0xff, 0xff), (0xa6, 0xff), (0x4c, 0x3d)]] + c = [hex (int ((3 * p - 1) * b + (2 - 3 * p) * a))[2:] + for (a, b) in [(0xff, 0xff), (0xa6, 0xff), (0x4c, 0x3d)]] else: - c = [hex (int ((3 * p - 2) * b + 3 * (1 - p) * a))[2:] for (a, b) in [(0xff, 0x1f), (0xff, 0xff), (0x3d, 0x1f)]] + c = [hex (int ((3 * p - 2) * b + 3 * (1 - p) * a))[2:] + for (a, b) in [(0xff, 0x1f), (0xff, 0xff), (0x3d, 0x1f)]] return ''.join (c) + +def update_word_count (text, filename, word_count): + return re.sub (r'(?m)^(\d+) *' + filename, + str (word_count).ljust (6) + filename, + text) + +po_msgid_re = re.compile (r'^msgid "(.*?)"(?:\n"(.*?)")*', re.M) + +def po_word_count (po_content): + s = ' '.join ([''.join (t) for t in po_msgid_re.findall (po_content)]) + return len (space_re.split (s)) + +sgml_tag_re = re.compile (r'<.*?>', re.S) + +def sgml_word_count (sgml_doc): + s = sgml_tag_re.sub ('', sgml_doc) + return len (space_re.split (s)) + def tely_word_count (tely_doc): ''' Calculate word count of a Texinfo document node by node. @@ -174,7 +188,12 @@ class TelyDocument (object): self.title = 'Untitled' self.level = ('u', 1) - included_files = [os.path.join (os.path.dirname (filename), t) for t in include_re.findall (self.contents)] + m = language_re.search (self.contents) + if m: + self.language = m.group (1) + + included_files = [os.path.join (os.path.dirname (filename), t) + for t in include_re.findall (self.contents)] self.included_files = [p for p in included_files if os.path.exists (p)] def print_title (self, section_number): @@ -186,6 +205,14 @@ class TranslatedTelyDocument (TelyDocument): TelyDocument.__init__ (self, filename) self.masterdocument = masterdocument + if not hasattr (self, 'language') \ + and hasattr (parent_translation, 'language'): + self.language = parent_translation.language + if hasattr (self, 'language'): + self.translation = translation[self.language] + else: + self.translation = lambda x: x + self.title = self.translation (self.title) ## record authoring information m = translators_re.search (self.contents) @@ -215,40 +242,47 @@ class TranslatedTelyDocument (TelyDocument): ## calculate translation percentage master_total_word_count = sum (masterdocument.word_count) - translation_word_count = sum ([masterdocument.word_count[k] * self.translated_nodes[k] - for k in range (min (len (masterdocument.word_count), len (self.translated_nodes)))]) - self.translation_percentage = 100 * translation_word_count / master_total_word_count + translation_word_count = \ + sum ([masterdocument.word_count[k] * self.translated_nodes[k] + for k in range (min (len (masterdocument.word_count), + len (self.translated_nodes)))]) + self.translation_percentage = \ + 100 * translation_word_count / master_total_word_count ## calculate how much the file is outdated - m = committish_re.search (self.contents) - if not m: - sys.stderr.write ('error: ' + filename + \ - ": no 'GIT committish: ' found.\nPlease check " + \ - 'the whole file against the original in English, then ' + \ - 'fill in HEAD committish in the header.\n') - sys.exit (1) - (diff_string, error) = read_pipe (diff_cmd % {'committish':m.group (1), 'original':masterdocument.filename}) + (diff_string, error) = \ + buildlib.check_translated_doc (masterdocument.filename, self.contents) if error: sys.stderr.write ('warning: %s: %s' % (self.filename, error)) self.uptodate_percentage = None else: diff = diff_string.splitlines () - insertions = sum ([len (l) - 1 for l in diff if l.startswith ('+') and not l.startswith ('+++')]) - deletions = sum ([len (l) - 1 for l in diff if l.startswith ('-') and not l.startswith ('---')]) - outdateness_percentage = 50.0 * (deletions + insertions) / (masterdocument.size + 0.5 * (deletions - insertions)) + insertions = sum ([len (l) - 1 for l in diff + if l.startswith ('+') + and not l.startswith ('+++')]) + deletions = sum ([len (l) - 1 for l in diff + if l.startswith ('-') + and not l.startswith ('---')]) + outdateness_percentage = 50.0 * (deletions + insertions) / \ + (masterdocument.size + 0.5 * (deletions - insertions)) self.uptodate_percentage = 100 - int (outdateness_percentage) if self.uptodate_percentage > 100: alternative = 50 - progress ("%s: strange uptodateness percentage %d %%, setting to %d %%" \ - % (self.filename, self.uptodate_percentage, alternative)) + progress ("%s: strange uptodateness percentage %d %%, \ +setting to %d %%" % (self.filename, self.uptodate_percentage, alternative)) self.uptodate_percentage = alternative elif self.uptodate_percentage < 1: alternative = 1 - progress ("%s: strange uptodateness percentage %d %%, setting to %d %%" \ - % (self.filename, self.uptodate_percentage, alternative)) + progress ("%s: strange uptodateness percentage %d %%, \ +setting to %d %%" % (self.filename, self.uptodate_percentage, alternative)) self.uptodate_percentage = alternative - def completeness (self, formats=['long']): + def completeness (self, formats=['long'], translated=False): + if translated: + translation = self.translation + else: + translation = lambda x: x + if isinstance (formats, str): formats = [formats] p = self.translation_percentage @@ -258,9 +292,15 @@ class TranslatedTelyDocument (TelyDocument): status = 'fully translated' else: status = 'partially translated' - return dict ([(f, format_table[status][f] % locals()) for f in formats]) + return dict ([(f, translation (format_table[status][f]) % locals()) + for f in formats]) + + def uptodateness (self, formats=['long'], translated=False): + if translated: + translation = self.translation + else: + translation = lambda x: x - def uptodateness (self, formats=['long']): if isinstance (formats, str): formats = [formats] p = self.uptodate_percentage @@ -275,28 +315,31 @@ class TranslatedTelyDocument (TelyDocument): if f == 'color' and p != None: l['color'] = percentage_color (p) else: - l[f] = format_table[status][f] % locals () + l[f] = translation (format_table[status][f]) % locals () return l - def gdp_status (self, translation=lambda s: s): + def gdp_status (self): if self.post_gdp: - return translation (format-table['post-GDP']) + return self.translation (format_table['post-GDP']) else: - return translation (format-table['pre-GDP']) + return self.translation (format_table['pre-GDP']) def short_html_status (self): s = ' ' if self.partially_translated: s += '
\n '.join (self.translators) + '
\n' if self.checkers: - s += ' ' + '
\n '.join (self.checkers) + '
\n' + s += ' ' + \ + '
\n '.join (self.checkers) + '
\n' c = self.completeness (['color', 'long']) - s += ' %(long)s
\n' % c + s += ' \ +%(long)s
\n' % c if self.partially_translated: u = self.uptodateness (['vague', 'color']) - s += ' %(vague)s
\n' % u + s += ' \ +%(vague)s
\n' % u s += ' \n' return s @@ -308,26 +351,81 @@ class TranslatedTelyDocument (TelyDocument): s += self.uptodateness ('abbr')['abbr'] + ' ' return s - def html_status (self): - # TODO - return '' + def html_status (self, numbering=SectionNumber ()): + if self.title == 'Untitled': + return '' + + if self.level[1] == 0: # if self is a master document + s = ''' + + ''' % self.print_title (numbering) + s += ''.join ([' \n' % self.translation (h) + for h in detailed_status_heads]) + s += ' \n' + s += ' \n \n' \ + % (self.translation (section_titles_string), + sum (self.masterdocument.word_count)) + + else: + s = ' \n \n' \ + % (self.print_title (numbering), + sum (self.masterdocument.word_count)) + + if self.partially_translated: + s += ' \n' + s += ' \n' + else: + s += ' \n' * 2 + + c = self.completeness (['color', 'short'], translated=True) + s += ' \n' % {'color': c['color'], + 'short': c['short']} + + if self.partially_translated: + u = self.uptodateness (['short', 'color'], translated=True) + s += ' \n' % {'color': u['color'], + 'short': u['short']} + else: + s += ' \n' + + s += ' \n \n' + s += ''.join ([i.translations[self.language].html_status (numbering) + for i in self.masterdocument.includes + if self.language in i.translations]) + + if self.level[1] == 0: # if self is a master document + s += '
%s%s
%s
(%d)
%s
(%d)		' + '
\n '.join (self.translators) + '		' + '
\n '.join (self.checkers) + '		\ +%(short)s\ +%(short)s' + self.gdp_status () + '
\n

\n' + return s class MasterTelyDocument (TelyDocument): - def __init__ (self, filename, parent_translations=dict ([(lang, None) for lang in langdefs.LANGDICT.keys()])): + def __init__ (self, + filename, + parent_translations=dict ([(lang, None) + for lang in langdefs.LANGDICT])): TelyDocument.__init__ (self, filename) self.size = len (self.contents) self.word_count = tely_word_count (self.contents) - translations = dict ([(lang, os.path.join (lang, filename)) for lang in langdefs.LANGDICT.keys()]) - self.translations = dict ([(lang, TranslatedTelyDocument (translations[lang], self, parent_translations.get (lang))) - for lang in langdefs.LANGDICT.keys() if os.path.exists (translations[lang])]) + translations = dict ([(lang, os.path.join (lang, filename)) + for lang in langdefs.LANGDICT]) + self.translations = \ + dict ([(lang, + TranslatedTelyDocument (translations[lang], + self, parent_translations.get (lang))) + for lang in langdefs.LANGDICT + if os.path.exists (translations[lang])]) if self.translations: - self.includes = [MasterTelyDocument (f, self.translations) for f in self.included_files] + self.includes = [MasterTelyDocument (f, self.translations) + for f in self.included_files] else: self.includes = [] - # TODO - def print_wc_priority (self): - return + def update_word_counts (self, s): + s = update_word_count (s, self.filename, sum (self.word_count)) + for i in self.includes: + s = i.update_word_counts (s) + return s def html_status (self, numbering=SectionNumber ()): if self.title == 'Untitled' or not self.translations: @@ -336,40 +434,44 @@ class MasterTelyDocument (TelyDocument): s = '''''' % self.print_title (numbering) - s += ''.join ([' \n' % l for l in self.translations.keys ()]) + s += ''.join ([' \n' % l for l in self.translations]) s += ' \n' s += ' \n \n' \ % sum (self.word_count) - else: + else: # if self is an included file s = ' \n \n' \ % (self.print_title (numbering), sum (self.word_count)) - s += ''.join ([t.short_html_status () for t in self.translations.values ()]) + s += ''.join ([t.short_html_status () + for t in self.translations.values ()]) s += ' \n' s += ''.join ([i.html_status (numbering) for i in self.includes]) - if self.level[1] == 0: + if self.level[1] == 0: # if self is a master document s += '
%s%s%s
Section titles
(%d)
%s
(%d)
\n

\n' return s - def text_status (self, numbering=SectionNumber (), colspec=[45,14]): + def text_status (self, numbering=SectionNumber (), colspec=[48,12]): if self.title == 'Untitled' or not self.translations: return '' s = '' if self.level[1] == 0: # if self is a master document s += (self.print_title (numbering) + ' ').ljust (colspec[0]) - s += ''.join (['%s'.ljust (colspec[1]) % l for l in self.translations.keys ()]) + s += ''.join (['%s'.ljust (colspec[1]) % l + for l in self.translations]) s += '\n' - s += ('Section titles (%d)' % sum (self.word_count)).ljust (colspec[0]) + s += ('Section titles (%d)' % \ + sum (self.word_count)).ljust (colspec[0]) else: s = '%s (%d) ' \ % (self.print_title (numbering), sum (self.word_count)) s = s.ljust (colspec[0]) - s += ''.join ([t.text_status ().ljust(colspec[1]) for t in self.translations.values ()]) + s += ''.join ([t.text_status ().ljust(colspec[1]) + for t in self.translations.values ()]) s += '\n\n' s += ''.join ([i.text_status (numbering) for i in self.includes]) @@ -378,37 +480,67 @@ class MasterTelyDocument (TelyDocument): return s +update_category_word_counts_re = re.compile (r'(?ms)^-(\d+)-(.*?\n)\d+ *total') + +counts_re = re.compile (r'(?m)^(\d+) ') + +def update_category_word_counts_sub (m): + return '-' + m.group (1) + '-' + m.group (2) + \ + str (sum ([int (c) + for c in counts_re.findall (m.group (2))])).ljust (6) + \ + 'total' + + progress ("Reading documents...") -tely_files = read_pipe ("find -maxdepth 2 -name '*.tely'")[0].splitlines () -master_docs = [MasterTelyDocument (os.path.normpath (filename)) for filename in tely_files] +tely_files = \ + buildlib.read_pipe ("find -maxdepth 2 -name '*.tely'")[0].splitlines () +tely_files.sort () +master_docs = [MasterTelyDocument (os.path.normpath (filename)) + for filename in tely_files] master_docs = [doc for doc in master_docs if doc.translations] main_status_page = open ('translations.template.html.in').read () -## TODO -#per_lang_status_pages = dict ([(l, open (os.path.join (l, 'translations.template.html')). read ()) -# for l in langdefs.LANGDICT.keys () -# if langdefs.LANGDICT[l].enabled]) +enabled_languages = [l for l in langdefs.LANGDICT + if langdefs.LANGDICT[l].enabled + and l != 'en'] +lang_status_pages = \ + dict ([(l, open (os.path.join (l, 'translations.template.html.in')). read ()) + for l in enabled_languages]) progress ("Generating status pages...") -date_time = read_pipe ('LANG= date -u')[0] +date_time = buildlib.read_pipe ('LANG= date -u')[0] -main_status_html = '

Last updated %s

\n' % date_time +main_status_html = last_updated_string % date_time main_status_html += '\n'.join ([doc.html_status () for doc in master_docs]) html_re = re.compile ('', re.I) end_body_re = re.compile ('', re.I) -main_status_page = html_re.sub (''' +html_header = ''' ''', main_status_page) +translations.template.html.in; DO NOT EDIT !-->''' + +main_status_page = html_re.sub (html_header, main_status_page) -main_status_page = end_body_re.sub (main_status_html + '\n', main_status_page) +main_status_page = end_body_re.sub (main_status_html + '\n', + main_status_page) open ('translations.html.in', 'w').write (main_status_page) +for l in enabled_languages: + date_time = buildlib.read_pipe ('LANG=%s date -u' % l)[0] + lang_status_pages[l] = translation[l] (last_updated_string) % date_time + lang_status_pages[l] + lang_status_page = html_re.sub (html_header, lang_status_pages[l]) + html_status = '\n'.join ([doc.translations[l].html_status () + for doc in master_docs + if l in doc.translations]) + lang_status_page = end_body_re.sub (html_status + '\n', + lang_status_page) + open (os.path.join (l, 'translations.html.in'), 'w').write (lang_status_page) + main_status_txt = '''Documentation translations status Generated %s NT = not translated @@ -421,3 +553,30 @@ main_status_txt += '\n'.join ([doc.text_status () for doc in master_docs]) status_txt_file = 'out/translations-status.txt' progress ("Writing %s..." % status_txt_file) open (status_txt_file, 'w').write (main_status_txt) + +translation_instructions_file = 'TRANSLATION' +progress ("Updating %s..." % translation_instructions_file) +translation_instructions = open (translation_instructions_file).read () + +for doc in master_docs: + translation_instructions = doc.update_word_counts (translation_instructions) + +for html_file in re.findall (r'(?m)^\d+ *(\S+?\.html\S*?)(?: |$)', + translation_instructions): + word_count = sgml_word_count (open (html_file).read ()) + translation_instructions = update_word_count (translation_instructions, + html_file, + word_count) + +for po_file in re.findall (r'(?m)^\d+ *(\S+?\.po\S*?)(?: |$)', + translation_instructions): + word_count = po_word_count (open (po_file).read ()) + translation_instructions = update_word_count (translation_instructions, + po_file, + word_count) + +translation_instructions = \ + update_category_word_counts_re.sub (update_category_word_counts_sub, + translation_instructions) + +open (translation_instructions_file, 'w').write (translation_instructions)