-#!/usr/bin/env python
-
-"""
-USAGE: translations-status.py BUILDSCRIPT-DIR LOCALEDIR
-
- This script must be run from Documentation/
-
- Reads template files translations.template.html.in
-and for each LANG in LANGUAGES LANG/translations.template.html.in
- Writes translations.html.in and for each LANG in LANGUAGES
-translations.LANG.html.in
- Writes out/translations-status.txt
- Updates word counts in TRANSLATION
-"""
-
-import sys
-import re
-import string
-import os
-
-import langdefs
-import buildlib
-
-def progress (str):
- sys.stderr.write (str + '\n')
-
-progress ("translations-status.py")
-
-_doc = lambda s: s
-
-# load gettext messages catalogs
-translation = langdefs.translation
-
-
-language_re = re.compile (r'^@documentlanguage (.+)', re.M)
-comments_re = re.compile (r'^@ignore\n(.|\n)*?\n@end ignore$|@c .*?$', re.M)
-space_re = re.compile (r'\s+', re.M)
-lilypond_re = re.compile (r'@lilypond({.*?}|(.|\n)*?\n@end lilypond$)', re.M)
-node_re = re.compile ('^@node .*?$', re.M)
-title_re = re.compile ('^@(top|chapter|(?:sub){0,2}section|' + \
-'(?:unnumbered|appendix)(?:(?:sub){0,2}sec)?) (.*?)$', re.M)
-include_re = re.compile ('^@include (.*?)$', re.M)
-
-translators_re = re.compile (r'^@c\s+Translators\s*:\s*(.*?)$', re.M | re.I)
-checkers_re = re.compile (r'^@c\s+Translation\s*checkers\s*:\s*(.*?)$',
- re.M | re.I)
-status_re = re.compile (r'^@c\s+Translation\s*status\s*:\s*(.*?)$', re.M | re.I)
-post_gdp_re = re.compile ('post.GDP', re.I)
-untranslated_node_str = '@untranslated'
-skeleton_str = '-- SKELETON FILE --'
-
-section_titles_string = _doc ('Section titles')
-last_updated_string = _doc (' <p><i>Last updated %s</i></p>\n')
-detailed_status_heads = [_doc ('Translators'), _doc ('Translation checkers'),
- _doc ('Translated'), _doc ('Up to date'),
- _doc ('Other info')]
-format_table = {
- 'not translated': {'color':'d0f0f8', 'short':_doc ('no'), 'abbr':'NT',
- 'long':_doc ('not translated')},
- 'partially translated': {'color':'dfef77',
- 'short':_doc ('partially (%(p)d %%)'),
- 'abbr':'%(p)d%%',
- 'long':_doc ('partially translated (%(p)d %%)')},
- 'fully translated': {'color':'1fff1f', 'short':_doc ('yes'), 'abbr':'FT',
- 'long': _doc ('translated')},
- 'up to date': {'short':_doc ('yes'), 'long':_doc ('up to date'),
- 'abbr':'100%%', 'vague':_doc ('up to date')},
- 'outdated': {'short':_doc ('partially'), 'abbr':'%(p)d%%',
- 'vague':_doc ('partially up to date')},
- 'N/A': {'short':_doc ('N/A'), 'abbr':'N/A', 'color':'d587ff', 'vague':''},
- 'pre-GDP':_doc ('pre-GDP'),
- 'post-GDP':_doc ('post-GDP')
-}
-
-texi_level = {
-# (Unumbered/Numbered/Lettered, level)
- 'top': ('u', 0),
- 'unnumbered': ('u', 1),
- 'unnumberedsec': ('u', 2),
- 'unnumberedsubsec': ('u', 3),
- 'chapter': ('n', 1),
- 'section': ('n', 2),
- 'subsection': ('n', 3),
- 'appendix': ('l', 1)
-}
-
-appendix_number_trans = string.maketrans ('@ABCDEFGHIJKLMNOPQRSTUVWXY',
- 'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
-
-class SectionNumber (object):
- def __init__ (self):
- self.__data = [[0,'u']]
-
- def __increase_last_index (self):
- type = self.__data[-1][1]
- if type == 'l':
- self.__data[-1][0] = \
- self.__data[-1][0].translate (appendix_number_trans)
- elif type == 'n':
- self.__data[-1][0] += 1
-
- def format (self):
- if self.__data[-1][1] == 'u':
- return ''
- return '.'.join ([str (i[0]) for i in self.__data if i[1] != 'u']) + ' '
-
- def increase (self, (type, level)):
- if level == 0:
- self.__data = [[0,'u']]
- while level + 1 < len (self.__data):
- del self.__data[-1]
- if level + 1 > len (self.__data):
- self.__data.append ([0, type])
- if type == 'l':
- self.__data[-1][0] = '@'
- if type == self.__data[-1][1]:
- self.__increase_last_index ()
- else:
- self.__data[-1] = ([0, type])
- if type == 'l':
- self.__data[-1][0] = 'A'
- elif type == 'n':
- self.__data[-1][0] = 1
- return self.format ()
-
-
-def percentage_color (percent):
- p = percent / 100.0
- if p < 0.33:
- c = [hex (int (3 * p * b + (1 - 3 * p) * a))[2:]
- for (a, b) in [(0xff, 0xff), (0x5c, 0xa6), (0x5c, 0x4c)]]
- elif p < 0.67:
- c = [hex (int ((3 * p - 1) * b + (2 - 3 * p) * a))[2:]
- for (a, b) in [(0xff, 0xff), (0xa6, 0xff), (0x4c, 0x3d)]]
- else:
- c = [hex (int ((3 * p - 2) * b + 3 * (1 - p) * a))[2:]
- for (a, b) in [(0xff, 0x1f), (0xff, 0xff), (0x3d, 0x1f)]]
- return ''.join (c)
-
-
-def update_word_count (text, filename, word_count):
- return re.sub (r'(?m)^(\d+) *' + filename,
- str (word_count).ljust (6) + filename,
- text)
-
-po_msgid_re = re.compile (r'^msgid "(.*?)"(?:\n"(.*?)")*', re.M)
-
-def po_word_count (po_content):
- s = ' '.join ([''.join (t) for t in po_msgid_re.findall (po_content)])
- return len (space_re.split (s))
-
-sgml_tag_re = re.compile (r'<.*?>', re.S)
-
-def sgml_word_count (sgml_doc):
- s = sgml_tag_re.sub ('', sgml_doc)
- return len (space_re.split (s))
-
-def tely_word_count (tely_doc):
- '''
- Calculate word count of a Texinfo document node by node.
-
- Take string tely_doc as an argument.
- Return a list of integers.
-
- Texinfo comments and @lilypond blocks are not included in word counts.
- '''
- tely_doc = comments_re.sub ('', tely_doc)
- tely_doc = lilypond_re.sub ('', tely_doc)
- nodes = node_re.split (tely_doc)
- return [len (space_re.split (n)) for n in nodes]
-
-
-class TelyDocument (object):
- def __init__ (self, filename):
- self.filename = filename
- self.contents = open (filename).read ()
-
- ## record title and sectionning level of first Texinfo section
- m = title_re.search (self.contents)
- if m:
- self.title = m.group (2)
- self.level = texi_level [m.group (1)]
- else:
- self.title = 'Untitled'
- self.level = ('u', 1)
-
- m = language_re.search (self.contents)
- if m:
- self.language = m.group (1)
-
- included_files = [os.path.join (os.path.dirname (filename), t)
- for t in include_re.findall (self.contents)]
- self.included_files = [p for p in included_files if os.path.exists (p)]
-
- def print_title (self, section_number):
- return section_number.increase (self.level) + self.title
-
-
-class TranslatedTelyDocument (TelyDocument):
- def __init__ (self, filename, masterdocument, parent_translation=None):
- TelyDocument.__init__ (self, filename)
-
- self.masterdocument = masterdocument
- if not hasattr (self, 'language') \
- and hasattr (parent_translation, 'language'):
- self.language = parent_translation.language
- if hasattr (self, 'language'):
- self.translation = translation[self.language]
- else:
- self.translation = lambda x: x
- self.title = self.translation (self.title)
-
- ## record authoring information
- m = translators_re.search (self.contents)
- if m:
- self.translators = [n.strip () for n in m.group (1).split (',')]
- else:
- self.translators = parent_translation.translators
- m = checkers_re.search (self.contents)
- if m:
- self.checkers = [n.strip () for n in m.group (1).split (',')]
- elif isinstance (parent_translation, TranslatedTelyDocument):
- self.checkers = parent_translation.checkers
- else:
- self.checkers = []
-
- ## check whether translation is pre- or post-GDP
- m = status_re.search (self.contents)
- if m:
- self.post_gdp = bool (post_gdp_re.search (m.group (1)))
- else:
- self.post_gdp = False
-
- ## record which parts (nodes) of the file are actually translated
- self.partially_translated = not skeleton_str in self.contents
- nodes = node_re.split (self.contents)
- self.translated_nodes = [not untranslated_node_str in n for n in nodes]
-
- ## calculate translation percentage
- master_total_word_count = sum (masterdocument.word_count)
- translation_word_count = \
- sum ([masterdocument.word_count[k] * self.translated_nodes[k]
- for k in range (min (len (masterdocument.word_count),
- len (self.translated_nodes)))])
- self.translation_percentage = \
- 100 * translation_word_count / master_total_word_count
-
- ## calculate how much the file is outdated
- (diff_string, error) = \
- buildlib.check_translated_doc (masterdocument.filename, self.filename, self.contents)
- if error:
- sys.stderr.write ('warning: %s: %s' % (self.filename, error))
- self.uptodate_percentage = None
- else:
- diff = diff_string.splitlines ()
- insertions = sum ([len (l) - 1 for l in diff
- if l.startswith ('+')
- and not l.startswith ('+++')])
- deletions = sum ([len (l) - 1 for l in diff
- if l.startswith ('-')
- and not l.startswith ('---')])
- outdateness_percentage = 50.0 * (deletions + insertions) / \
- (masterdocument.size + 0.5 * (deletions - insertions))
- self.uptodate_percentage = 100 - int (outdateness_percentage)
- if self.uptodate_percentage > 100:
- alternative = 50
- progress ("%s: strange uptodateness percentage %d %%, \
-setting to %d %%" % (self.filename, self.uptodate_percentage, alternative))
- self.uptodate_percentage = alternative
- elif self.uptodate_percentage < 1:
- alternative = 1
- progress ("%s: strange uptodateness percentage %d %%, \
-setting to %d %%" % (self.filename, self.uptodate_percentage, alternative))
- self.uptodate_percentage = alternative
-
- def completeness (self, formats=['long'], translated=False):
- if translated:
- translation = self.translation
- else:
- translation = lambda x: x
-
- if isinstance (formats, str):
- formats = [formats]
- p = self.translation_percentage
- if p == 0:
- status = 'not translated'
- elif p == 100:
- status = 'fully translated'
- else:
- status = 'partially translated'
- return dict ([(f, translation (format_table[status][f]) % locals())
- for f in formats])
-
- def uptodateness (self, formats=['long'], translated=False):
- if translated:
- translation = self.translation
- else:
- translation = lambda x: x
-
- if isinstance (formats, str):
- formats = [formats]
- p = self.uptodate_percentage
- if p == None:
- status = 'N/A'
- elif p == 100:
- status = 'up to date'
- else:
- status = 'outdated'
- l = {}
- for f in formats:
- if f == 'color' and p != None:
- l['color'] = percentage_color (p)
- else:
- l[f] = translation (format_table[status][f]) % locals ()
- return l
-
- def gdp_status (self):
- if self.post_gdp:
- return self.translation (format_table['post-GDP'])
- else:
- return self.translation (format_table['pre-GDP'])
-
- def short_html_status (self):
- s = ' <td>'
- if self.partially_translated:
- s += '<br>\n '.join (self.translators) + '<br>\n'
- if self.checkers:
- s += ' <small>' + \
- '<br>\n '.join (self.checkers) + '</small><br>\n'
-
- c = self.completeness (['color', 'long'])
- s += ' <span style="background-color: #%(color)s">\
-%(long)s</span><br>\n' % c
-
- if self.partially_translated:
- u = self.uptodateness (['vague', 'color'])
- s += ' <span style="background-color: #%(color)s">\
-%(vague)s</span><br>\n' % u
-
- s += ' </td>\n'
- return s
-
- def text_status (self):
- s = self.completeness ('abbr')['abbr'] + ' '
-
- if self.partially_translated:
- s += self.uptodateness ('abbr')['abbr'] + ' '
- return s
-
- def html_status (self, numbering=SectionNumber ()):
- if self.title == 'Untitled':
- return ''
-
- if self.level[1] == 0: # if self is a master document
- s = '''<table align="center" border="2">
- <tr align="center">
- <th>%s</th>''' % self.print_title (numbering)
- s += ''.join ([' <th>%s</th>\n' % self.translation (h)
- for h in detailed_status_heads])
- s += ' </tr>\n'
- s += ' <tr align="left">\n <td>%s<br>(%d)</td>\n' \
- % (self.translation (section_titles_string),
- sum (self.masterdocument.word_count))
-
- else:
- s = ' <tr align="left">\n <td>%s<br>(%d)</td>\n' \
- % (self.print_title (numbering),
- sum (self.masterdocument.word_count))
-
- if self.partially_translated:
- s += ' <td>' + '<br>\n '.join (self.translators) + '</td>\n'
- s += ' <td>' + '<br>\n '.join (self.checkers) + '</td>\n'
- else:
- s += ' <td></td>\n' * 2
-
- c = self.completeness (['color', 'short'], translated=True)
- s += ' <td><span style="background-color: #%(color)s">\
-%(short)s</span></td>\n' % {'color': c['color'],
- 'short': c['short']}
-
- if self.partially_translated:
- u = self.uptodateness (['short', 'color'], translated=True)
- s += ' <td><span style="background-color: #%(color)s">\
-%(short)s</span></td>\n' % {'color': u['color'],
- 'short': u['short']}
- else:
- s += ' <td></td>\n'
-
- s += ' <td>' + self.gdp_status () + '</td>\n </tr>\n'
- s += ''.join ([i.translations[self.language].html_status (numbering)
- for i in self.masterdocument.includes
- if self.language in i.translations])
-
- if self.level[1] == 0: # if self is a master document
- s += '</table>\n<p></p>\n'
- return s
-
-class MasterTelyDocument (TelyDocument):
- def __init__ (self,
- filename,
- parent_translations=dict ([(lang, None)
- for lang in langdefs.LANGDICT])):
- TelyDocument.__init__ (self, filename)
- self.size = len (self.contents)
- self.word_count = tely_word_count (self.contents)
- translations = dict ([(lang, os.path.join (lang, filename))
- for lang in langdefs.LANGDICT])
- self.translations = \
- dict ([(lang,
- TranslatedTelyDocument (translations[lang],
- self, parent_translations.get (lang)))
- for lang in langdefs.LANGDICT
- if os.path.exists (translations[lang])])
- if self.translations:
- self.includes = [MasterTelyDocument (f, self.translations)
- for f in self.included_files]
- else:
- self.includes = []
-
- def update_word_counts (self, s):
- s = update_word_count (s, self.filename, sum (self.word_count))
- for i in self.includes:
- s = i.update_word_counts (s)
- return s
-
- def html_status (self, numbering=SectionNumber ()):
- if self.title == 'Untitled' or not self.translations:
- return ''
- if self.level[1] == 0: # if self is a master document
- s = '''<table align="center" border="2">
- <tr align="center">
- <th>%s</th>''' % self.print_title (numbering)
- s += ''.join ([' <th>%s</th>\n' % l for l in self.translations])
- s += ' </tr>\n'
- s += ' <tr align="left">\n <td>Section titles<br>(%d)</td>\n' \
- % sum (self.word_count)
-
- else: # if self is an included file
- s = ' <tr align="left">\n <td>%s<br>(%d)</td>\n' \
- % (self.print_title (numbering), sum (self.word_count))
-
- s += ''.join ([t.short_html_status ()
- for t in self.translations.values ()])
- s += ' </tr>\n'
- s += ''.join ([i.html_status (numbering) for i in self.includes])
-
- if self.level[1] == 0: # if self is a master document
- s += '</table>\n<p></p>\n'
- return s
-
- def text_status (self, numbering=SectionNumber (), colspec=[48,12]):
- if self.title == 'Untitled' or not self.translations:
- return ''
-
- s = ''
- if self.level[1] == 0: # if self is a master document
- s += (self.print_title (numbering) + ' ').ljust (colspec[0])
- s += ''.join (['%s'.ljust (colspec[1]) % l
- for l in self.translations])
- s += '\n'
- s += ('Section titles (%d)' % \
- sum (self.word_count)).ljust (colspec[0])
-
- else:
- s = '%s (%d) ' \
- % (self.print_title (numbering), sum (self.word_count))
- s = s.ljust (colspec[0])
-
- s += ''.join ([t.text_status ().ljust(colspec[1])
- for t in self.translations.values ()])
- s += '\n\n'
- s += ''.join ([i.text_status (numbering) for i in self.includes])
-
- if self.level[1] == 0:
- s += '\n'
- return s
-
-
-update_category_word_counts_re = re.compile (r'(?ms)^-(\d+)-(.*?\n)\d+ *total')
-
-counts_re = re.compile (r'(?m)^(\d+) ')
-
-def update_category_word_counts_sub (m):
- return '-' + m.group (1) + '-' + m.group (2) + \
- str (sum ([int (c)
- for c in counts_re.findall (m.group (2))])).ljust (6) + \
- 'total'
-
-
-progress ("Reading documents...")
-
-tely_files = \
- buildlib.read_pipe ("find -maxdepth 2 -name '*.tely'")[0].splitlines ()
-tely_files.sort ()
-master_docs = [MasterTelyDocument (os.path.normpath (filename))
- for filename in tely_files]
-master_docs = [doc for doc in master_docs if doc.translations]
-
-main_status_page = open ('translations.template.html.in').read ()
-
-enabled_languages = [l for l in langdefs.LANGDICT
- if langdefs.LANGDICT[l].enabled
- and l != 'en']
-lang_status_pages = \
- dict ([(l, open (os.path.join (l, 'translations.template.html.in')). read ())
- for l in enabled_languages])
-
-progress ("Generating status pages...")
-
-date_time = buildlib.read_pipe ('LANG= date -u')[0]
-
-main_status_html = last_updated_string % date_time
-main_status_html += '\n'.join ([doc.html_status () for doc in master_docs])
-
-html_re = re.compile ('<html>', re.I)
-end_body_re = re.compile ('</body>', re.I)
-
-html_header = '''<html>
-<!-- This page is automatically generated by translation-status.py from
-translations.template.html.in; DO NOT EDIT !-->'''
-
-main_status_page = html_re.sub (html_header, main_status_page)
-
-main_status_page = end_body_re.sub (main_status_html + '\n</body>',
- main_status_page)
-
-open ('translations.html.in', 'w').write (main_status_page)
-
-for l in enabled_languages:
- date_time = buildlib.read_pipe ('LANG=%s date -u' % l)[0]
- lang_status_pages[l] = translation[l] (last_updated_string) % date_time + lang_status_pages[l]
- lang_status_page = html_re.sub (html_header, lang_status_pages[l])
- html_status = '\n'.join ([doc.translations[l].html_status ()
- for doc in master_docs
- if l in doc.translations])
- lang_status_page = end_body_re.sub (html_status + '\n</body>',
- lang_status_page)
- open (os.path.join (l, 'translations.html.in'), 'w').write (lang_status_page)
-
-main_status_txt = '''Documentation translations status
-Generated %s
-NT = not translated
-FT = fully translated
-
-''' % date_time
-
-main_status_txt += '\n'.join ([doc.text_status () for doc in master_docs])
-
-status_txt_file = 'out/translations-status.txt'
-progress ("Writing %s..." % status_txt_file)
-open (status_txt_file, 'w').write (main_status_txt)
-
-translation_instructions_file = 'TRANSLATION'
-progress ("Updating %s..." % translation_instructions_file)
-translation_instructions = open (translation_instructions_file).read ()
-
-for doc in master_docs:
- translation_instructions = doc.update_word_counts (translation_instructions)
-
-for html_file in re.findall (r'(?m)^\d+ *(\S+?\.html\S*?)(?: |$)',
- translation_instructions):
- word_count = sgml_word_count (open (html_file).read ())
- translation_instructions = update_word_count (translation_instructions,
- html_file,
- word_count)
-
-for po_file in re.findall (r'(?m)^\d+ *(\S+?\.po\S*?)(?: |$)',
- translation_instructions):
- word_count = po_word_count (open (po_file).read ())
- translation_instructions = update_word_count (translation_instructions,
- po_file,
- word_count)
-
-translation_instructions = \
- update_category_word_counts_re.sub (update_category_word_counts_sub,
- translation_instructions)
-
-open (translation_instructions_file, 'w').write (translation_instructions)