Improvements to the i18n workflow tools.

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -405,7 +405,7 @@ class @Problem
     formulaequationinput: (element) ->
       $(element).find('input').on 'input', ->
         $p = $(element).find('p.status')
-        `// Translators: the word Answer here is about answering a problem the student must solve.`
+        `// Translators: the word unanswered here is about answering a problem the student must solve.`
         $p.text gettext("unanswered")
         $p.parent().removeClass().addClass "unanswered"
 
@@ -434,7 +434,7 @@ class @Problem
     textline: (element) ->
       $(element).find('input').on 'input', ->
         $p = $(element).find('p.status')
-        `// Translators: the word Answer here is about answering a problem the student must solve.`
+        `// Translators: the word unanswered here is about answering a problem the student must solve.`
         $p.text gettext("unanswered")
         $p.parent().removeClass().addClass "unanswered"
 

i18n/converter.py

@@ -21,9 +21,9 @@ class Converter(object):
     #   HTML:   <B>, </B>, <BR/>, <textformat leading="10">
     #   Python: %(date)s, %(name)s
     tag_pattern = re.compile(r'''
-        (<[-\w" .:?=/]*>)   |       # <tag>
-        ({[^}]*})           |       # {tag}
-        (%\([^)]*\)\w)      |       # %(tag)s
+        (<[^>]+>)           |       # <tag>
+        ({[^}]+})           |       # {tag}
+        (%\([\w]+\)\w)      |       # %(tag)s
         (&\w+;)             |       # &entity;
         (&\#\d+;)           |       # &#1234;
         (&\#x[0-9a-f]+;)            # &#xABCD;

i18n/dummy.py

@@ -1,56 +1,70 @@
-from converter import Converter
+# -*- coding: utf-8 -*-
+r"""
+Creates new localization properties files in a dummy language.
+
+Each property file is derived from the equivalent en_US file, with these
+transformations applied:
+
+1. Every vowel is replaced with an equivalent with extra accent marks.
 
-# Creates new localization properties files in a dummy language
-# Each property file is derived from the equivalent en_US file, except
-# 1. Every vowel is replaced with an equivalent with extra accent marks
-# 2. Every string is padded out to +30% length to simulate verbose languages (e.g. German)
-#    to see if layout and flows work properly
-# 3. Every string is terminated with a '#' character to make it easier to detect truncation
+2. Every string is padded out to +30% length to simulate verbose languages
+   (such as German) to see if layout and flows work properly.
 
+3. Every string is terminated with a '#' character to make it easier to detect
+   truncation.
 
-# --------------------------------
-# Example use:
-# >>> from dummy import Dummy
-# >>> c = Dummy()
-# >>> c.convert("hello my name is Bond, James Bond")
-# u'h\xe9ll\xf6 my n\xe4m\xe9 \xefs B\xf6nd, J\xe4m\xe9s B\xf6nd Lorem i#'
-#
-# >>> c.convert('don\'t convert <a href="href">tag ids</a>')
-# u'd\xf6n\'t \xe7\xf6nv\xe9rt <a href="href">t\xe4g \xefds</a> Lorem ipsu#'
-#
-# >>> c.convert('don\'t convert %(name)s tags on %(date)s')
-# u"d\xf6n't \xe7\xf6nv\xe9rt %(name)s t\xe4gs \xf6n %(date)s Lorem ips#"
+Example use::
 
+    >>> from dummy import Dummy
+    >>> c = Dummy()
+    >>> c.convert("My name is Bond, James Bond")
+    u'M\xfd n\xe4m\xe9 \xefs B\xf8nd, J\xe4m\xe9s B\xf8nd \u2360\u03c3\u044f\u0454\u043c \u03b9\u03c1#'
+    >>> print c.convert("My name is Bond, James Bond")
+    Mý nämé ïs Bønd, Jämés Bønd Ⱡσяєм ιρ#
+    >>> print c.convert("don't convert <a href='href'>tag ids</a>")
+    døn't çønvért <a href='href'>täg ïds</a> Ⱡσяєм ιρѕυ#
+    >>> print c.convert("don't convert %(name)s tags on %(date)s")
+    døn't çønvért %(name)s tägs øn %(date)s Ⱡσяєм ιρѕ#
+
+"""
+
+from converter import Converter
 
 # Substitute plain characters with accented lookalikes.
 # http://tlt.its.psu.edu/suggestions/international/web/codehtml.html#accent
-TABLE = {'A': u'\xC0',
-         'a': u'\xE4',
-         'b': u'\xDF',
-         'C': u'\xc7',
-         'c': u'\xE7',
-         'E': u'\xC9',
-         'e': u'\xE9',
-         'I': U'\xCC',
-         'i': u'\xEF',
-         'O': u'\xD8',
-         'o': u'\xF8',
-         'U': u'\xDB',
-         'u': u'\xFC',
-         'Y': u'\xDD',
-         'y': u'\xFD',
-         }
-
+TABLE = {
+    'A': u'À',
+    'a': u'ä',
+    'b': u'ß',
+    'C': u'Ç',
+    'c': u'ç',
+    'E': u'É',
+    'e': u'é',
+    'I': u'Ì',
+    'i': u'ï',
+    'O': u'Ø',
+    'o': u'ø',
+    'U': u'Û',
+    'u': u'ü',
+    'Y': u'Ý',
+    'y': u'ý',
+}
 
 
 # The print industry's standard dummy text, in use since the 1500s
-# see http://www.lipsum.com/
-LOREM = ' Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed ' \
-        'do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad ' \
-        'minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ' \
-        'ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate ' \
-        'velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat ' \
-        'cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. '
+# see http://www.lipsum.com/, then fed through a "fancy-text" converter.
+# The string should start with a space.
+LOREM = " " + " ".join(     # join and split just make the string easier here.
+    u"""
+    Ⱡσяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢тєтυя α∂ιριѕι¢ιηg єłιт, ѕє∂ ∂σ єιυѕмσ∂
+    тємρσя ιη¢ι∂ι∂υηт υт łαвσяє єт ∂σłσяє мαgηα αłιqυα. υт єηιм α∂ мιηιм
+    νєηιαм, qυιѕ ησѕтяυ∂ єχєя¢ιтαтιση υłłαм¢σ łαвσяιѕ ηιѕι υт αłιqυιρ єχ єα
+    ¢σммσ∂σ ¢σηѕєqυαт.  ∂υιѕ αυтє ιяυяє ∂σłσя ιη яєρяєнєη∂єяιт ιη νσłυρтαтє
+    νєłιт єѕѕє ¢ιłłυм ∂σłσяє єυ ƒυgιαт ηυłłα ραяιαтυя. єχ¢єρтєυя ѕιηт σ¢¢αє¢αт
+    ¢υρι∂αтαт ηση ρяσι∂єηт, ѕυηт ιη ¢υłρα qυι σƒƒι¢ια ∂єѕєяυηт мσłłιт αηιм ι∂
+    єѕт łαвσяυм.
+    """.split()
+)
 
 # To simulate more verbose languages (like German), pad the length of a string
 # by a multiple of PAD_FACTOR
@@ -85,20 +99,6 @@ def terminate(self, string):
         """replaces the final char of string with #"""
         return string[:-1] + '#'
 
-    def init_msgs(self, msgs):
-        """
-        Make sure the first msg in msgs has a plural property.
-        msgs is list of instances of polib.POEntry
-        """
-        if not msgs:
-            return
-        headers = msgs[0].get_property('msgstr')
-        has_plural = any(header.startswith('Plural-Forms:') for header in headers)
-        if not has_plural:
-            # Apply declaration for English pluralization rules
-            plural = "Plural-Forms: nplurals=2; plural=(n != 1);\\n"
-            headers.append(plural)
-
     def convert_msg(self, msg):
         """
         Takes one POEntry object and converts it (adds a dummy translation to it)
@@ -114,8 +114,10 @@ def convert_msg(self, msg):
             # translate singular and plural
             foreign_single = self.convert(source)
             foreign_plural = self.convert(plural)
-            plural = {'0': self.final_newline(source, foreign_single),
-                      '1': self.final_newline(plural, foreign_plural)}
+            plural = {
+                '0': self.final_newline(source, foreign_single),
+                '1': self.final_newline(plural, foreign_plural),
+            }
             msg.msgstr_plural = plural
         else:
             foreign = self.convert(source)

i18n/extract.py

@@ -45,7 +45,7 @@ def main():
         remove_file(source_msgs_dir.joinpath(filename))
 
     # Extract strings from mako templates.
-    babel_mako_cmd = 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT)
+    babel_mako_cmd = 'pybabel extract -F %s -c "Translators:" . -o %s' % (BABEL_CONFIG, BABEL_OUT)
 
     # Extract strings from django source files.
     make_django_cmd = (

i18n/generate.py

@@ -60,9 +60,12 @@ def merge(locale, target='django.po', fail_if_missing=True):
 def clean_metadata(file):
     """
     Clean up redundancies in the metadata caused by merging.
-    This reads in a PO file and simply saves it back out again.
     """
-    pofile(file).save()
+    # Reading in the .po file and saving it again fixes redundancies.
+    pomsgs = pofile(file)
+    # The msgcat tool marks the metadata as fuzzy, but it's ok as it is.
+    pomsgs.metadata_is_fuzzy = False
+    pomsgs.save()
 
 
 def validate_files(dir, files_to_merge):

i18n/make_dummy.py

@@ -38,9 +38,15 @@ def main(file, locale):
         raise IOError('File does not exist: %s' % file)
     pofile = polib.pofile(file)
     converter = Dummy()
-    converter.init_msgs(pofile.translated_entries())
     for msg in pofile:
         converter.convert_msg(msg)
+
+    # If any message has a plural, then the file needs plural information.
+    # Apply declaration for English pluralization rules so that ngettext will
+    # do something reasonable.
+    if any(m.msgid_plural for m in pofile):
+        pofile.metadata['Plural-Forms'] = 'nplurals=2; plural=(n != 1);'
+
     new_file = new_filename(file, locale)
     create_dir_if_necessary(new_file)
     pofile.save(new_file)

i18n/tests/test_converter.py

@@ -1,5 +1,8 @@
+"""Tests of i18n/converter.py"""
+
 import os
 from unittest import TestCase
+import ddt
 
 import converter
 
@@ -11,36 +14,48 @@ def inner_convert_string(self, string):
         return string.upper()
 
 
+@ddt.ddt
 class TestConverter(TestCase):
     """
     Tests functionality of i18n/converter.py
     """
 
-    def test_converter(self):
+    @ddt.data(
+        # no tags
+        ('big bad wolf',
+         'BIG BAD WOLF'),
+        # one html tag
+        ('big <strong>bad</strong> wolf',
+         'BIG <strong>BAD</strong> WOLF'),
+        # two html tags
+        ('big <b>bad</b> gray <i>wolf</i>',
+         'BIG <b>BAD</b> GRAY <i>WOLF</i>'),
+        # html tags with attributes
+        ('<a href="foo">bar</a> baz',
+         '<a href="foo">BAR</a> BAZ'),
+        ("<a href='foo'>bar</a> baz",
+         "<a href='foo'>BAR</a> BAZ"),
+        # one python tag
+        ('big %(adjective)s wolf',
+         'BIG %(adjective)s WOLF'),
+        # two python tags
+        ('big %(adjective)s gray %(noun)s',
+         'BIG %(adjective)s GRAY %(noun)s'),
+        # both kinds of tags
+        ('<strong>big</strong> %(adjective)s %(noun)s',
+         '<strong>BIG</strong> %(adjective)s %(noun)s'),
+        # .format-style tags
+        ('The {0} barn is {1!r}.',
+         'THE {0} BARN IS {1!r}.'),
+        # HTML entities
+        ('<b>&copy; 2013 edX, &#xa0;</b>',
+         '<b>&copy; 2013 EDX, &#xa0;</b>'),
+    )
+    def test_converter(self, data):
         """
         Tests with a simple converter (converts strings to uppercase).
         Assert that embedded HTML and python tags are not converted.
         """
-        c = UpcaseConverter()
-        test_cases = [
-            # no tags
-            ('big bad wolf', 'BIG BAD WOLF'),
-            # one html tag
-            ('big <strong>bad</strong> wolf', 'BIG <strong>BAD</strong> WOLF'),
-            # two html tags
-            ('big <b>bad</b> <i>wolf</i>', 'BIG <b>BAD</b> <i>WOLF</i>'),
-            # one python tag
-            ('big %(adjective)s wolf', 'BIG %(adjective)s WOLF'),
-            # two python tags
-            ('big %(adjective)s %(noun)s', 'BIG %(adjective)s %(noun)s'),
-            # both kinds of tags
-            ('<strong>big</strong> %(adjective)s %(noun)s',
-             '<strong>BIG</strong> %(adjective)s %(noun)s'),
-            # .format-style tags
-            ('The {0} barn is {1!r}.', 'THE {0} BARN IS {1!r}.'),
-            # HTML entities
-            ('<b>&copy; 2013 edX, &#xa0;</b>', '<b>&copy; 2013 EDX, &#xa0;</b>'),
-        ]
-        for source, expected in test_cases:
-            result = c.convert(source)
-            self.assertEquals(result, expected)
+        source, expected = data
+        result = UpcaseConverter().convert(source)
+        self.assertEquals(result, expected)

i18n/tests/test_dummy.py

@@ -1,10 +1,16 @@
+# -*- coding: utf-8 -*-
+"""Tests of i18n/dummy.py"""
+
 import os, string, random
 from unittest import TestCase
+
+import ddt
 from polib import POEntry
 
 import dummy
 
 
+@ddt.ddt
 class TestDummy(TestCase):
     """
     Tests functionality of i18n/dummy.py
@@ -13,39 +19,52 @@ class TestDummy(TestCase):
     def setUp(self):
         self.converter = dummy.Dummy()
 
-    def test_dummy(self):
+    def assertUnicodeEquals(self, str1, str2):
+        """Just like assertEquals, but doesn't put Unicode into the fail message.
+
+        Either nose, or rake, or something, deals very badly with unusual
+        Unicode characters in the assertions, so we use repr here to keep
+        things safe.
+
+        """
+        self.assertEquals(
+            str1, str2,
+            "Mismatch: %r != %r" % (str1, str2),
+        )
+
+    @ddt.data(
+        (u"hello my name is Bond, James Bond",
+         u"héllø mý nämé ïs Bønd, Jämés Bønd Ⱡσяєм ι#"),
+
+        (u"don't convert <a href='href'>tag ids</a>",
+         u"døn't çønvért <a href='href'>täg ïds</a> Ⱡσяєм ιρѕυ#"),
+
+        (u"don't convert %(name)s tags on %(date)s",
+         u"døn't çønvért %(name)s tägs øn %(date)s Ⱡσяєм ιρѕ#"),
+    )
+    def test_dummy(self, data):
         """
         Tests with a dummy converter (adds spurious accents to strings).
         Assert that embedded HTML and python tags are not converted.
         """
-        test_cases = [
-            ("hello my name is Bond, James Bond",
-             u'h\xe9ll\xf8 m\xfd n\xe4m\xe9 \xefs B\xf8nd, J\xe4m\xe9s B\xf8nd Lorem i#'),
-
-            ('don\'t convert <a href="href">tag ids</a>',
-             u'd\xf8n\'t \xe7\xf8nv\xe9rt <a href="href">t\xe4g \xefds</a> Lorem ipsu#'),
-
-            ('don\'t convert %(name)s tags on %(date)s',
-             u"d\xf8n't \xe7\xf8nv\xe9rt %(name)s t\xe4gs \xf8n %(date)s Lorem ips#")
-        ]
-        for source, expected in test_cases:
-            result = self.converter.convert(source)
-            self.assertEquals(result, expected)
+        source, expected = data
+        result = self.converter.convert(source)
+        self.assertUnicodeEquals(result, expected)
 
     def test_singular(self):
         entry = POEntry()
         entry.msgid = 'A lovely day for a cup of tea.'
-        expected = u'\xc0 l\xf8v\xe9l\xfd d\xe4\xfd f\xf8r \xe4 \xe7\xfcp \xf8f t\xe9\xe4. Lorem i#'
+        expected = u'À løvélý däý før ä çüp øf téä. Ⱡσяєм ι#'
         self.converter.convert_msg(entry)
-        self.assertEquals(entry.msgstr, expected)
+        self.assertUnicodeEquals(entry.msgstr, expected)
 
     def test_plural(self):
         entry = POEntry()
         entry.msgid = 'A lovely day for a cup of tea.'
         entry.msgid_plural = 'A lovely day for some cups of tea.'
-        expected_s = u'\xc0 l\xf8v\xe9l\xfd d\xe4\xfd f\xf8r \xe4 \xe7\xfcp \xf8f t\xe9\xe4. Lorem i#'
-        expected_p = u'\xc0 l\xf8v\xe9l\xfd d\xe4\xfd f\xf8r s\xf8m\xe9 \xe7\xfcps \xf8f t\xe9\xe4. Lorem ip#'
+        expected_s = u'À løvélý däý før ä çüp øf téä. Ⱡσяєм ι#'
+        expected_p = u'À løvélý däý før sømé çüps øf téä. Ⱡσяєм ιρ#'
         self.converter.convert_msg(entry)
         result = entry.msgstr_plural
-        self.assertEquals(result['0'], expected_s)
-        self.assertEquals(result['1'], expected_p)
+        self.assertUnicodeEquals(result['0'], expected_s)
+        self.assertUnicodeEquals(result['1'], expected_p)