[IMP] tools: create TranslationModuleReader class

Instead of previous long methods, use a class to clarify what the
export actually does.
The TranslationModuleReader is written to copy the API of the
TranslationFileWriter.
This way, exporting translations is reduced to:
1. create a reader that will fetch all module translations (either
   from db or from static files)
2. create a writer in a specific format (po or csv)
3. export the content from the reader to the writer

Simplify the writer by deducing modules from exported translations
instead of fetching it again in a oneliner (this way can benefit from
yield operations)

Remove the 'all_installed' possibility in modules as it was not
working (creating query with 2 WHERE clause).

The new methode _get_translatable_records works on a per model basis.
This will allow a big performance gain as the previous code was
making a .exists() for each record individually.
In the future, this method could be removed as the main goal is to
test the presence of the rare attribute _translate=False.

1 files changed, 196 insertions, 141 deletions

diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py
index 6ab3c118437..120ba52fdb9 100644
--- a/odoo/tools/translate.py
+++ b/odoo/tools/translate.py
@@ -13,7 +13,7 @@ import re
 import tarfile
 import tempfile
 import threading
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 from datetime import datetime
 from os.path import join
 
@@ -647,13 +647,13 @@ class PoFileReader:
                     continue
                 _logger.error("malformed po file: unknown occurrence: %s", occurrence)
 
-def TranslationFileWriter(target, fileformat='po', lang=None, modules=None):
+def TranslationFileWriter(target, fileformat='po', lang=None):
     """ Iterate over translation file to return Odoo translation entries """
     if fileformat == 'csv':
         return CSVFileWriter(target)
 
     if fileformat == 'po':
-        return PoFileWriter(target, modules=modules, lang=lang)
+        return PoFileWriter(target, lang=lang)
 
     if fileformat == 'tgz':
         return TarFileWriter(target, lang=lang)
@@ -677,33 +677,16 @@ class CSVFileWriter:
 
 class PoFileWriter:
     """ Iterate over po file to return Odoo translation entries """
-    def __init__(self, target, modules, lang):
-        import odoo.release as release
+    def __init__(self, target, lang):
 
         self.buffer = target
         self.lang = lang
         self.po = polib.POFile()
 
-        self.po.header = "Translation of %s.\n" \
-                    "This file contains the translation of the following modules:\n" \
-                    "%s" % (release.description, ''.join("\t* %s\n" % m for m in modules))
-        now = datetime.utcnow().strftime('%Y-%m-%d %H:%M+0000')
-        self.po.metadata = {
-            'Project-Id-Version': "%s %s" % (release.description, release.version),
-            'Report-Msgid-Bugs-To': '',
-            'POT-Creation-Date': now,
-            'PO-Revision-Date': now,
-            'Last-Translator': '',
-            'Language-Team': '',
-            'MIME-Version': '1.0',
-            'Content-Type': 'text/plain; charset=UTF-8',
-            'Content-Transfer-Encoding': '',
-            'Plural-Forms': '',
-        }
-
     def write_rows(self, rows):
         # we now group the translations by source. That means one translation per source.
         grouped_rows = {}
+        modules = set([])
         for module, type, name, res_id, src, trad, comments in rows:
             row = grouped_rows.setdefault(src, {})
             row.setdefault('modules', set()).add(module)
@@ -711,6 +694,7 @@ class PoFileWriter:
                 row['translation'] = trad
             row.setdefault('tnrs', []).append((type, name, res_id))
             row.setdefault('comments', set()).update(comments)
+            modules.add(module)
 
         for src, row in sorted(grouped_rows.items()):
             if not self.lang:
@@ -718,7 +702,25 @@ class PoFileWriter:
                 row['translation'] = ''
             elif not row.get('translation'):
                 row['translation'] = ''
-            self.add_entry(row['modules'], row['tnrs'], src, row['translation'], row['comments'])
+            self.add_entry(row['modules'], sorted(row['tnrs']), src, row['translation'], row['comments'])
+
+        import odoo.release as release
+        self.po.header = "Translation of %s.\n" \
+                    "This file contains the translation of the following modules:\n" \
+                    "%s" % (release.description, ''.join("\t* %s\n" % m for m in modules))
+        now = datetime.utcnow().strftime('%Y-%m-%d %H:%M+0000')
+        self.po.metadata = {
+            'Project-Id-Version': "%s %s" % (release.description, release.version),
+            'Report-Msgid-Bugs-To': '',
+            'POT-Creation-Date': now,
+            'PO-Revision-Date': now,
+            'Last-Translator': '',
+            'Language-Team': '',
+            'MIME-Version': '1.0',
+            'Content-Type': 'text/plain; charset=UTF-8',
+            'Content-Transfer-Encoding': '',
+            'Plural-Forms': '',
+        }
 
         # buffer expects bytes
         self.buffer.write(str(self.po).encode())
@@ -763,7 +765,7 @@ class TarFileWriter:
 
         for mod, modrows in rows_by_module.items():
             with io.BytesIO() as buf:
-                po = PoFileWriter(buf, modules=[mod], lang=self.lang)
+                po = PoFileWriter(buf, lang=self.lang)
                 po.write_rows(modrows)
                 buf.seek(0)
 
@@ -781,14 +783,10 @@ class TarFileWriter:
         self.tar.close()
 
 # Methods to export the translation file
-
 def trans_export(lang, modules, buffer, format, cr):
-
-    translations = trans_generate(lang, modules, cr)
-    modules = set(t[0] for t in translations)
-    writer = TranslationFileWriter(buffer, fileformat=format, lang=lang, modules=modules)
-    writer.write_rows(translations)
-    del translations
+    reader = TranslationModuleReader(cr, modules=modules, lang=lang)
+    writer = TranslationFileWriter(buffer, fileformat=format, lang=lang)
+    writer.write_rows(reader)
 
 
 def trans_parse_rml(de):
@@ -872,12 +870,44 @@ def babel_extract_qweb(fileobj, keywords, comment_tags, options):
     _extract_translatable_qweb_terms(tree.getroot(), handle_text)
     return result
 
+ImdInfo = namedtuple('ExternalId', ['name', 'model', 'res_id', 'module'])
 
-def trans_generate(lang, modules, cr):
-    env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
-    to_translate = set()
 
-    def push_translation(module, type, name, id, source, comments=None):
+class TranslationModuleReader:
+    """ Retrieve translated records per module
+
+    :param cr: cursor to database to export
+    :param modules: list of modules to filter the exported terms, can be ['all']
+                    records with no external id are always ignored
+    :param lang: language code to retrieve the translations
+                 retrieve source terms only if not set
+    """
+
+    def __init__(self, cr, modules=None, lang=None):
+        self._cr = cr
+        self._modules = modules or ['all']
+        self._lang = lang
+        self.env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
+        self._to_translate = []
+        self._path_list = [(path, True) for path in odoo.addons.__path__]
+        self._installed_modules = [
+            m['name']
+            for m in self.env['ir.module.module'].search_read([('state', '=', 'installed')], fields=['name'])
+        ]
+
+        self._export_translatable_records()
+        self._export_translatable_resources()
+
+
+    def __iter__(self):
+        """ Export ir.translation values for all retrieved records """
+
+        IrTranslation = self.env['ir.translation']
+        for module, source, name, res_id, ttype, comments in self._to_translate:
+            trans = IrTranslation._get_source(name, ttype, self._lang, source) if self._lang else ""
+            yield (module, ttype, name, res_id, source, encode(trans) or '', comments)
+
+    def _push_translation(self, module, ttype, name, res_id, source, comments=None):
         # empty and one-letter terms are ignored, they probably are not meant to be
         # translated, and would be very hard to translate anyway.
         sanitized_term = (source or '').strip()
@@ -886,84 +916,98 @@ def trans_generate(lang, modules, cr):
         if not sanitized_term or len(sanitized_term) <= 1:
             return
 
-        tnx = (module, source, name, id, type, tuple(comments or ()))
-        to_translate.add(tnx)
-
-    def translatable_model(record):
-        if not record._translate:
-            return False
-
-        if record._name == 'ir.model.fields.selection':
-            record = record.field_id
-        if record._name == 'ir.model.fields':
-            field_name = record.name
-            field_model = env.get(record.model)
-            if (field_model is None or not field_model._translate or
-                    field_name not in field_model._fields):
-                return False
+        self._to_translate.append((module, source, name, res_id, ttype, tuple(comments or ())))
 
-        return True
-
-    query = 'SELECT min(name), model, res_id, module FROM ir_model_data'
-
-    if 'all_installed' in modules:
-        query += ' WHERE module IN ( SELECT name FROM ir_module_module WHERE state = \'installed\') '
-
-    if 'all' not in modules:
-        query += ' WHERE module IN %s'
-        query_param = (tuple(modules),)
-    else:
-        query += ' WHERE module != %s'
-        query_param = ('__export__',)
-
-    query += ' GROUP BY model, res_id, module ORDER BY module, model, min(name)'
+    def _get_translatable_records(self, records):
+        """ Filter the records that are translatable
 
-    cr.execute(query, query_param)
+        A record is considered as untranslatable if:
+        - it does not exist
+        - the model is flagged with _translate=False
+        - it is a field of a model flagged with _translate=False
+        - it is a selection of a field of a model flagged with _translate=False
 
-    for (xml_name, model, res_id, module) in cr.fetchall():
-        xml_name = "%s.%s" % (module, xml_name)
+        :param records: a list of namedtuple ImdInfo belonging to the same model
+        """
+        model = next(iter(records)).model
+        if model not in self.env:
+            _logger.error("Unable to find object %r", model)
+            return self.browse()
+
+        if not self.env[model]._translate:
+            return self.env[model].browse()
+
+        res_ids = [r.res_id for r in records]
+        records = self.env[model].browse(res_ids).exists()
+        if len(records) < len(res_ids):
+            missing_ids = set(records.ids) - set(res_ids)
+            _logger.warning("Unable to find objects %r with id %d", model, ', '.join(missing_ids))
+            if not records:
+                return records
+
+        if model == 'ir.model.fields.selection':
+            fields = defaultdict(list)
+            for selection in records:
+                fields[selection.field_id] = selection
+            for field, selection in fields.items():
+                field_name = field.name
+                field_model = self.env.get(field.model)
+                if (field_model is None or not field_model._translate or
+                        field_name not in field_model._fields):
+                    # the selection is linked to a model with _translate=False, remove it
+                    records -= selection
+        elif model == 'ir.model.fields':
+            for field in records:
+                field_name = field.name
+                field_model = self.env.get(field.model)
+                if (field_model is None or not field_model._translate or
+                        field_name not in field_model._fields):
+                    # the field is linked to a model with _translate=False, remove it
+                    records -= field
+
+        return records
+
+
+    def _export_translatable_records(self):
+        """ Export translations of all translated records having an external id """
+
+        query = """SELECT min(name), model, res_id, module
+                     FROM ir_model_data
+                    WHERE module = ANY(%s)
+                 GROUP BY model, res_id, module
+                 ORDER BY module, model, min(name)"""
+
+        if 'all' not in self._modules:
+            query_param = list(self._modules)
+        else:
+            query_param = self._installed_modules
 
-        if model not in env:
-            _logger.error(u"Unable to find object %r", model)
-            continue
+        self._cr.execute(query, (query_param,))
 
-        record = env[model].browse(res_id)
-        if not record.exists():
-            _logger.warning(u"Unable to find object %r with id %d", model, res_id)
-            continue
+        records_per_model = defaultdict(dict)
+        for (xml_name, model, res_id, module) in self._cr.fetchall():
+            records_per_model[model][res_id] = ImdInfo(xml_name, model, res_id, module)
 
-        if not translatable_model(record):
-            continue
+        for model, imd_per_id in records_per_model.items():
+            records = self._get_translatable_records(imd_per_id.values())
+            if not records:
+                continue
 
-        for field_name, field in record._fields.items():
-            if field.translate:
-                name = model + "," + field_name
-                try:
-                    value = record[field_name] or ''
-                except Exception:
-                    continue
-                for term in set(field.get_trans_terms(value)):
-                    trans_type = 'model_terms' if callable(field.translate) else 'model'
-                    push_translation(module, trans_type, name, xml_name, term)
-
-        # End of data for ir.model.data query results
-
-    installed_modules = [
-        m['name']
-        for m in env['ir.module.module'].search_read([('state', '=', 'installed')], fields=['name'])
-    ]
-
-    path_list = [(path, True) for path in odoo.addons.__path__]
-    # Also scan these non-addon paths
-    for bin_path in ['osv', 'report', 'modules', 'service', 'tools']:
-        path_list.append((os.path.join(config['root_path'], bin_path), True))
-    # non-recursive scan for individual files in root directory but without
-    # scanning subdirectories that may contain addons
-    path_list.append((config['root_path'], False))
-    _logger.debug("Scanning modules at paths: %s", path_list)
-
-    def get_module_from_path(path):
-        for (mp, rec) in path_list:
+            for record in records:
+                xml_name = "%s.%s" % (imd_per_id[record.id].module, imd_per_id[record.id].name)
+                for field_name, field in record._fields.items():
+                    if field.translate:
+                        name = model + "," + field_name
+                        try:
+                            value = record[field_name] or ''
+                        except Exception:
+                            continue
+                        for term in set(field.get_trans_terms(value)):
+                            trans_type = 'model_terms' if callable(field.translate) else 'model'
+                            self._push_translation(module, trans_type, name, xml_name, term)
+
+    def _get_module_from_path(self, path):
+        for (mp, rec) in self._path_list:
             mp = os.path.join(mp, '')
             dirname = os.path.join(os.path.dirname(path), '')
             if rec and path.startswith(mp) and dirname != mp:
@@ -971,22 +1015,24 @@ def trans_generate(lang, modules, cr):
                 return path.split(os.path.sep)[0]
         return 'base' # files that are not in a module are considered as being in 'base' module
 
-    def verified_module_filepaths(fname, path, root):
+    def _verified_module_filepaths(self, fname, path, root):
         fabsolutepath = join(root, fname)
         frelativepath = fabsolutepath[len(path):]
         display_path = "addons%s" % frelativepath
-        module = get_module_from_path(fabsolutepath)
-        if ('all' in modules or module in modules) and module in installed_modules:
+        module = self._get_module_from_path(fabsolutepath)
+        if ('all' in self._modules or module in self._modules) and module in self._installed_modules:
             if os.path.sep != '/':
                 display_path = display_path.replace(os.path.sep, '/')
             return module, fabsolutepath, frelativepath, display_path
         return None, None, None, None
 
-    def babel_extract_terms(fname, path, root, extract_method="python", trans_type='code',
+    def _babel_extract_terms(self, fname, path, root, extract_method="python", trans_type='code',
                                extra_comments=None, extract_keywords={'_': None}):
-        module, fabsolutepath, _, display_path = verified_module_filepaths(fname, path, root)
+
+        module, fabsolutepath, _, display_path = self._verified_module_filepaths(fname, path, root)
+        if not module:
+            return
         extra_comments = extra_comments or []
-        if not module: return
         src_file = open(fabsolutepath, 'rb')
         options = {}
         if extract_method == 'python':
@@ -996,41 +1042,50 @@ def trans_generate(lang, modules, cr):
                 # Babel 0.9.6 yields lineno, message, comments
                 # Babel 1.3 yields lineno, message, comments, context
                 lineno, message, comments = extracted[:3]
-                push_translation(module, trans_type, display_path, lineno,
+                self._push_translation(module, trans_type, display_path, lineno,
                                  encode(message), comments + extra_comments)
         except Exception:
             _logger.exception("Failed to extract terms from %s", fabsolutepath)
         finally:
             src_file.close()
 
-    for (path, recursive) in path_list:
-        _logger.debug("Scanning files of modules at %s", path)
-        for root, dummy, files in os.walk(path, followlinks=True):
-            for fname in fnmatch.filter(files, '*.py'):
-                babel_extract_terms(fname, path, root,
-                                    extract_keywords={'_': None, '_lt': None})
-            # Javascript source files in the static/src/js directory, rest is ignored (libs)
-            if fnmatch.fnmatch(root, '*/static/src/js*'):
-                for fname in fnmatch.filter(files, '*.js'):
-                    babel_extract_terms(fname, path, root, 'javascript',
-                                        extra_comments=[WEB_TRANSLATION_COMMENT],
-                                        extract_keywords={'_t': None, '_lt': None})
-            # QWeb template files
-            if fnmatch.fnmatch(root, '*/static/src/xml*'):
-                for fname in fnmatch.filter(files, '*.xml'):
-                    babel_extract_terms(fname, path, root, 'odoo.tools.translate:babel_extract_qweb',
-                                        extra_comments=[WEB_TRANSLATION_COMMENT])
-            if not recursive:
-                # due to topdown, first iteration is in first level
-                break
-
-    out = []
-    # translate strings marked as to be translated
-    Translation = env['ir.translation']
-    for module, source, name, id, type, comments in sorted(to_translate):
-        trans = Translation._get_source(name, type, lang, source) if lang else ""
-        out.append((module, type, name, id, source, encode(trans) or '', comments))
-    return out
+    def _export_translatable_resources(self):
+        """ Export translations for static terms
+        
+        This will include:
+        - the python strings marked with _() or _lt()
+        - the javascript strings marked with _t() or _lt() inside static/src/js/
+        - the strings inside Qweb files inside static/src/xml/
+        """
+
+        # Also scan these non-addon paths
+        for bin_path in ['osv', 'report', 'modules', 'service', 'tools']:
+            self._path_list.append((os.path.join(config['root_path'], bin_path), True))
+        # non-recursive scan for individual files in root directory but without
+        # scanning subdirectories that may contain addons
+        self._path_list.append((config['root_path'], False))
+        _logger.debug("Scanning modules at paths: %s", self._path_list)
+
+        for (path, recursive) in self._path_list:
+            _logger.debug("Scanning files of modules at %s", path)
+            for root, dummy, files in os.walk(path, followlinks=True):
+                for fname in fnmatch.filter(files, '*.py'):
+                    self._babel_extract_terms(fname, path, root,
+                                              extract_keywords={'_': None, '_lt': None})
+                # Javascript source files in the static/src/js directory, rest is ignored (libs)
+                if fnmatch.fnmatch(root, '*/static/src/js*'):
+                    for fname in fnmatch.filter(files, '*.js'):
+                        self._babel_extract_terms(fname, path, root, 'javascript',
+                                                  extra_comments=[WEB_TRANSLATION_COMMENT],
+                                                  extract_keywords={'_t': None, '_lt': None})
+                # QWeb template files
+                if fnmatch.fnmatch(root, '*/static/src/xml*'):
+                    for fname in fnmatch.filter(files, '*.xml'):
+                        self._babel_extract_terms(fname, path, root, 'odoo.tools.translate:babel_extract_qweb',
+                                                  extra_comments=[WEB_TRANSLATION_COMMENT])
+                if not recursive:
+                    # due to topdown, first iteration is in first level
+                    break
 
 
 def trans_load(cr, filename, lang, verbose=True, create_empty_translation=False, overwrite=False):