Merge with trunk-server

bzr revid: ysa@tinyerp.com-20101230092459-b1xlf2trdw76zn6k

4 files changed, 66 insertions, 16 deletions

diff --git a/bin/tools/config.py b/bin/tools/config.py
index 0c47c2a61bc..bc9229e0b72 100644
--- a/bin/tools/config.py
+++ b/bin/tools/config.py
@@ -58,6 +58,8 @@ class configmanager(object):
             'xmlrpcs': True,
             'translate_in': None,
             'translate_out': None,
+            'overwrite_existing_translations': False,
+            'load_language': None,
             'language': None,
             'pg_path': None,
             'admin_passwd': 'admin',
@@ -94,7 +96,7 @@ class configmanager(object):
             'publisher_warranty_url': 'http://services.openerp.com/publisher-warranty/',
         }
         
-        self.blacklist_for_save = set(["publisher_warranty_url"])
+        self.blacklist_for_save = set(["publisher_warranty_url", "load_language"])
 
         self.misc = {}
         self.config_file = fname
@@ -209,12 +211,16 @@ class configmanager(object):
             "Option '-l' is mandatory in case of importation"
             )
 
+        group.add_option('--load-language', dest="load_language",
+                         help="specifies the languages for the translations you want to be loaded")
         group.add_option('-l', "--language", dest="language",
                          help="specify the language of the translation file. Use it with --i18n-export or --i18n-import")
         group.add_option("--i18n-export", dest="translate_out",
                          help="export all sentences to be translated to a CSV file, a PO file or a TGZ archive and exit")
         group.add_option("--i18n-import", dest="translate_in",
                          help="import a CSV or a PO file with translations and exit. The '-l' option is required.")
+        group.add_option("--i18n-overwrite", dest="overwrite_existing_translations", action="store_true", default=False,
+                         help="overwrites existing translation terms on importing a CSV or a PO file.")
         group.add_option("--modules", dest="translate_modules",
                          help="specify modules to export. Use in combination with --i18n-export")
         group.add_option("--addons-path", dest="addons_path",
@@ -240,6 +246,9 @@ class configmanager(object):
         die(opt.translate_in and (not opt.language or not opt.db_name),
             "the i18n-import option cannot be used without the language (-l) and the database (-d) options")
 
+        die(opt.overwrite_existing_translations and (not opt.translate_in),
+            "the i18n-overwrite option cannot be used without the i18n-import option")
+
         die(opt.translate_out and (not opt.db_name),
             "the i18n-export option cannot be used without the database (-d) option")
 
@@ -285,7 +294,8 @@ class configmanager(object):
                 self.options[arg] = getattr(opt, arg)
 
         keys = [
-            'language', 'translate_out', 'translate_in', 'debug_mode', 'smtp_ssl',
+            'language', 'translate_out', 'translate_in', 'overwrite_existing_translations',
+            'debug_mode', 'smtp_ssl', 'load_language',
             'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog',
             'list_db', 'xmlrpcs',
             'test_file', 'test_disable', 'test_commit', 'test_report_directory'
@@ -443,7 +453,7 @@ class configmanager(object):
         loglevelnames = dict(zip(self._LOGLEVELS.values(), self._LOGLEVELS.keys()))
         p.add_section('options')
         for opt in sorted(self.options.keys()):
-            if opt in ('version', 'language', 'translate_out', 'translate_in', 'init', 'update'):
+            if opt in ('version', 'language', 'translate_out', 'translate_in', 'overwrite_existing_translations', 'init', 'update'):
                 continue
             if opt in self.blacklist_for_save:
                 continue
diff --git a/bin/tools/convert.py b/bin/tools/convert.py
index e46d37f688a..db8d71843ba 100644
--- a/bin/tools/convert.py
+++ b/bin/tools/convert.py
@@ -45,10 +45,12 @@ import netsvc
 import osv
 import pooler
 from config import config
-from osv.orm import except_orm, SKIPPED_ELEMENT_TYPES
 from tools.translate import _
 from yaml_import import convert_yaml_import
 
+# List of etree._Element subclasses that we choose to ignore when parsing XML.
+from tools import SKIPPED_ELEMENT_TYPES
+
 # Import of XML records requires the unsafe eval as well,
 # almost everywhere, which is ok because it supposedly comes
 # from trusted data, but at least we make it obvious now.
@@ -865,7 +867,7 @@ form: module.record_id""" % (xml_id,)
                             self.__logger.error('Parse error in %s:%d: \n%s',
                                                 rec.getroottree().docinfo.URL,
                                                 rec.sourceline,
-                                                etree.tostring(rec).strip())
+                                                etree.tostring(rec).strip(), exc_info=True)
                             self.cr.rollback()
                             raise
         return True
@@ -945,7 +947,7 @@ def convert_csv_import(cr, module, fname, csvcontent, idref=None, mode='init',
     result, rows, warning_msg, dummy = pool.get(model).import_data(cr, uid, fields, datas,mode, module, noupdate, filename=fname_partial)
     if result < 0:
         # Report failed import and abort module install
-        raise except_orm(_('Module loading failed'), _('Loading %s/%s failed:\n %s') % (module, fname, warning_msg))
+        raise Exception(_('Module loading failed: file %s/%s could not be processed:\n %s') % (module, fname, warning_msg))
     if config.get('import_partial'):
         data = pickle.load(file(config.get('import_partial')))
         data[fname_partial] = 0
diff --git a/bin/tools/misc.py b/bin/tools/misc.py
index 2515803f452..390be6b7837 100644
--- a/bin/tools/misc.py
+++ b/bin/tools/misc.py
@@ -44,6 +44,7 @@ from email.Header import Header
 from email.Utils import formatdate, COMMASPACE
 from email import Encoders
 from itertools import islice, izip
+from lxml import etree
 from which import which
 if sys.version_info[:2] < (2, 4):
     from threadinglocal import local
@@ -60,6 +61,10 @@ from lru import LRU
 
 _logger = logging.getLogger('tools')
 
+# List of etree._Element subclasses that we choose to ignore when parsing XML.
+# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
+SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase)
+
 # initialize a database with base/base.sql
 def init_db(cr):
     import addons
@@ -257,7 +262,7 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False):
             return fo
     if os.path.splitext(name)[1] == '.rml':
         raise IOError, 'Report %s doesn\'t exist or deleted : ' %str(name)
-    raise IOError, 'File not found : '+str(name)
+    raise IOError, 'File not found : %s' % name
 
 
 #----------------------------------------------------------
diff --git a/bin/tools/translate.py b/bin/tools/translate.py
index 3ad66f3d041..ea98bd60b77 100644
--- a/bin/tools/translate.py
+++ b/bin/tools/translate.py
@@ -33,14 +33,14 @@ import tarfile
 import tempfile
 import threading
 from os.path import join
-import logging
 
 from datetime import datetime
 from lxml import etree
 
-import tools, pooler
+import tools
 import netsvc
 from tools.misc import UpdateableStr
+from tools.misc import SKIPPED_ELEMENT_TYPES
 
 _LOCALE2WIN32 = {
     'af_ZA': 'Afrikaans_South Africa',
@@ -159,7 +159,7 @@ class GettextAlias(object):
         # find current DB based on thread/worker db name (see netsvc)
         db_name = getattr(threading.currentThread(), 'dbname', None)
         if db_name:
-            return pooler.get_db_only(dbname)
+            return pooler.get_db_only(db_name)
 
     def _get_cr(self, frame):
         is_new_cr = False
@@ -219,6 +219,7 @@ class GettextAlias(object):
                 logger.debug('no translation language detected, skipping translation for "%r" ', source)
         except Exception:
             logger.debug('translation went wrong for "%r", skipped', source)
+                # if so, double-check the root/base translations filenames
         finally:
             if cr and is_new_cr:
                 cr.close()
@@ -468,12 +469,13 @@ def trans_export(lang, modules, buffer, format, dbname=None):
     _process(format, modules, trans, buffer, lang, newlang)
     del trans
 
-
 def trans_parse_xsl(de):
     res = []
     for n in de:
         if n.get("t"):
-            for m in [j for j in n if j.text]:
+            for m in n:
+                if isinstance(m, SKIPPED_ELEMENT_TYPES) or not m.text:
+                    continue
                 l = m.text.strip().replace('\n',' ')
                 if len(l):
                     res.append(l.encode("utf8"))
@@ -483,7 +485,9 @@ def trans_parse_xsl(de):
 def trans_parse_rml(de):
     res = []
     for n in de:
-        for m in [j for j in n if j.text]:
+        for m in n:
+            if isinstance(m, SKIPPED_ELEMENT_TYPES) or not m.text:
+                continue
             string_list = [s.replace('\n', ' ').strip() for s in re.split('\[\[.+?\]\]', m.text)]
             for s in string_list:
                 if s:
@@ -749,6 +753,10 @@ def trans_generate(lang, modules, dbname=None):
     else :
         path_list = [root_path,] + apaths
 
+    # Also scan these non-addon paths
+    for bin_path in ['osv', 'report' ]:
+        path_list.append(os.path.join(tools.config['root_path'], bin_path))
+
     logger.debug("Scanning modules at paths: ", path_list)
 
     mod_paths = []
@@ -768,6 +776,8 @@ def trans_generate(lang, modules, dbname=None):
             if module in installed_modules:
                 frelativepath = str("addons" + frelativepath)
             ite = re_dquotes.finditer(code_string)
+            code_offset = 0
+            code_line = 1
             for i in ite:
                 src = i.group(1)
                 if src.startswith('""'):
@@ -775,11 +785,18 @@ def trans_generate(lang, modules, dbname=None):
                     src = src[2:-2]
                 else:
                     src = join_dquotes.sub(r'\1', src)
+                # try to count the lines from the last pos to our place:
+                code_line += code_string[code_offset:i.start(1)].count('\n')
                 # now, since we did a binary read of a python source file, we
                 # have to expand pythonic escapes like the interpreter does.
                 src = src.decode('string_escape')
-                push_translation(module, terms_type, frelativepath, 0, encode(src))
+                push_translation(module, terms_type, frelativepath, code_line, encode(src))
+                code_line += i.group(1).count('\n')
+                code_offset = i.end() # we have counted newlines up to the match end
+
             ite = re_quotes.finditer(code_string)
+            code_offset = 0 #reset counters
+            code_line = 1
             for i in ite:
                 src = i.group(1)
                 if src.startswith("''"):
@@ -787,8 +804,11 @@ def trans_generate(lang, modules, dbname=None):
                     src = src[2:-2]
                 else:
                     src = join_quotes.sub(r'\1', src)
+                code_line += code_string[code_offset:i.start(1)].count('\n')
                 src = src.decode('string_escape')
-                push_translation(module, terms_type, frelativepath, 0, encode(src))
+                push_translation(module, terms_type, frelativepath, code_line, encode(src))
+                code_line += i.group(1).count('\n')
+                code_offset = i.end() # we have counted newlines up to the match end
 
     for path in path_list:
         logger.debug("Scanning files of modules at %s", path)
@@ -900,7 +920,7 @@ def trans_load_data(db_name, fileobj, fileformat, lang, lang_name=None, verbose=
                 args.append(('res_id', '=', dic['res_id']))
             ids = trans_obj.search(cr, uid, args)
             if ids:
-                if context.get('overwrite'):
+                if context.get('overwrite') and dic['value']:
                     trans_obj.write(cr, uid, ids, {'value': dic['value']})
             else:
                 trans_obj.create(cr, uid, dic)
@@ -952,5 +972,18 @@ def resetlocale():
         except locale.Error:
             continue
 
+def load_language(cr, lang):
+    """Loads a translation terms for a language.
+    Used mainly to automate language loading at db initialization.
+    
+    :param lang: language ISO code with optional _underscore_ and l10n flavor (ex: 'fr', 'fr_BE', but not 'fr-BE')
+    :type lang: str
+    """
+    pool = pooler.get_pool(cr.dbname)
+    language_installer = pool.get('base.language.install')
+    uid = 1
+    oid = language_installer.create(cr, uid, {'lang': lang})
+    language_installer.lang_install(cr, uid, [oid], context=None)
+
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: