[FIX] base_import: parse format from relation fields.

Before this commit, the parser (date, datetime, float, monetary, ...) was only
called on field from current model, and not on sub field.

Eg: product_id.price was not parsed as float, because product_id != float

This commit add an overiddable function to get all the parsers: get_parsers

This commit closes #14572

1 files changed, 106 insertions, 35 deletions

diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py
index 2a497106735..f01ee42c9c1 100644
--- a/addons/base_import/models/base_import.py
+++ b/addons/base_import/models/base_import.py
@@ -66,7 +66,7 @@ class Import(models.TransientModel):
     file_type = fields.Char('File Type')
 
     @api.model
-    def get_fields(self, model, depth=FIELDS_RECURSION_LIMIT):
+    def get_fields(self, model, depth=FIELDS_RECURSION_LIMIT, full=False):
         """ Recursively get fields for the provided model (through
         fields_get) and filter them according to importability
 
@@ -120,7 +120,8 @@ class Import(models.TransientModel):
             'type': 'id',
         }]
         model_fields = Model.fields_get()
-        blacklist = models.MAGIC_COLUMNS + [Model.CONCURRENCY_CHECK_FIELD]
+        blacklist = ([] if full else models.MAGIC_COLUMNS) + [Model.CONCURRENCY_CHECK_FIELD]
+
         for name, field in model_fields.iteritems():
             if name in blacklist:
                 continue
@@ -152,7 +153,7 @@ class Import(models.TransientModel):
                     dict(field_value, name='.id', string=_("Database ID"), type='id'),
                 ]
             elif field['type'] == 'one2many' and depth:
-                field_value['fields'] = self.get_fields(field['relation'], depth=depth-1)
+                field_value['fields'] = self.get_fields(field['relation'], depth=depth - 1, full=full)
                 if self.user_has_groups('base.group_no_one'):
                     field_value['fields'].append({'id': '.id', 'name': '.id', 'string': _("Database ID"), 'required': False, 'fields': [], 'type': 'id'})
 
@@ -587,44 +588,114 @@ class Import(models.TransientModel):
 
     @api.model
     def _parse_float_from_data(self, data, index, name, options):
+        """
+            DEPRECATED, use _parse_float_data instead !
+
+            Convert ``float string`` into `float python`` using ``xxx_separators`` from options.
+
+            :param list(list(str)) data: the content of file to import
+            :param (str, int) indexes: tuple with (field name, index in data)
+            :param options: format-specific options. {encoding, quoting, separator, headers}
+                            Separators options: 'float_thousand_separator', 'float_decimal_separator'
+            :returns: alter the data directly
+            :raises ValueError: in case the float could not be converted
+        """
+        return self._parse_float_data(data, [(name, index, 'float')], options)
+
+    @api.model
+    def _parse_float_data(self, data, indexes, options):
+        """ Convert ``float string`` into `float python`` using ``xxx_separators`` from options.
+
+            :param list(list(str)) data: the content of file to import
+            :param list((str, int, str)) indexes: (('field_name', index, str), ('field_name2', index2, str), ...)
+            :param options: format-specific options. {encoding, quoting, separator, headers}
+                            Separators options: 'float_thousand_separator', 'float_decimal_separator'
+            :returns: alter the data directly
+            :raises ValueError: if the float could not be converted
+        """
         thousand_separator = options.get('float_thousand_separator', ' ')
         decimal_separator = options.get('float_decimal_separator', '.')
         for line in data:
-            if not line[index]:
-                continue
-            line[index] = line[index].replace(thousand_separator, '').replace(decimal_separator, '.')
-            old_value = line[index]
-            line[index] = self._remove_currency_symbol(line[index])
-            if line[index] is False:
-                raise ValueError(_("Column %s contains incorrect values (value: %s)" % (name, old_value)))
+            for (name, index, dummy) in indexes:
+                if not line[index]:
+                    continue
+                line[index] = line[index].replace(thousand_separator, '').replace(decimal_separator, '.')
+                old_value = line[index]
+                line[index] = self._remove_currency_symbol(line[index])
+                if line[index] is False:
+                    raise ValueError(_("Column %s contains incorrect values (value: %s)" % (name, old_value)))
+
+    @api.model
+    def _parse_date_data(self, data, indexes, options):
+        """ Convert ``date/datetime string`` into `date/datetime python`` using ``xxx_format`` from options.
+
+            :param list(list(str)) data: the content of file to import
+            :param list((str, int, str)) indexes: (('field_name', index, str), ('field_name2', index2, str), ...)
+            :param options: format-specific options. {encoding, quoting, separator, headers}
+                            Separators options: 'date_format', 'datetime_format'
+            :returns: alter the data directly
+            :raises ValueError: if the date/datetime could not be converted
+            :raises Exception: if the format is improperly configured or other unexpected error.
+        """
+        dt = datetime.datetime
+        for (name, index, field_type) in indexes:
+            server_format = DEFAULT_SERVER_DATE_FORMAT if field_type == 'date' else DEFAULT_SERVER_DATETIME_FORMAT
+            if options.get('%s_format' % field_type, server_format) != server_format:
+                user_format = ustr(options.get('%s_format' % field_type)).encode('utf-8')
+                for num, line in enumerate(data):
+                    if line[index]:
+                        try:
+                            line[index] = dt.strftime(dt.strptime(ustr(line[index]).encode('utf-8'), user_format), server_format)
+                        except ValueError, e:
+                            raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, ustr(e.message)))
+                        except Exception, e:
+                            raise ValueError(_("Error Parsing Date [%s:L%d]: %s") % (name, num + 1, ustr(e.message)))
+
+    def _get_field_type(self, field_name_split, all_fields):
+        """ Return the type of the field.
+
+            :param [str) field_name_split: the complete field name spliited on '/'. Eg: vendor_id/product_id/price = [vendor_id, product_id, price]
+            :param (str, (...)) all_fields: a tuple of all fields available (limit of FIELDS_RECURSION_LIMIT).
+                                            ('<field>', ('fields'={ <recursive> }, type='<type>'))
+        """
+        field = field_name_split[0]
+
+        if len(field_name_split) == 1:
+            return all_fields[field][1]
+
+        all_fields = dict(map(lambda d: (d['name'], (d['fields'], d['type'])), all_fields[field][0]))
+        return self._get_field_type(field_name_split[1:], all_fields)
+
+    @api.model
+    def get_parsers(self):
+        return {
+            'date': self._parse_date_data,
+            'datetime': self._parse_date_data,
+            'float': self._parse_float_data,
+            'monetary': self._parse_float_data,
+        }
 
     @api.multi
     def _parse_import_data(self, data, import_fields, options):
-        # Get fields of type date/datetime
-        all_fields = self.env[self.res_model].fields_get()
-        for name, field in all_fields.iteritems():
-            if field['type'] in ('date', 'datetime') and name in import_fields:
-                # Parse date
-                index = import_fields.index(name)
-                dt = datetime.datetime
-                server_format = DEFAULT_SERVER_DATE_FORMAT if field['type'] == 'date' else DEFAULT_SERVER_DATETIME_FORMAT
-
-                if options.get('%s_format' % field['type'], server_format) != server_format:
-                    user_format = ustr(options.get('%s_format' % field['type'])).encode('utf-8')
-                    for num, line in enumerate(data):
-                        if line[index]:
-                            try:
-                                line[index] = dt.strftime(dt.strptime(ustr(line[index]).encode('utf-8'), user_format), server_format)
-                            except ValueError, e:
-                                raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, ustr(e.message)))
-                            except Exception, e:
-                                raise ValueError(_("Error Parsing Date [%s:L%d]: %s") % (name, num + 1, ustr(e.message)))
-
-            elif field['type'] in ('float', 'monetary') and name in import_fields:
-                # Parse float, sometimes float values from file have currency symbol or () to denote a negative value
-                # We should be able to manage both case
-                index = import_fields.index(name)
-                self._parse_float_from_data(data, index, name, options)
+        all_fields = self.get_fields(self.res_model, full=True)
+
+        # preprocess all_field to have: ('<field>', ('fields'={ <recursive> }, type='<type>'))
+        all_fields = dict(map(lambda d: (d['name'], (d['fields'], d['type'])), all_fields))
+
+        # group parser by field to import
+        to_parse = {}
+        for f in import_fields:
+            index = import_fields.index(f)
+            detected_type = self._get_field_type(f.split('/'), all_fields)
+            if detected_type in self.get_parsers().keys():
+                func = self.get_parsers()[detected_type].__name__
+                to_parse.setdefault(func, []).append((f, index, detected_type))
+
+        # meth = method parser
+        # vals = (('filed_name', index, type), ('field_name2', index2, type), ...)
+        for meth, vals in to_parse.iteritems():
+            getattr(self, meth)(data, vals, options)
+
         return data
 
     @api.multi