diff options
Diffstat (limited to 'addons/base_import/models/base_import.py')
| -rw-r--r-- | addons/base_import/models/base_import.py | 133 |
1 files changed, 104 insertions, 29 deletions
diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index 0eb7e7b7580..4f2c262fab6 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import chardet import datetime import io import itertools @@ -45,6 +46,37 @@ EXTENSIONS = { for mime, (ext, handler, req) in FILE_TYPE_DICT.items() } +class Base(models.AbstractModel): + _inherit = 'base' + + @api.model + def get_import_templates(self): + """ + Get the import templates label and path. + + :return: a list(dict) containing label and template path + like ``[{'label': 'foo', 'template': 'path'}]`` + """ + return [] + +class ImportMapping(models.Model): + """ mapping of previous column:field selections + + This is useful when repeatedly importing from a third-party + system: column names generated by the external system may + not match Odoo's field names or labels. This model is used + to save the mapping between column names and fields so that + next time a user imports from the same third-party systems + we can automatically match the columns to the correct field + without them having to re-enter the mapping every single + time. + """ + _name = 'base_import.mapping' + + res_model = fields.Char(index=True) + column_name = fields.Char() + field_name = fields.Char() + class Import(models.TransientModel): @@ -251,15 +283,10 @@ class Import(models.TransientModel): def _read_csv(self, options): """ Returns a CSV-parsed iterator of all non-empty lines in the file :throws csv.Error: if an error is detected during CSV parsing - :throws UnicodeDecodeError: if ``options.encoding`` is incorrect """ csv_data = self.file - - # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet - encoding = options.get('encoding', 'utf-8') - if encoding != 'utf-8': - # csv module expect utf-8, see http://docs.python.org/2/library/csv.html - csv_data = csv_data.decode(encoding).encode('utf-8') + encoding = chardet.detect(csv_data)['encoding'] + csv_data = csv_data.decode(encoding).encode('utf-8') csv_iterator = pycompat.csv_reader( io.BytesIO(csv_data), @@ -376,7 +403,7 @@ class Import(models.TransientModel): if results: return results - return ['id', 'text', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html'] + return ['id', 'text', 'boolean', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html'] @api.model def _find_type_from_preview(self, options, preview): @@ -400,6 +427,7 @@ class Import(models.TransientModel): :rtype: list(Field) """ string_match = None + IrTranslation = self.env['ir.translation'] for field in fields: # FIXME: should match all translations & original # TODO: use string distance (levenshtein? hamming?) @@ -409,6 +437,9 @@ class Import(models.TransientModel): # matching string are not reliable way because # strings have no unique constraint string_match = field + translated_header = IrTranslation._get_source('ir.model.fields,field_description', 'model', self.env.lang, header).lower() + if translated_header == field['string'].lower(): + string_match = field if string_match: # this behavior is only applied if there is no matching field['name'] return [string_match] @@ -440,23 +471,31 @@ class Import(models.TransientModel): Will consume the first line of the ``rows`` iterator. - Returns a pair of (None, None) if headers were not requested - or the list of headers and a dict mapping cell indices - to key paths in the ``fields`` tree + Returns the list of headers and a dict mapping cell indices + to key paths in the ``fields`` tree. If headers were not + requested, both collections are empty. :param Iterator rows: :param dict fields: :param dict options: - :rtype: (None, None) | (list(str), dict(int: list(str))) + :rtype: (list(str), dict(int: list(str))) """ if not options.get('headers'): return [], {} headers = next(rows) - return headers, { - index: [field['name'] for field in self._match_header(header, fields, options)] or None - for index, header in enumerate(headers) - } + matches = {} + mapping_records = self.env['base_import.mapping'].search_read([('res_model', '=', self.res_model)], ['column_name', 'field_name']) + mapping_fields = {rec['column_name']: rec['field_name'] for rec in mapping_records} + for index, header in enumerate(headers): + match_field = [] + mapping_field_name = mapping_fields.get(header.lower()) + if mapping_field_name: + match_field = mapping_field_name.split('/') + if not match_field: + match_field = [field['name'] for field in self._match_header(header, fields, options)] + matches[index] = match_field or None + return headers, matches @api.multi def parse_preview(self, options, count=10): @@ -469,7 +508,7 @@ class Import(models.TransientModel): :param int count: number of preview lines to generate :param options: format-specific options. - CSV: {encoding, quoting, separator, headers} + CSV: {quoting, separator, headers} :type options: {str, str, str, bool} :returns: {fields, matches, headers, preview} | {error, preview} :rtype: {dict(str: dict(...)), dict(int, list(str)), list(str), list(list(str))} | {str, str} @@ -482,14 +521,23 @@ class Import(models.TransientModel): # Match should have consumed the first row (iif headers), get # the ``count`` next rows for preview preview = list(itertools.islice(rows, count)) - assert preview, "CSV file seems to have no content" + assert preview, "file seems to have no content" header_types = self._find_type_from_preview(options, preview) - if options.get('keep_matches', False) and len(options.get('fields', [])): + if options.get('keep_matches') and len(options.get('fields', [])): matches = {} for index, match in enumerate(options.get('fields')): if match: matches[index] = match.split('/') + if options.get('keep_matches'): + advanced_mode = options.get('advanced') + else: + # Check is label contain relational field + has_relational_header = any(len(models.fix_import_export_id_paths(col)) > 1 for col in headers) + # Check is matches fields have relational field + has_relational_match = any(len(match) > 1 for field, match in matches.items() if match) + advanced_mode = has_relational_header or has_relational_match + return { 'fields': fields, 'matches': matches or False, @@ -497,6 +545,7 @@ class Import(models.TransientModel): 'headers_type': header_types or False, 'preview': preview, 'options': options, + 'advanced_mode': advanced_mode, 'debug': self.user_has_groups('base.group_no_one'), } except Exception as error: @@ -505,7 +554,7 @@ class Import(models.TransientModel): # preview to a list in the return. _logger.debug("Error during parsing preview", exc_info=True) preview = None - if self.file_type == 'text/csv': + if self.file_type == 'text/csv' and self.file: preview = self.file[:ERROR_PREVIEW_BYTES].decode('iso-8859-1') return { 'error': str(error), @@ -627,7 +676,13 @@ class Import(models.TransientModel): try: line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format) except ValueError as e: - raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e)) + try: + # Allow to import date in datetime fields + if field['type'] == 'datetime': + user_format = pycompat.to_native(options.get('date_format')) + line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format) + except ValueError as e: + raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e)) except Exception as e: raise ValueError(_("Error Parsing Date [%s:L%d]: %s") % (name, num + 1, e)) # Check if the field is in import_field and is a relational (followed by /) @@ -643,12 +698,14 @@ class Import(models.TransientModel): return data @api.multi - def do(self, fields, options, dryrun=False): + def do(self, fields, columns, options, dryrun=False): """ Actual execution of the import :param fields: import mapping: maps each column to a field, ``False`` for the columns to ignore :type fields: list(str|bool) + :param columns: columns label + :type columns: list(str|bool) :param dict options: :param bool dryrun: performs all import operations (and validations) but rollbacks writes, allows @@ -661,7 +718,7 @@ class Import(models.TransientModel): error message associated with the error (a string) and ``record`` the data which failed to import (or ``false`` if that data isn't available or provided) - :rtype: list({type, message, record}) + :rtype: dict(ids: list(int), messages: list({type, message, record})) """ self.ensure_one() self._cr.execute('SAVEPOINT import') @@ -671,11 +728,13 @@ class Import(models.TransientModel): # Parse date and float field data = self._parse_import_data(data, import_fields, options) except ValueError as error: - return [{ - 'type': 'error', - 'message': pycompat.text_type(error), - 'record': False, - }] + return { + 'messages': [{ + 'type': 'error', + 'message': pycompat.text_type(error), + 'record': False, + }] + } _logger.info('importing %d rows...', len(data)) @@ -698,4 +757,20 @@ class Import(models.TransientModel): except psycopg2.InternalError: pass - return import_result['messages'] + # Insert/Update mapping columns when import complete successfully + if import_result['ids'] and options.get('headers'): + BaseImportMapping = self.env['base_import.mapping'] + for index, column_name in enumerate(columns): + if column_name: + # Update to latest selected field + exist_records = BaseImportMapping.search([('res_model', '=', self.res_model), ('column_name', '=', column_name)]) + if exist_records: + exist_records.write({'field_name': fields[index]}) + else: + BaseImportMapping.create({ + 'res_model': self.res_model, + 'column_name': column_name, + 'field_name': fields[index] + }) + + return import_result |