[IMP] import/export UIs

Task 40692

Various changes to import/export (mainly) UIs:

* default to excel & "full" (non-import-compatible) export
* auto-detect encoding of CSV using chardet
* remember column -> field mapping after having imported a file (useful
   for repeated imports where auto-matching failed)
* better handle localised booleans & column names
* automatically select source list view's fields when exporting
* better integrate import templates feature and add a number of templates

7 files changed, 245 insertions, 152 deletions

diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py
index 0eb7e7b7580..4f2c262fab6 100644
--- a/addons/base_import/models/base_import.py
+++ b/addons/base_import/models/base_import.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import chardet
 import datetime
 import io
 import itertools
@@ -45,6 +46,37 @@ EXTENSIONS = {
     for mime, (ext, handler, req) in FILE_TYPE_DICT.items()
 }
 
+class Base(models.AbstractModel):
+    _inherit = 'base'
+
+    @api.model
+    def get_import_templates(self):
+        """
+        Get the import templates label and path.
+
+        :return: a list(dict) containing label and template path
+                 like ``[{'label': 'foo', 'template': 'path'}]``
+        """
+        return []
+
+class ImportMapping(models.Model):
+    """ mapping of previous column:field selections
+
+    This is useful when repeatedly importing from a third-party
+    system: column names generated by the external system may
+    not match Odoo's field names or labels. This model is used
+    to save the mapping between column names and fields so that
+    next time a user imports from the same third-party systems
+    we can automatically match the columns to the correct field
+    without them having to re-enter the mapping every single
+    time.
+    """
+    _name = 'base_import.mapping'
+
+    res_model = fields.Char(index=True)
+    column_name = fields.Char()
+    field_name = fields.Char()
+
 
 class Import(models.TransientModel):
 
@@ -251,15 +283,10 @@ class Import(models.TransientModel):
     def _read_csv(self, options):
         """ Returns a CSV-parsed iterator of all non-empty lines in the file
             :throws csv.Error: if an error is detected during CSV parsing
-            :throws UnicodeDecodeError: if ``options.encoding`` is incorrect
         """
         csv_data = self.file
-
-        # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet
-        encoding = options.get('encoding', 'utf-8')
-        if encoding != 'utf-8':
-            # csv module expect utf-8, see http://docs.python.org/2/library/csv.html
-            csv_data = csv_data.decode(encoding).encode('utf-8')
+        encoding = chardet.detect(csv_data)['encoding']
+        csv_data = csv_data.decode(encoding).encode('utf-8')
 
         csv_iterator = pycompat.csv_reader(
             io.BytesIO(csv_data),
@@ -376,7 +403,7 @@ class Import(models.TransientModel):
 
         if results:
             return results
-        return ['id', 'text', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html']
+        return ['id', 'text', 'boolean', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html']
 
     @api.model
     def _find_type_from_preview(self, options, preview):
@@ -400,6 +427,7 @@ class Import(models.TransientModel):
             :rtype: list(Field)
         """
         string_match = None
+        IrTranslation = self.env['ir.translation']
         for field in fields:
             # FIXME: should match all translations & original
             # TODO: use string distance (levenshtein? hamming?)
@@ -409,6 +437,9 @@ class Import(models.TransientModel):
                 # matching string are not reliable way because
                 # strings have no unique constraint
                 string_match = field
+            translated_header = IrTranslation._get_source('ir.model.fields,field_description', 'model', self.env.lang, header).lower()
+            if translated_header == field['string'].lower():
+                string_match = field
         if string_match:
             # this behavior is only applied if there is no matching field['name']
             return [string_match]
@@ -440,23 +471,31 @@ class Import(models.TransientModel):
 
             Will consume the first line of the ``rows`` iterator.
 
-            Returns a pair of (None, None) if headers were not requested
-            or the list of headers and a dict mapping cell indices
-            to key paths in the ``fields`` tree
+            Returns the list of headers and a dict mapping cell indices
+            to key paths in the ``fields`` tree. If headers were not
+            requested, both collections are empty.
 
             :param Iterator rows:
             :param dict fields:
             :param dict options:
-            :rtype: (None, None) | (list(str), dict(int: list(str)))
+            :rtype: (list(str), dict(int: list(str)))
         """
         if not options.get('headers'):
             return [], {}
 
         headers = next(rows)
-        return headers, {
-            index: [field['name'] for field in self._match_header(header, fields, options)] or None
-            for index, header in enumerate(headers)
-        }
+        matches = {}
+        mapping_records = self.env['base_import.mapping'].search_read([('res_model', '=', self.res_model)], ['column_name', 'field_name'])
+        mapping_fields = {rec['column_name']: rec['field_name'] for rec in mapping_records}
+        for index, header in enumerate(headers):
+            match_field = []
+            mapping_field_name = mapping_fields.get(header.lower())
+            if mapping_field_name:
+                match_field = mapping_field_name.split('/')
+            if not match_field:
+                match_field = [field['name'] for field in self._match_header(header, fields, options)]
+            matches[index] = match_field or None
+        return headers, matches
 
     @api.multi
     def parse_preview(self, options, count=10):
@@ -469,7 +508,7 @@ class Import(models.TransientModel):
 
             :param int count: number of preview lines to generate
             :param options: format-specific options.
-                            CSV: {encoding, quoting, separator, headers}
+                            CSV: {quoting, separator, headers}
             :type options: {str, str, str, bool}
             :returns: {fields, matches, headers, preview} | {error, preview}
             :rtype: {dict(str: dict(...)), dict(int, list(str)), list(str), list(list(str))} | {str, str}
@@ -482,14 +521,23 @@ class Import(models.TransientModel):
             # Match should have consumed the first row (iif headers), get
             # the ``count`` next rows for preview
             preview = list(itertools.islice(rows, count))
-            assert preview, "CSV file seems to have no content"
+            assert preview, "file seems to have no content"
             header_types = self._find_type_from_preview(options, preview)
-            if options.get('keep_matches', False) and len(options.get('fields', [])):
+            if options.get('keep_matches') and len(options.get('fields', [])):
                 matches = {}
                 for index, match in enumerate(options.get('fields')):
                     if match:
                         matches[index] = match.split('/')
 
+            if options.get('keep_matches'):
+                advanced_mode = options.get('advanced')
+            else:
+                # Check is label contain relational field
+                has_relational_header = any(len(models.fix_import_export_id_paths(col)) > 1 for col in headers)
+                # Check is matches fields have relational field
+                has_relational_match = any(len(match) > 1 for field, match in matches.items() if match)
+                advanced_mode = has_relational_header or has_relational_match
+
             return {
                 'fields': fields,
                 'matches': matches or False,
@@ -497,6 +545,7 @@ class Import(models.TransientModel):
                 'headers_type': header_types or False,
                 'preview': preview,
                 'options': options,
+                'advanced_mode': advanced_mode,
                 'debug': self.user_has_groups('base.group_no_one'),
             }
         except Exception as error:
@@ -505,7 +554,7 @@ class Import(models.TransientModel):
             # preview to a list in the return.
             _logger.debug("Error during parsing preview", exc_info=True)
             preview = None
-            if self.file_type == 'text/csv':
+            if self.file_type == 'text/csv' and self.file:
                 preview = self.file[:ERROR_PREVIEW_BYTES].decode('iso-8859-1')
             return {
                 'error': str(error),
@@ -627,7 +676,13 @@ class Import(models.TransientModel):
                             try:
                                 line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format)
                             except ValueError as e:
-                                raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e))
+                                try:
+                                    # Allow to import date in datetime fields
+                                    if field['type'] == 'datetime':
+                                        user_format = pycompat.to_native(options.get('date_format'))
+                                        line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format)
+                                except ValueError as e:
+                                    raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e))
                             except Exception as e:
                                 raise ValueError(_("Error Parsing Date [%s:L%d]: %s") % (name, num + 1, e))
             # Check if the field is in import_field and is a relational (followed by /)
@@ -643,12 +698,14 @@ class Import(models.TransientModel):
         return data
 
     @api.multi
-    def do(self, fields, options, dryrun=False):
+    def do(self, fields, columns, options, dryrun=False):
         """ Actual execution of the import
 
         :param fields: import mapping: maps each column to a field,
                        ``False`` for the columns to ignore
         :type fields: list(str|bool)
+        :param columns: columns label
+        :type columns: list(str|bool)
         :param dict options:
         :param bool dryrun: performs all import operations (and
                             validations) but rollbacks writes, allows
@@ -661,7 +718,7 @@ class Import(models.TransientModel):
                   error message associated with the error (a string)
                   and ``record`` the data which failed to import (or
                   ``false`` if that data isn't available or provided)
-        :rtype: list({type, message, record})
+        :rtype: dict(ids: list(int), messages: list({type, message, record}))
         """
         self.ensure_one()
         self._cr.execute('SAVEPOINT import')
@@ -671,11 +728,13 @@ class Import(models.TransientModel):
             # Parse date and float field
             data = self._parse_import_data(data, import_fields, options)
         except ValueError as error:
-            return [{
-                'type': 'error',
-                'message': pycompat.text_type(error),
-                'record': False,
-            }]
+            return {
+                'messages': [{
+                    'type': 'error',
+                    'message': pycompat.text_type(error),
+                    'record': False,
+                }]
+            }
 
         _logger.info('importing %d rows...', len(data))
 
@@ -698,4 +757,20 @@ class Import(models.TransientModel):
         except psycopg2.InternalError:
             pass
 
-        return import_result['messages']
+        # Insert/Update mapping columns when import complete successfully
+        if import_result['ids'] and options.get('headers'):
+            BaseImportMapping = self.env['base_import.mapping']
+            for index, column_name in enumerate(columns):
+                if column_name:
+                    # Update to latest selected field
+                    exist_records = BaseImportMapping.search([('res_model', '=', self.res_model), ('column_name', '=', column_name)])
+                    if exist_records:
+                        exist_records.write({'field_name': fields[index]})
+                    else:
+                        BaseImportMapping.create({
+                            'res_model': self.res_model,
+                            'column_name': column_name,
+                            'field_name': fields[index]
+                        })
+
+        return import_result
diff --git a/addons/base_import/security/ir.model.access.csv b/addons/base_import/security/ir.model.access.csv
index 7ed063df1c2..6844f024d3e 100644
--- a/addons/base_import/security/ir.model.access.csv
+++ b/addons/base_import/security/ir.model.access.csv
@@ -12,4 +12,5 @@ access_base_import_tests_models_m2o_required_related,base.import.tests.models.m2
 access_base_import_tests_models_o2m,base.import.tests.models.o2m,model_base_import_tests_models_o2m,base.group_user,1,1,1,1
 access_base_import_tests_models_o2m_child,base.import.tests.models.o2m.child,model_base_import_tests_models_o2m_child,base.group_user,1,1,1,1
 access_base_import_tests_models_preview,base.import.tests.models.preview,model_base_import_tests_models_preview,base.group_user,1,1,1,1
+access_base_import_mapping,base.import.mapping,model_base_import_mapping,base.group_user,1,1,1,1
 
diff --git a/addons/base_import/static/csv/res.partner.csv b/addons/base_import/static/csv/res.partner.csv
deleted file mode 100644
index 5414512f4df..00000000000
--- a/addons/base_import/static/csv/res.partner.csv
+++ /dev/null
@@ -1,7 +0,0 @@
-ID,Name,Company Type,Related Company,Customer,Supplier,Address Type,Street,Street2,City,State,ZIP,Country,Website,Phone,Mobile,Email
-res_partner_1,Grocery Outlet,Company,,TRUE,FALSE,,51 Federal Street,#401,San Francisco,California,94107,United States,groceryoutlet.com,123456789,,
-res_partner_2,Bob,Individual,Grocery Outlet,,,Contact,,,,,,,,,4157654321,
-res_partner_3,Bill,Individual,Grocery Outlet,,,Other address,645 21st Street,,San Francisco,California,94110,United States,,,4517864512,
-res_partner_4,Warehouse,Individual,Grocery Outlet,,,Shipping address,645 21st Street,,San Francisco,California,94110,United States,,415659874,,bill@odoo.com
-res_partner_5,Accounting dep.,Individual,Grocery Outlet,,,Invoice address,51 Federal Street,#404,San Francisco,California,94107,United States,,123456780,,
-res_partner_6,Agrolait,Company,,TRUE,FALSE,,3442 2nd Avenue,,New York,New York,11201,United States,agrolait.com,654978123,,
diff --git a/addons/base_import/static/src/js/import_action.js b/addons/base_import/static/src/js/import_action.js
index d995e9cf055..68dc6c663fa 100644
--- a/addons/base_import/static/src/js/import_action.js
+++ b/addons/base_import/static/src/js/import_action.js
@@ -47,7 +47,6 @@ function jsonp(form, attributes, callback) {
 var DataImport = AbstractAction.extend(ControlPanelMixin, {
     template: 'ImportView',
     opts: [
-        {name: 'encoding', label: _lt("Encoding:"), value: 'utf-8'},
         {name: 'separator', label: _lt("Separator:"), value: ','},
         {name: 'quoting', label: _lt("Text Delimiter:"), value: '"'}
     ],
@@ -60,7 +59,6 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     events: {
         // 'change .oe_import_grid input': 'import_dryrun',
         'change .oe_import_file': 'loaded_file',
-        'click .oe_import_file_reload': 'loaded_file',
         'change input.oe_import_has_header, .js_import_options input': 'settings_changed',
         'change input.oe_import_advanced_mode': function (e) {
             this.do_not_change_match = true;
@@ -111,9 +109,21 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         action.display_name = _t('Import a File'); // Displayed in the breadcrumbs
         this.do_not_change_match = false;
     },
+    /**
+     * @override
+     */
+    willStart: function () {
+        var self = this;
+        return this._rpc({
+            model: this.res_model,
+            method: 'get_import_templates',
+            context: this.parent_context,
+        }).then(function (result) {
+            self.importTemplates = result;
+        });
+    },
     start: function () {
         var self = this;
-        this.setup_encoding_picker();
         this.setup_separator_picker();
         this.setup_float_format_picker();
 
@@ -124,7 +134,6 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
                 self.$('input[name=import_id]').val(id);
 
                 self.renderButtons();
-                self.renderImportLink();
                 var status = {
                     cp_content: {$buttons: self.$buttons},
                 };
@@ -145,36 +154,15 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         this.$buttons = $(QWeb.render("ImportView.buttons", this));
         this.$buttons.filter('.o_import_validate').on('click', this.validate.bind(this));
         this.$buttons.filter('.o_import_import').on('click', this.import.bind(this));
+        this.$buttons.filter('.o_import_file_reload').on('click', this.loaded_file.bind(this));
+        this.$buttons.filter('.oe_import_file').on('click', function () {
+            self.$('.oe_import_file').click();
+        });
         this.$buttons.filter('.o_import_cancel').on('click', function(e) {
             e.preventDefault();
             self.exit();
         });
     },
-    renderImportLink: function() {
-        if (this.res_model == 'res.partner') {
-            this.$(".import-link").prop({"text": _t(" Import Template for Customers"), "href": "/base_import/static/csv/res.partner.csv"});
-            this.$(".template-import").removeClass("hidden");
-        }
-    },
-    setup_encoding_picker: function () {
-        this.$('input.oe_import_encoding').select2({
-            width: '160px',
-            query: function (q) {
-                var make = function (term) { return {id: term, text: term}; };
-                var suggestions = _.map(
-                    ('utf-8 utf-16 windows-1252 latin1 latin2 big5 ' +
-                     'gb18030 shift_jis windows-1251 koir8_r').split(/\s+/),
-                    make);
-                if (q.term) {
-                    suggestions.unshift(make(q.term));
-                }
-                q.callback({results: suggestions});
-            },
-            initSelection: function (e, c) {
-                return c({id: 'utf-8', text: 'utf-8'});
-            }
-        }).select2('val', 'utf-8');
-    },
     setup_separator_picker: function () {
         this.$('input.oe_import_separator').select2({
             width: '160px',
@@ -196,26 +184,33 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         });
     },
     setup_float_format_picker: function () {
-        var sug_query = function (q) {
-                    var suggestions = [
-                        {id: ',', text: _t("Comma")},
-                        {id: '.', text: _t("Dot")},
-                    ];
-                    if (q.term) {
-                        suggestions.unshift({id: q.term, text: q.term});
-                    }
-                    q.callback({results: suggestions});
-                };
+        var sub_query = function (q, options) {
+            var suggestions = [
+                {id: ',', text: _t("Comma")},
+                {id: '.', text: _t("Dot")},
+            ];
+            if (options && options.no_separator) {
+                suggestions.push({id: '', text: _t("No Separator")});
+            }
+            if (q.term) {
+                suggestions.unshift({id: q.term, text: q.term});
+            }
+            q.callback({results: suggestions});
+        };
         this.$('input.oe_import_float_thousand_separator').select2({
             width: '160px',
-            query: sug_query,
+            query: function (q) {
+                sub_query(q, {no_separator: true})
+            },
             initSelection: function (e, c) {
                 return c({id: ',', text: _t("Comma")});
             },
         });
         this.$('input.oe_import_float_decimal_separator').select2({
             width: '160px',
-            query: sug_query,
+            query: function (q) {
+                sub_query(q);
+            },
             initSelection: function (e, c) {
                 return c({id: ',', text: _t("Dot")});
             },
@@ -254,9 +249,7 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     //- File & settings change section
     onfile_loaded: function () {
         var file = this.$('.oe_import_file')[0].files[0];
-        this.$('.oe_import_file_show').val(file !== undefined && file.name || '');
-        this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload'))
-                .prop('disabled', true);
+        this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('hidden');
         if (!this.$('input.oe_import_file').val()) { return this['settings_changed'](); }
         this.$('.oe_import_date_format').val('');
         this.$('.oe_import_datetime_format').val('');
@@ -268,15 +261,14 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         if ((file.type && _.last(file.type.split('/')) === "csv") || ( _.last(file.name.split('.')) === "csv")) {
             import_toggle = true;
         }
-        this.$el.find('.oe_import_toggle').toggle(import_toggle);
+        this.$el.find('.oe_import_box').toggle(import_toggle);
         jsonp(this.$el, {
             url: '/base_import/set_file'
         }, this.proxy('settings_changed'));
     },
     onpreviewing: function () {
         var self = this;
-        this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload'))
-                .prop('disabled', true);
+        this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('hidden');
         this.$el.addClass('oe_import_with_file');
         // TODO: test that write // succeeded?
         this.$el.removeClass('oe_import_preview_error oe_import_error');
@@ -295,26 +287,32 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     },
     onpreview_error: function (event, from, to, result) {
         this.$('.oe_import_options').show();
-        this.$('.oe_import_file_reload').prop('disabled', false);
+        this.$buttons.filter('.o_import_file_reload').removeClass('hidden');
         this.$el.addClass('oe_import_preview_error oe_import_error');
+        this.$el.find('.oe_import_box, .oe_import_with_file').removeClass('hidden');
+        this.$el.find('.o_view_nocontent').addClass('hidden');
         this.$('.oe_import_error_report').html(
                 QWeb.render('ImportView.preview.error', result));
     },
     onpreview_success: function (event, from, to, result) {
         var self = this;
-        this.$buttons.filter('.o_import_import').removeClass('btn-primary');
-        this.$buttons.filter('.o_import_validate').addClass('btn-primary');
-        this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload'))
-                .prop('disabled', false);
+        this.$buttons.filter('.oe_import_file')
+            .text(_t('Load New File'))
+            .removeClass('btn-primary').addClass('btn-default')
+            .blur();
+        this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').removeClass('hidden');
+        this.$el.find('.oe_import_box, .oe_import_with_file').removeClass('hidden');
+        this.$el.find('.o_view_nocontent').addClass('hidden');
         this.$el.addClass('oe_import_preview');
+        this.$('input.oe_import_advanced_mode').prop('checked', result.advanced_mode);
         this.$('.oe_import_grid').html(QWeb.render('ImportView.preview', result));
 
         if (result.headers.length === 1) {
             this.$('.oe_import_options').show();
-            this.onresults(null, null, null, [{
+            this.onresults(null, null, null, {'messages': [{
                 type: 'warning',
                 message: _t("A single column was found in the file, this often means the file separator is incorrect")
-            }]);
+            }]});
         }
 
         this.$('.oe_import_date_format').val(time.strftime_to_moment_format(result.options.date_format));
@@ -461,6 +459,9 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         var fields = this.$('.oe_import_fields input.oe_import_match_field').map(function (index, el) {
             return $(el).select2('val') || false;
         }).get();
+        var columns = this.$('.oe_import_grid-header .oe_import_grid-cell .o_import_header_name').map(function () {
+            return $(this).text().trim().toLowerCase() || false;
+        }).get();
         var tracking_disable = 'tracking_disable' in kwargs ? kwargs.tracking_disable : !this.$('#oe_import_tracking').prop('checked')
         var defer_parent_store = 'defer_parent_store' in kwargs ? kwargs.defer_parent_store : !!this.$('#oe_import_deferparentstore').prop('checked')
         delete kwargs.tracking_disable;
@@ -472,18 +473,18 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         return this._rpc({
                 model: 'base_import.import',
                 method: 'do',
-                args: [this.id, fields, this.import_options()],
+                args: [this.id, fields, columns, this.import_options()],
                 kwargs : kwargs,
             }).fail(function (error, event) {
                 // In case of unexpected exception, convert
                 // "JSON-RPC error" to an import failure, and
                 // prevent default handling (warning dialog)
                 if (event) { event.preventDefault(); }
-                return $.when([{
+                return $.when({'messages': [{
                     type: 'error',
                     record: false,
                     message: error.data.arguments && error.data.arguments[1] || error.message,
-                }]);
+                }]});
             }) ;
     },
     onvalidate: function () {
@@ -492,27 +493,26 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     },
     onimport: function () {
         var self = this;
-        return this.call_import({ dryrun: false }).done(function (message) {
+        return this.call_import({ dryrun: false }).done(function (results) {
+            var message = results.messages;
             if (!_.any(message, function (message) {
                     return message.type === 'error'; })) {
-                self['import_succeeded']();
+                self['import_succeeded'](results);
                 return;
             }
-            self['import_failed'](message);
+            self['import_failed'](results);
         });
     },
-    onimported: function () {
+    onimported: function (event, from, to, results) {
+        this.do_notify(_t("Import completed"), _.str.sprintf(_t("%d records were successfully imported"), results.ids.length));
         this.exit();
     },
     exit: function () {
         this.trigger_up('history_back');
     },
-    onresults: function (event, from, to, message) {
+    onresults: function (event, from, to, results) {
+        var message = results.messages;
         var no_messages = _.isEmpty(message);
-        this.$buttons.filter('.o_import_import').toggleClass('btn-primary', no_messages);
-        this.$buttons.filter('.o_import_import').toggleClass('btn-default', !no_messages);
-        this.$buttons.filter('.o_import_validate').toggleClass('btn-primary', !no_messages);
-        this.$buttons.filter('.o_import_validate').toggleClass('btn-default', no_messages);
         if (no_messages) {
             message.push({
                 type: 'info',
@@ -591,7 +591,7 @@ StateMachine.create({
         { name: 'import', from: ['preview_success', 'results'], to: 'importing' },
         { name: 'import_succeeded', from: 'importing', to: 'imported'},
         { name: 'import_failed', from: 'importing', to: 'results' }
-    ]
+    ],
 });
 
 return {
diff --git a/addons/base_import/static/src/scss/base_import.scss b/addons/base_import/static/src/scss/base_import.scss
index 9a4ce56f5c2..1ba7b05043f 100644
--- a/addons/base_import/static/src/scss/base_import.scss
+++ b/addons/base_import/static/src/scss/base_import.scss
@@ -1,6 +1,5 @@
 .oe_import {
     @include o-webclient-padding($top: 8px);
-    background-color: white;
     overflow: auto;
     position: absolute; // Needed for chrome
     top: 0;
@@ -27,7 +26,7 @@
     }
 
     .oe_import_box {
-        padding: 16px;
+        padding: 8px;
         background: #F0EEEE;
         border-radius: $border-radius-base;
         border: solid 1px #dddddd;
@@ -39,9 +38,6 @@
         }
     }
 
-    .oe_import_toggle {
-        margin-top: 8px;
-    }
     a.oe_import_toggle {
         display: block;
         &:before {
@@ -94,6 +90,12 @@
             margin: 2px 0;
             list-style: none;
             border-radius: $border-radius-base;
+            &.bg-error {
+                @extend .bg-danger;
+            }
+            &.text-error {
+                @extend .text-danger;
+            }
         }
     }
 
diff --git a/addons/base_import/static/src/xml/base_import.xml b/addons/base_import/static/src/xml/base_import.xml
index fcd5b239cb6..df9fbc3e5d1 100644
--- a/addons/base_import/static/src/xml/base_import.xml
+++ b/addons/base_import/static/src/xml/base_import.xml
@@ -6,34 +6,12 @@
             <input type="hidden" name="session_id"
                t-att-value="widget.session.session_id"/>
             <input type="hidden" name="import_id"/>
-            <div class="oe_import_box col-sm-9">
-                <div class="col-sm-12">
-                    <p>Select a CSV or Excel file to import. <a href="https://www.odoo.com/documentation/user/11.0/general/base_import/import_faq.html" target="new" class="pull-right">Help</a></p>
-                </div>
-                <div class="col-sm-10">
-                    <div class="input-group">
-                      <input type="text" class="oe_import_file_show form-control" placeholder="No file chosen..."/>
-                      <span class="input-group-btn">
-                        <button class="btn btn-primary">
-                        <input accept=".csv, .xls, .xlsx, .xlsm, .ods" id-attf-id="file_#{_id}"
-                       name="file" id="my-file-selector" class="oe_import_file" type="file" style="display:none;"/>
-                        Load File
-                        </button>
-                      </span>
-                      <span class="input-group-btn">
-                        <button type="button" class="btn btn-default oe_import_file_reload" disabled="disabled">Reload File</button>
-                      </span>
-                    </div>
-                    <div class="template-import mt24 hidden">
-                        <h5>
-                            <a href="#" class="import-link fa fa-download" aria-label="Download" title="Download"></a>
-                        </h5>
-                    </div>
-                </div>
-
-                <div class="oe_import_with_file col-sm-12">
+            <div class="oe_import_box col-md-12 hidden">
+                <input accept=".csv, .xls, .xlsx, .xlsm, .ods" t-attf-id="file_#{_id}"
+                      name="file" class="oe_import_file" type="file" style="display:none;"/>
+                <div class="oe_import_with_file">
                     <a href="#" class="oe_import_toggle">
-                        Options…</a>
+                        Formatting Options…</a>
                     <div class="oe_import_toggled oe_import_options js_import_options col-sm-5">
                         <p t-foreach="widget.opts" t-as="option">
                             <!-- no @name, avoid submission when file_update called -->
@@ -57,7 +35,7 @@
                 </div>
             </div>
 
-            <div class="oe_import_with_file oe_padding col-sm-12">
+            <div class="oe_import_with_file oe_padding col-sm-12 hidden">
                 <h2>Map your columns to import</h2>
                 <div title="If the model uses openchatter, history tracking
                             will set up subscriptions and send notifications
@@ -90,16 +68,35 @@
                 simpler especially when the file has many columns.</p>
 
                 <div class="oe_import_error_report"></div>
-                <table class="table-striped table-bordered oe_import_grid" />
+                <div class="table-responsive">
+                    <table class="table-striped table-bordered oe_import_grid bg-white" />
+                </div>
+            </div>
+            <div class="o_view_nocontent">
+                <div class="o_nocontent_help">
+                    <p class="o_view_nocontent_smiling_face">
+                        Select a CSV or Excel file to import.
+                    </p>
+                    <p>
+                        Excel files are recommended as fields formatting is automatic.
+                    </p>
+                    <div class="mt16 mb4">Need Help?</div>
+                    <div t-foreach="widget.importTemplates" t-as="template">
+                        <a t-att-href="template.template" aria-label="Download" title="Download">
+                            <i class="fa fa-download"/> <span><t t-esc="template.label"/></span>
+                        </a>
+                    </div>
+                    <a href="https://www.odoo.com/documentation/user/11.0/general/base_import/import_faq.html" target="new">Import FAQ</a>
+                </div>
             </div>
         </form>
     </t>
 
     <t t-name="ImportView.buttons">
-        <button type="button" class="btn btn-sm btn-primary o_import_button o_import_validate"
-                disabled="disabled">Test Import</button>
-        <button type="button" class="btn btn-sm btn-default o_import_button o_import_import"
-                disabled="disabled">Import</button>
+        <button type="button" class="btn btn-sm btn-primary o_import_import hidden">Import</button>
+        <button type="button" class="btn btn-sm btn-default o_import_validate hidden">Test Import</button>
+        <button type="button" class="btn btn-sm btn-primary oe_import_file">Load File</button>
+        <button type="button" class="btn btn-sm btn-default o_import_file_reload hidden">Reload File</button>
         <button type="button" class="btn btn-sm btn-default o_import_cancel">Cancel</button>
     </t>
 
@@ -107,7 +104,7 @@
         <thead>
             <tr t-if="headers" class="oe_import_grid-header">
                 <td t-foreach="headers" t-as="header" class="oe_import_grid-cell"
-                    ><t t-esc="header"/></td>
+                    ><span class="o_import_header_name"><t t-esc="header"/></span></td>
             </tr>
             <tr class="oe_import_fields">
                 <!-- Iterate on first row to ensure we have all columns -->
@@ -126,7 +123,7 @@
     <t t-name="ImportView.preview.error">
         <div class="oe_import_report bg-danger text-danger">
             <p>Import preview failed due to: <t t-esc="error"/>.</p>
-            <p>For CSV files, the issue could be an incorrect encoding.</p>
+            <p>For CSV files, you may need to select the correct separator.</p>
             <p t-if="preview">Here is the start of the file we could not import:</p>
         </div>
         <pre t-if="preview"><t t-esc="preview"/></pre>
diff --git a/addons/base_import/tests/test_base_import.py b/addons/base_import/tests/test_base_import.py
index df2d18a3feb..3f5830a8db2 100644
--- a/addons/base_import/tests/test_base_import.py
+++ b/addons/base_import/tests/test_base_import.py
@@ -189,6 +189,31 @@ class TestMatchHeadersMultiple(TransactionCase):
         )
 
 
+class TestColumnMapping(TransactionCase):
+
+    def test_column_mapping(self):
+        import_record = self.env['base_import.import'].create({
+            'res_model': 'base_import.tests.models.preview',
+            'file': u"Name,Some Value,value\n"
+                    u"chhagan,10,1\n"
+                    u"magan,20,2\n".encode('utf-8'),
+            'file_type': 'text/csv',
+            'file_name': 'data.csv',
+        })
+        import_record.do(
+            ['name', 'somevalue', 'othervalue'],
+            ['Name', 'Some Value', 'value'],
+            {'quoting': '"', 'separator': ',', 'headers': True},
+            True
+        )
+        fields = self.env['base_import.mapping'].search_read(
+            [('res_model', '=', 'base_import.tests.models.preview')],
+            ['column_name', 'field_name']
+        )
+        self.assertItemsEqual([f['column_name'] for f in fields], ['Name', 'Some Value', 'value'])
+        self.assertItemsEqual([f['field_name'] for f in fields], ['somevalue', 'name', 'othervalue'])
+
+
 class TestPreview(TransactionCase):
 
     def make_import(self):
@@ -207,7 +232,7 @@ class TestPreview(TransactionCase):
             'quoting': '"',
             'separator': ',',
         })
-        self.assertTrue('error' in result)
+        self.assertFalse('error' in result)
 
     @mute_logger('odoo.addons.base_import.models.base_import')
     def test_csv_errors(self):
@@ -216,14 +241,12 @@ class TestPreview(TransactionCase):
         result = import_wizard.parse_preview({
             'quoting': 'foo',
             'separator': ',',
-            'encoding': 'euc_kr',
         })
         self.assertTrue('error' in result)
 
         result = import_wizard.parse_preview({
             'quoting': '"',
             'separator': 'bob',
-            'encoding': 'euc_kr',
         })
         self.assertTrue('error' in result)
 
@@ -258,7 +281,7 @@ class TestPreview(TransactionCase):
             ['qux', '5', '6'],
         ])
         # Ensure we only have the response fields we expect
-        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'debug'])
+        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'advanced_mode', 'debug'])
 
     @unittest.skipUnless(can_import('xlrd'), "XLRD module not available")
     def test_xls_success(self):
@@ -288,7 +311,7 @@ class TestPreview(TransactionCase):
             ['qux', '5', '6'],
         ])
         # Ensure we only have the response fields we expect
-        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'debug'])
+        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'advanced_mode', 'debug'])
 
     @unittest.skipUnless(can_import('xlrd.xlsx'), "XLRD/XLSX not available")
     def test_xlsx_success(self):
@@ -318,7 +341,7 @@ class TestPreview(TransactionCase):
             ['qux', '5', '6'],
         ])
         # Ensure we only have the response fields we expect
-        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'debug'])
+        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'advanced_mode', 'debug'])
 
     @unittest.skipUnless(can_import('odf'), "ODFPY not available")
     def test_ods_success(self):
@@ -348,7 +371,7 @@ class TestPreview(TransactionCase):
             ['aux', '5', '6'],
         ])
         # Ensure we only have the response fields we expect
-        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'debug'])
+        self.assertItemsEqual(list(result), ['matches', 'headers', 'fields', 'preview', 'headers_type', 'options', 'advanced_mode', 'debug'])
 
 
 class test_convert_import_data(TransactionCase):
@@ -388,6 +411,7 @@ class test_convert_import_data(TransactionCase):
 
         results = import_wizard.do(
             ['name', 'date', 'create_date'],
+            [],
             {
                 'date_format': '%Y年%m月%d日',
                 'datetime_format': '%Y-%m-%d %H:%M',
@@ -398,7 +422,7 @@ class test_convert_import_data(TransactionCase):
         )
 
         # if results empty, no errors
-        self.assertItemsEqual(results, [])
+        self.assertItemsEqual(results['messages'], [])
 
     def test_parse_relational_fields(self):
         """ Ensure that relational fields float and date are correctly
@@ -570,5 +594,6 @@ class test_failures(TransactionCase):
         })
         results = import_wizard.do(
             ['name', 'db_datas'],
+            [],
             {'headers': True, 'separator': ',', 'quoting': '"'})
-        self.assertFalse(results, "results should be empty on successful import")
+        self.assertFalse(results['messages'], "results should be empty on successful import")