[IMP] import/export UIs

Task 40692

Various changes to import/export (mainly) UIs:

* default to excel & "full" (non-import-compatible) export
* auto-detect encoding of CSV using chardet
* remember column -> field mapping after having imported a file (useful
   for repeated imports where auto-matching failed)
* better handle localised booleans & column names
* automatically select source list view's fields when exporting
* better integrate import templates feature and add a number of templates

4 files changed, 106 insertions, 114 deletions

diff --git a/addons/base_import/static/csv/res.partner.csv b/addons/base_import/static/csv/res.partner.csv
deleted file mode 100644
index 5414512f4df..00000000000
--- a/addons/base_import/static/csv/res.partner.csv
+++ /dev/null
@@ -1,7 +0,0 @@
-ID,Name,Company Type,Related Company,Customer,Supplier,Address Type,Street,Street2,City,State,ZIP,Country,Website,Phone,Mobile,Email
-res_partner_1,Grocery Outlet,Company,,TRUE,FALSE,,51 Federal Street,#401,San Francisco,California,94107,United States,groceryoutlet.com,123456789,,
-res_partner_2,Bob,Individual,Grocery Outlet,,,Contact,,,,,,,,,4157654321,
-res_partner_3,Bill,Individual,Grocery Outlet,,,Other address,645 21st Street,,San Francisco,California,94110,United States,,,4517864512,
-res_partner_4,Warehouse,Individual,Grocery Outlet,,,Shipping address,645 21st Street,,San Francisco,California,94110,United States,,415659874,,bill@odoo.com
-res_partner_5,Accounting dep.,Individual,Grocery Outlet,,,Invoice address,51 Federal Street,#404,San Francisco,California,94107,United States,,123456780,,
-res_partner_6,Agrolait,Company,,TRUE,FALSE,,3442 2nd Avenue,,New York,New York,11201,United States,agrolait.com,654978123,,
diff --git a/addons/base_import/static/src/js/import_action.js b/addons/base_import/static/src/js/import_action.js
index d995e9cf055..68dc6c663fa 100644
--- a/addons/base_import/static/src/js/import_action.js
+++ b/addons/base_import/static/src/js/import_action.js
@@ -47,7 +47,6 @@ function jsonp(form, attributes, callback) {
 var DataImport = AbstractAction.extend(ControlPanelMixin, {
     template: 'ImportView',
     opts: [
-        {name: 'encoding', label: _lt("Encoding:"), value: 'utf-8'},
         {name: 'separator', label: _lt("Separator:"), value: ','},
         {name: 'quoting', label: _lt("Text Delimiter:"), value: '"'}
     ],
@@ -60,7 +59,6 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     events: {
         // 'change .oe_import_grid input': 'import_dryrun',
         'change .oe_import_file': 'loaded_file',
-        'click .oe_import_file_reload': 'loaded_file',
         'change input.oe_import_has_header, .js_import_options input': 'settings_changed',
         'change input.oe_import_advanced_mode': function (e) {
             this.do_not_change_match = true;
@@ -111,9 +109,21 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         action.display_name = _t('Import a File'); // Displayed in the breadcrumbs
         this.do_not_change_match = false;
     },
+    /**
+     * @override
+     */
+    willStart: function () {
+        var self = this;
+        return this._rpc({
+            model: this.res_model,
+            method: 'get_import_templates',
+            context: this.parent_context,
+        }).then(function (result) {
+            self.importTemplates = result;
+        });
+    },
     start: function () {
         var self = this;
-        this.setup_encoding_picker();
         this.setup_separator_picker();
         this.setup_float_format_picker();
 
@@ -124,7 +134,6 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
                 self.$('input[name=import_id]').val(id);
 
                 self.renderButtons();
-                self.renderImportLink();
                 var status = {
                     cp_content: {$buttons: self.$buttons},
                 };
@@ -145,36 +154,15 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         this.$buttons = $(QWeb.render("ImportView.buttons", this));
         this.$buttons.filter('.o_import_validate').on('click', this.validate.bind(this));
         this.$buttons.filter('.o_import_import').on('click', this.import.bind(this));
+        this.$buttons.filter('.o_import_file_reload').on('click', this.loaded_file.bind(this));
+        this.$buttons.filter('.oe_import_file').on('click', function () {
+            self.$('.oe_import_file').click();
+        });
         this.$buttons.filter('.o_import_cancel').on('click', function(e) {
             e.preventDefault();
             self.exit();
         });
     },
-    renderImportLink: function() {
-        if (this.res_model == 'res.partner') {
-            this.$(".import-link").prop({"text": _t(" Import Template for Customers"), "href": "/base_import/static/csv/res.partner.csv"});
-            this.$(".template-import").removeClass("hidden");
-        }
-    },
-    setup_encoding_picker: function () {
-        this.$('input.oe_import_encoding').select2({
-            width: '160px',
-            query: function (q) {
-                var make = function (term) { return {id: term, text: term}; };
-                var suggestions = _.map(
-                    ('utf-8 utf-16 windows-1252 latin1 latin2 big5 ' +
-                     'gb18030 shift_jis windows-1251 koir8_r').split(/\s+/),
-                    make);
-                if (q.term) {
-                    suggestions.unshift(make(q.term));
-                }
-                q.callback({results: suggestions});
-            },
-            initSelection: function (e, c) {
-                return c({id: 'utf-8', text: 'utf-8'});
-            }
-        }).select2('val', 'utf-8');
-    },
     setup_separator_picker: function () {
         this.$('input.oe_import_separator').select2({
             width: '160px',
@@ -196,26 +184,33 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         });
     },
     setup_float_format_picker: function () {
-        var sug_query = function (q) {
-                    var suggestions = [
-                        {id: ',', text: _t("Comma")},
-                        {id: '.', text: _t("Dot")},
-                    ];
-                    if (q.term) {
-                        suggestions.unshift({id: q.term, text: q.term});
-                    }
-                    q.callback({results: suggestions});
-                };
+        var sub_query = function (q, options) {
+            var suggestions = [
+                {id: ',', text: _t("Comma")},
+                {id: '.', text: _t("Dot")},
+            ];
+            if (options && options.no_separator) {
+                suggestions.push({id: '', text: _t("No Separator")});
+            }
+            if (q.term) {
+                suggestions.unshift({id: q.term, text: q.term});
+            }
+            q.callback({results: suggestions});
+        };
         this.$('input.oe_import_float_thousand_separator').select2({
             width: '160px',
-            query: sug_query,
+            query: function (q) {
+                sub_query(q, {no_separator: true})
+            },
             initSelection: function (e, c) {
                 return c({id: ',', text: _t("Comma")});
             },
         });
         this.$('input.oe_import_float_decimal_separator').select2({
             width: '160px',
-            query: sug_query,
+            query: function (q) {
+                sub_query(q);
+            },
             initSelection: function (e, c) {
                 return c({id: ',', text: _t("Dot")});
             },
@@ -254,9 +249,7 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     //- File & settings change section
     onfile_loaded: function () {
         var file = this.$('.oe_import_file')[0].files[0];
-        this.$('.oe_import_file_show').val(file !== undefined && file.name || '');
-        this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload'))
-                .prop('disabled', true);
+        this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('hidden');
         if (!this.$('input.oe_import_file').val()) { return this['settings_changed'](); }
         this.$('.oe_import_date_format').val('');
         this.$('.oe_import_datetime_format').val('');
@@ -268,15 +261,14 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         if ((file.type && _.last(file.type.split('/')) === "csv") || ( _.last(file.name.split('.')) === "csv")) {
             import_toggle = true;
         }
-        this.$el.find('.oe_import_toggle').toggle(import_toggle);
+        this.$el.find('.oe_import_box').toggle(import_toggle);
         jsonp(this.$el, {
             url: '/base_import/set_file'
         }, this.proxy('settings_changed'));
     },
     onpreviewing: function () {
         var self = this;
-        this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload'))
-                .prop('disabled', true);
+        this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('hidden');
         this.$el.addClass('oe_import_with_file');
         // TODO: test that write // succeeded?
         this.$el.removeClass('oe_import_preview_error oe_import_error');
@@ -295,26 +287,32 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     },
     onpreview_error: function (event, from, to, result) {
         this.$('.oe_import_options').show();
-        this.$('.oe_import_file_reload').prop('disabled', false);
+        this.$buttons.filter('.o_import_file_reload').removeClass('hidden');
         this.$el.addClass('oe_import_preview_error oe_import_error');
+        this.$el.find('.oe_import_box, .oe_import_with_file').removeClass('hidden');
+        this.$el.find('.o_view_nocontent').addClass('hidden');
         this.$('.oe_import_error_report').html(
                 QWeb.render('ImportView.preview.error', result));
     },
     onpreview_success: function (event, from, to, result) {
         var self = this;
-        this.$buttons.filter('.o_import_import').removeClass('btn-primary');
-        this.$buttons.filter('.o_import_validate').addClass('btn-primary');
-        this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload'))
-                .prop('disabled', false);
+        this.$buttons.filter('.oe_import_file')
+            .text(_t('Load New File'))
+            .removeClass('btn-primary').addClass('btn-default')
+            .blur();
+        this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').removeClass('hidden');
+        this.$el.find('.oe_import_box, .oe_import_with_file').removeClass('hidden');
+        this.$el.find('.o_view_nocontent').addClass('hidden');
         this.$el.addClass('oe_import_preview');
+        this.$('input.oe_import_advanced_mode').prop('checked', result.advanced_mode);
         this.$('.oe_import_grid').html(QWeb.render('ImportView.preview', result));
 
         if (result.headers.length === 1) {
             this.$('.oe_import_options').show();
-            this.onresults(null, null, null, [{
+            this.onresults(null, null, null, {'messages': [{
                 type: 'warning',
                 message: _t("A single column was found in the file, this often means the file separator is incorrect")
-            }]);
+            }]});
         }
 
         this.$('.oe_import_date_format').val(time.strftime_to_moment_format(result.options.date_format));
@@ -461,6 +459,9 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         var fields = this.$('.oe_import_fields input.oe_import_match_field').map(function (index, el) {
             return $(el).select2('val') || false;
         }).get();
+        var columns = this.$('.oe_import_grid-header .oe_import_grid-cell .o_import_header_name').map(function () {
+            return $(this).text().trim().toLowerCase() || false;
+        }).get();
         var tracking_disable = 'tracking_disable' in kwargs ? kwargs.tracking_disable : !this.$('#oe_import_tracking').prop('checked')
         var defer_parent_store = 'defer_parent_store' in kwargs ? kwargs.defer_parent_store : !!this.$('#oe_import_deferparentstore').prop('checked')
         delete kwargs.tracking_disable;
@@ -472,18 +473,18 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
         return this._rpc({
                 model: 'base_import.import',
                 method: 'do',
-                args: [this.id, fields, this.import_options()],
+                args: [this.id, fields, columns, this.import_options()],
                 kwargs : kwargs,
             }).fail(function (error, event) {
                 // In case of unexpected exception, convert
                 // "JSON-RPC error" to an import failure, and
                 // prevent default handling (warning dialog)
                 if (event) { event.preventDefault(); }
-                return $.when([{
+                return $.when({'messages': [{
                     type: 'error',
                     record: false,
                     message: error.data.arguments && error.data.arguments[1] || error.message,
-                }]);
+                }]});
             }) ;
     },
     onvalidate: function () {
@@ -492,27 +493,26 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, {
     },
     onimport: function () {
         var self = this;
-        return this.call_import({ dryrun: false }).done(function (message) {
+        return this.call_import({ dryrun: false }).done(function (results) {
+            var message = results.messages;
             if (!_.any(message, function (message) {
                     return message.type === 'error'; })) {
-                self['import_succeeded']();
+                self['import_succeeded'](results);
                 return;
             }
-            self['import_failed'](message);
+            self['import_failed'](results);
         });
     },
-    onimported: function () {
+    onimported: function (event, from, to, results) {
+        this.do_notify(_t("Import completed"), _.str.sprintf(_t("%d records were successfully imported"), results.ids.length));
         this.exit();
     },
     exit: function () {
         this.trigger_up('history_back');
     },
-    onresults: function (event, from, to, message) {
+    onresults: function (event, from, to, results) {
+        var message = results.messages;
         var no_messages = _.isEmpty(message);
-        this.$buttons.filter('.o_import_import').toggleClass('btn-primary', no_messages);
-        this.$buttons.filter('.o_import_import').toggleClass('btn-default', !no_messages);
-        this.$buttons.filter('.o_import_validate').toggleClass('btn-primary', !no_messages);
-        this.$buttons.filter('.o_import_validate').toggleClass('btn-default', no_messages);
         if (no_messages) {
             message.push({
                 type: 'info',
@@ -591,7 +591,7 @@ StateMachine.create({
         { name: 'import', from: ['preview_success', 'results'], to: 'importing' },
         { name: 'import_succeeded', from: 'importing', to: 'imported'},
         { name: 'import_failed', from: 'importing', to: 'results' }
-    ]
+    ],
 });
 
 return {
diff --git a/addons/base_import/static/src/scss/base_import.scss b/addons/base_import/static/src/scss/base_import.scss
index 9a4ce56f5c2..1ba7b05043f 100644
--- a/addons/base_import/static/src/scss/base_import.scss
+++ b/addons/base_import/static/src/scss/base_import.scss
@@ -1,6 +1,5 @@
 .oe_import {
     @include o-webclient-padding($top: 8px);
-    background-color: white;
     overflow: auto;
     position: absolute; // Needed for chrome
     top: 0;
@@ -27,7 +26,7 @@
     }
 
     .oe_import_box {
-        padding: 16px;
+        padding: 8px;
         background: #F0EEEE;
         border-radius: $border-radius-base;
         border: solid 1px #dddddd;
@@ -39,9 +38,6 @@
         }
     }
 
-    .oe_import_toggle {
-        margin-top: 8px;
-    }
     a.oe_import_toggle {
         display: block;
         &:before {
@@ -94,6 +90,12 @@
             margin: 2px 0;
             list-style: none;
             border-radius: $border-radius-base;
+            &.bg-error {
+                @extend .bg-danger;
+            }
+            &.text-error {
+                @extend .text-danger;
+            }
         }
     }
 
diff --git a/addons/base_import/static/src/xml/base_import.xml b/addons/base_import/static/src/xml/base_import.xml
index fcd5b239cb6..df9fbc3e5d1 100644
--- a/addons/base_import/static/src/xml/base_import.xml
+++ b/addons/base_import/static/src/xml/base_import.xml
@@ -6,34 +6,12 @@
             <input type="hidden" name="session_id"
                t-att-value="widget.session.session_id"/>
             <input type="hidden" name="import_id"/>
-            <div class="oe_import_box col-sm-9">
-                <div class="col-sm-12">
-                    <p>Select a CSV or Excel file to import. <a href="https://www.odoo.com/documentation/user/11.0/general/base_import/import_faq.html" target="new" class="pull-right">Help</a></p>
-                </div>
-                <div class="col-sm-10">
-                    <div class="input-group">
-                      <input type="text" class="oe_import_file_show form-control" placeholder="No file chosen..."/>
-                      <span class="input-group-btn">
-                        <button class="btn btn-primary">
-                        <input accept=".csv, .xls, .xlsx, .xlsm, .ods" id-attf-id="file_#{_id}"
-                       name="file" id="my-file-selector" class="oe_import_file" type="file" style="display:none;"/>
-                        Load File
-                        </button>
-                      </span>
-                      <span class="input-group-btn">
-                        <button type="button" class="btn btn-default oe_import_file_reload" disabled="disabled">Reload File</button>
-                      </span>
-                    </div>
-                    <div class="template-import mt24 hidden">
-                        <h5>
-                            <a href="#" class="import-link fa fa-download" aria-label="Download" title="Download"></a>
-                        </h5>
-                    </div>
-                </div>
-
-                <div class="oe_import_with_file col-sm-12">
+            <div class="oe_import_box col-md-12 hidden">
+                <input accept=".csv, .xls, .xlsx, .xlsm, .ods" t-attf-id="file_#{_id}"
+                      name="file" class="oe_import_file" type="file" style="display:none;"/>
+                <div class="oe_import_with_file">
                     <a href="#" class="oe_import_toggle">
-                        Options…</a>
+                        Formatting Options…</a>
                     <div class="oe_import_toggled oe_import_options js_import_options col-sm-5">
                         <p t-foreach="widget.opts" t-as="option">
                             <!-- no @name, avoid submission when file_update called -->
@@ -57,7 +35,7 @@
                 </div>
             </div>
 
-            <div class="oe_import_with_file oe_padding col-sm-12">
+            <div class="oe_import_with_file oe_padding col-sm-12 hidden">
                 <h2>Map your columns to import</h2>
                 <div title="If the model uses openchatter, history tracking
                             will set up subscriptions and send notifications
@@ -90,16 +68,35 @@
                 simpler especially when the file has many columns.</p>
 
                 <div class="oe_import_error_report"></div>
-                <table class="table-striped table-bordered oe_import_grid" />
+                <div class="table-responsive">
+                    <table class="table-striped table-bordered oe_import_grid bg-white" />
+                </div>
+            </div>
+            <div class="o_view_nocontent">
+                <div class="o_nocontent_help">
+                    <p class="o_view_nocontent_smiling_face">
+                        Select a CSV or Excel file to import.
+                    </p>
+                    <p>
+                        Excel files are recommended as fields formatting is automatic.
+                    </p>
+                    <div class="mt16 mb4">Need Help?</div>
+                    <div t-foreach="widget.importTemplates" t-as="template">
+                        <a t-att-href="template.template" aria-label="Download" title="Download">
+                            <i class="fa fa-download"/> <span><t t-esc="template.label"/></span>
+                        </a>
+                    </div>
+                    <a href="https://www.odoo.com/documentation/user/11.0/general/base_import/import_faq.html" target="new">Import FAQ</a>
+                </div>
             </div>
         </form>
     </t>
 
     <t t-name="ImportView.buttons">
-        <button type="button" class="btn btn-sm btn-primary o_import_button o_import_validate"
-                disabled="disabled">Test Import</button>
-        <button type="button" class="btn btn-sm btn-default o_import_button o_import_import"
-                disabled="disabled">Import</button>
+        <button type="button" class="btn btn-sm btn-primary o_import_import hidden">Import</button>
+        <button type="button" class="btn btn-sm btn-default o_import_validate hidden">Test Import</button>
+        <button type="button" class="btn btn-sm btn-primary oe_import_file">Load File</button>
+        <button type="button" class="btn btn-sm btn-default o_import_file_reload hidden">Reload File</button>
         <button type="button" class="btn btn-sm btn-default o_import_cancel">Cancel</button>
     </t>
 
@@ -107,7 +104,7 @@
         <thead>
             <tr t-if="headers" class="oe_import_grid-header">
                 <td t-foreach="headers" t-as="header" class="oe_import_grid-cell"
-                    ><t t-esc="header"/></td>
+                    ><span class="o_import_header_name"><t t-esc="header"/></span></td>
             </tr>
             <tr class="oe_import_fields">
                 <!-- Iterate on first row to ensure we have all columns -->
@@ -126,7 +123,7 @@
     <t t-name="ImportView.preview.error">
         <div class="oe_import_report bg-danger text-danger">
             <p>Import preview failed due to: <t t-esc="error"/>.</p>
-            <p>For CSV files, the issue could be an incorrect encoding.</p>
+            <p>For CSV files, you may need to select the correct separator.</p>
             <p t-if="preview">Here is the start of the file we could not import:</p>
         </div>
         <pre t-if="preview"><t t-esc="preview"/></pre>