[IMP] base_import: reintroduce encoding selector & override guess

* if an encoding is explicitly specified, use it and don't guess
* otherwise guess and return the guessed encoding so it can be
  displayed in the configuration UI
* fix less-than-stellar configuration & behaviour of select2 inputs
  to properly reflect underlying values as they get modified, to
  correctly handle future configuration guesses

3 files changed, 75 insertions, 17 deletions

diff --git a/addons/base_import/tests/__init__.py b/addons/base_import/tests/__init__.py
index 1b9a63683a8..d0786a33fdb 100644
--- a/addons/base_import/tests/__init__.py
+++ b/addons/base_import/tests/__init__.py
@@ -2,3 +2,4 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from . import test_base_import
+from . import test_csv_magic
diff --git a/addons/base_import/tests/test_base_import.py b/addons/base_import/tests/test_base_import.py
index bb95bc52699..3f5830a8db2 100644
--- a/addons/base_import/tests/test_base_import.py
+++ b/addons/base_import/tests/test_base_import.py
@@ -452,23 +452,6 @@ class test_convert_import_data(TransactionCase):
         self.assertEqual(float(result[0][-1]), 5.69)
         self.assertEqual(str(result[0][-2]), '2017-10-12')
 
-    def test_parse_float(self):
-        w = self.env['base_import.import'].create({
-            'res_model': 'base_import.tests.models.float',
-        })
-        data = w._parse_import_data(
-            [
-                ['1.62'], ['-1.62'], ['+1.62'], ['  +1.62  '], ['(1.62)'],
-                ["1'234'567,89"], ["1.234.567'89"]
-            ],
-            ['value'], {}
-        )
-        self.assertEqual(
-            [d[0] for d in data],
-            ['1.62', '-1.62', '+1.62', '+1.62', '-1.62',
-             '1234567.89', '1234567.89']
-        )
-
     def test_filtered(self):
         """ If ``False`` is provided as field mapping for a column,
         that column should be removed from importable data
diff --git a/addons/base_import/tests/test_csv_magic.py b/addons/base_import/tests/test_csv_magic.py
new file mode 100644
index 00000000000..2b088c37afe
--- /dev/null
+++ b/addons/base_import/tests/test_csv_magic.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for various autodetection magics for CSV imports
+"""
+import codecs
+
+from odoo.tests import common
+
+class TestEncoding(common.TransactionCase):
+    """
+    create + parse_preview -> check result options
+    """
+
+    def _check_text(self, text, encodings, **options):
+        options.setdefault('quoting', '"')
+        options.setdefault('separator', '\t')
+        test_text = "text\tnumber\tdate\tdatetime\n%s\t1.23.45,67\t\t\n" % text
+        for encoding in ['utf-8', 'utf-16', 'utf-32', *encodings]:
+            preview = self._make_import(
+                test_text.encode(encoding)).parse_preview(dict(options))
+
+            guessed = preview['options']['encoding']
+            self.assertIsNotNone(guessed, encoding)
+            self.assertEqual(codecs.lookup(guessed).name, codecs.lookup(encoding).name)
+
+    def test_autodetect_encoding(self):
+        """ Check that import preview can detect & return encoding
+        """
+        self._check_text("Iñtërnâtiônàlizætiøn", ['iso-8859-1'])
+
+        self._check_text("やぶら小路の藪柑子。海砂利水魚の、食う寝る処に住む処、パイポパイポ パイポのシューリンガン。", ['eucjp', 'shift_jis', 'iso2022_jp'])
+
+        self._check_text("대통령은 제4항과 제5항의 규정에 의하여 확정된 법률을 지체없이 공포하여야 한다, 탄핵의 결정.", ['euc_kr', 'iso2022_kr'])
+
+    # + control in widget
+    def test_override_detection(self):
+        """ ensure an explicitly specified encoding is not overridden by the
+        auto-detection
+        """
+        s = "Iñtërnâtiônàlizætiøn".encode('utf-8')
+        r = self._make_import(b'text\n' + s)\
+            .parse_preview({
+            'quoting': '"',
+            'separator': '\t',
+            'encoding': 'iso-8859-1',
+        })
+        self.assertEqual(r['options']['encoding'], 'iso-8859-1')
+        self.assertEqual(r['preview'], [['text'], [s.decode('iso-8859-1')]])
+
+    def _make_import(self, contents):
+        return self.env['base_import.import'].create({
+            'res_model': 'base_import.tests.models.complex',
+            'file_name': 'f',
+            'file_type': 'text/csv',
+            'file': contents,
+        })
+
+class TestNumberSeparators(common.TransactionCase):
+    def test_parse_float(self):
+        w = self.env['base_import.import'].create({
+            'res_model': 'base_import.tests.models.float',
+        })
+        data = w._parse_import_data(
+            [
+                ['1.62'], ['-1.62'], ['+1.62'], ['  +1.62  '], ['(1.62)'],
+                ["1'234'567,89"], ["1.234.567'89"]
+            ],
+            ['value'], {}
+        )
+        self.assertEqual(
+            [d[0] for d in data],
+            ['1.62', '-1.62', '+1.62', '+1.62', '-1.62',
+             '1234567.89', '1234567.89']
+        )