[MERGE] forward port branch 12.0 up to e11bacfe512

2 files changed, 26 insertions, 3 deletions

diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py
index 5e75af21f91..36e76ef7f04 100644
--- a/addons/base_import/models/base_import.py
+++ b/addons/base_import/models/base_import.py
@@ -2,6 +2,7 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import base64
+import codecs
 import collections
 import unicodedata
 
@@ -32,6 +33,12 @@ DEFAULT_IMAGE_REGEX = r"(?:http|https)://.*(?:png|jpe?g|tiff?|gif|bmp)"
 DEFAULT_IMAGE_CHUNK_SIZE = 32768
 IMAGE_FIELDS = ["icon", "image", "logo", "picture"]
 _logger = logging.getLogger(__name__)
+BOM_MAP = {
+    'utf-16le': codecs.BOM_UTF16_LE,
+    'utf-16be': codecs.BOM_UTF16_BE,
+    'utf-32le': codecs.BOM_UTF32_LE,
+    'utf-32be': codecs.BOM_UTF32_BE,
+}
 
 try:
     import xlrd
@@ -321,6 +328,13 @@ class Import(models.TransientModel):
         encoding = options.get('encoding')
         if not encoding:
             encoding = options['encoding'] = chardet.detect(csv_data)['encoding'].lower()
+            # some versions of chardet (e.g. 2.3.0 but not 3.x) will return
+            # utf-(16|32)(le|be), which for python means "ignore / don't strip
+            # BOM". We don't want that, so rectify the encoding to non-marked
+            # IFF the guessed encoding is LE/BE and csv_data starts with a BOM
+            bom = BOM_MAP.get(encoding)
+            if bom and csv_data.startswith(bom):
+                encoding = options['encoding'] = encoding[:-2]
 
         if encoding != 'utf-8':
             csv_data = csv_data.decode(encoding).encode('utf-8')
diff --git a/addons/base_import/tests/test_csv_magic.py b/addons/base_import/tests/test_csv_magic.py
index 0a35a4731c1..b7975337361 100644
--- a/addons/base_import/tests/test_csv_magic.py
+++ b/addons/base_import/tests/test_csv_magic.py
@@ -26,18 +26,27 @@ class TestEncoding(ImportCase):
         options.setdefault('separator', '\t')
         test_text = "text\tnumber\tdate\tdatetime\n%s\t1.23.45,67\t\t\n" % text
         for encoding in ['utf-8', 'utf-16', 'utf-32', *encodings]:
+            if isinstance(encoding, tuple):
+                encoding, es = encoding
+            else:
+                es = [encoding]
             preview = self._make_import(
                 test_text.encode(encoding)).parse_preview(dict(options))
 
             self.assertIsNone(preview.get('error'))
             guessed = preview['options']['encoding']
-            self.assertIsNotNone(guessed, encoding)
-            self.assertEqual(codecs.lookup(guessed).name, codecs.lookup(encoding).name)
+            self.assertIsNotNone(guessed)
+            self.assertIn(
+                codecs.lookup(guessed).name, [
+                    codecs.lookup(e).name
+                    for e in es
+                ]
+            )
 
     def test_autodetect_encoding(self):
         """ Check that import preview can detect & return encoding
         """
-        self._check_text("Iñtërnâtiônàlizætiøn", ['iso-8859-1'])
+        self._check_text("Iñtërnâtiônàlizætiøn", [('iso-8859-1', ['iso-8859-1', 'iso-8859-2'])])
 
         self._check_text("やぶら小路の藪柑子。海砂利水魚の、食う寝る処に住む処、パイポパイポ パイポのシューリンガン。", ['eucjp', 'shift_jis', 'iso2022_jp'])