From e12b6013d5e52bfbad93dc7fad40f17658490475 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 6 May 2019 09:16:25 +0000 Subject: [FIX] base_import: tests failing with older chardet Some distributions still bundle chardet 2.3, which have some guessing divergences / incompatibilities with python (resolved in 3.x): 1. UTF-{16,32} with BOM is guessed as LE/BE, which when used to decode the string doesn't strip out the BOM. Handle this by checking if the BOM is present and converting the encoding name to the non-marked version in that case. 2. The ISO-8859-1 test string is guessed as ISO-8859-2 (TBF the decoding does make some sense). Allow multiple targets/guesses to "fix" that. closes odoo/odoo#33179 Signed-off-by: Xavier Morel (xmo) --- addons/base_import/models/base_import.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'addons/base_import/models') diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index 4e4945fd37c..dd29bf396d1 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -2,6 +2,7 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 +import codecs import collections import unicodedata @@ -32,6 +33,12 @@ DEFAULT_IMAGE_REGEX = r"(?:http|https)://.*(?:png|jpe?g|tiff?|gif|bmp)" DEFAULT_IMAGE_CHUNK_SIZE = 32768 IMAGE_FIELDS = ["icon", "image", "logo", "picture"] _logger = logging.getLogger(__name__) +BOM_MAP = { + 'utf-16le': codecs.BOM_UTF16_LE, + 'utf-16be': codecs.BOM_UTF16_BE, + 'utf-32le': codecs.BOM_UTF32_LE, + 'utf-32be': codecs.BOM_UTF32_BE, +} try: import xlrd @@ -321,6 +328,13 @@ class Import(models.TransientModel): encoding = options.get('encoding') if not encoding: encoding = options['encoding'] = chardet.detect(csv_data)['encoding'].lower() + # some versions of chardet (e.g. 2.3.0 but not 3.x) will return + # utf-(16|32)(le|be), which for python means "ignore / don't strip + # BOM". We don't want that, so rectify the encoding to non-marked + # IFF the guessed encoding is LE/BE and csv_data starts with a BOM + bom = BOM_MAP.get(encoding) + if bom and csv_data.startswith(bom): + encoding = options['encoding'] = encoding[:-2] if encoding != 'utf-8': csv_data = csv_data.decode(encoding).encode('utf-8') -- cgit v1.2.3