From 36ee8a78ece08c3fd18ec2cd4904e464641e0b5d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 7 Aug 2018 16:34:11 +0200
Subject: [IMP] base_import: attempt to automatically guess the separator
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It would probably be even better to iterate the file content and get
the non-quoted non-alphanumeric characters as separator
candidates (instead of a hard-coded list) however Python does not seem
to have a decoding iterator (taking bytes and yielding an iterator of
codepoints or even grapheme clusters) — incidentally uniseg seems to
require up-front decoding as well — so that's not really convenient as
we may be dealing with large-ish files and not want to load it
entirely in memory.

An alternative would be to use TextIOWrapper and iterate the file by
buffers of a few ks, and classify that based on either codepoints or
grapheme clusters.
---
 addons/base_import/tests/test_csv_magic.py | 94 +++++++++++++++++++++++++++---
 1 file changed, 87 insertions(+), 7 deletions(-)

(limited to 'addons/base_import/tests/test_csv_magic.py')

diff --git a/addons/base_import/tests/test_csv_magic.py b/addons/base_import/tests/test_csv_magic.py
index 2b088c37afe..92724ec360b 100644
--- a/addons/base_import/tests/test_csv_magic.py
+++ b/addons/base_import/tests/test_csv_magic.py
@@ -6,7 +6,17 @@ import codecs
 
 from odoo.tests import common
 
-class TestEncoding(common.TransactionCase):
+class ImportCase(common.TransactionCase):
+    def _make_import(self, contents):
+        return self.env['base_import.import'].create({
+            'res_model': 'base_import.tests.models.complex',
+            'file_name': 'f',
+            'file_type': 'text/csv',
+            'file': contents,
+        })
+
+
+class TestEncoding(ImportCase):
     """
     create + parse_preview -> check result options
     """
@@ -47,13 +57,83 @@ class TestEncoding(common.TransactionCase):
         self.assertEqual(r['options']['encoding'], 'iso-8859-1')
         self.assertEqual(r['preview'], [['text'], [s.decode('iso-8859-1')]])
 
-    def _make_import(self, contents):
-        return self.env['base_import.import'].create({
-            'res_model': 'base_import.tests.models.complex',
-            'file_name': 'f',
-            'file_type': 'text/csv',
-            'file': contents,
+class TestFileSeparator(ImportCase):
+
+    def setUp(self):
+        super().setUp()
+        self.imp = self._make_import(
+"""c|f
+a|1
+b|2
+c|3
+d|4
+""")
+
+    def test_explicit_success(self):
+        r = self.imp.parse_preview({
+            'separator': '|',
+            'headers': True,
+            'quoting': '"',
+        })
+        self.assertEqual(r['headers'], ['c', 'f'])
+        self.assertEqual(r['preview'], [
+            ['a', '1'],
+            ['b', '2'],
+            ['c', '3'],
+            ['d', '4'],
+        ])
+        self.assertEqual(r['options']['separator'], '|')
+
+    def test_explicit_fail(self):
+        """ Don't protect user against making mistakes
+        """
+        r = self.imp.parse_preview({
+            'separator': ',',
+            'headers': True,
+            'quoting': '"',
+        })
+        self.assertEqual(r['headers'], ['c|f'])
+        self.assertEqual(r['preview'], [
+            ['a|1'],
+            ['b|2'],
+            ['c|3'],
+            ['d|4'],
+        ])
+        self.assertEqual(r['options']['separator'], ',')
+
+    def test_guess_ok(self):
+        r = self.imp.parse_preview({
+            'separator': '',
+            'headers': True,
+            'quoting': '"',
+        })
+        self.assertEqual(r['headers'], ['c', 'f'])
+        self.assertEqual(r['preview'], [
+            ['a', '1'],
+            ['b', '2'],
+            ['c', '3'],
+            ['d', '4'],
+        ])
+        self.assertEqual(r['options']['separator'], '|')
+
+    def test_noguess(self):
+        """ If the guesser has no idea what the separator is, it defaults to
+        "," but should not set that value
+        """
+        imp = self._make_import('c\na\nb\nc\nd')
+        r = imp.parse_preview({
+            'separator': '',
+            'headers': True,
+            'quoting': '"',
         })
+        self.assertEqual(r['headers'], ['c'])
+        self.assertEqual(r['preview'], [
+            ['a'],
+            ['b'],
+            ['c'],
+            ['d'],
+        ])
+        self.assertEqual(r['options']['separator'], '')
 
 class TestNumberSeparators(common.TransactionCase):
     def test_parse_float(self):
-- 
cgit v1.2.3