[FIX] P3: CSV reading & writing

12 files changed, 128 insertions, 108 deletions

diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py
index f2bab8b52b3..7201b194193 100644
--- a/addons/base_import/models/base_import.py
+++ b/addons/base_import/models/base_import.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import csv
 import datetime
 import io
 import itertools
@@ -262,14 +261,13 @@ class Import(models.TransientModel):
             # csv module expect utf-8, see http://docs.python.org/2/library/csv.html
             csv_data = csv_data.decode(encoding).encode('utf-8')
 
-        csv_iterator = csv.reader(
+        csv_iterator = pycompat.csv_reader(
             io.BytesIO(csv_data),
             quotechar=str(options['quoting']),
             delimiter=str(options['separator']))
 
         return (
-            [item.decode('utf-8') for item in row]
-            for row in csv_iterator
+            row for row in csv_iterator
             if any(x for x in row if x.strip())
         )
 
diff --git a/addons/base_import/tests/test_base_import.py b/addons/base_import/tests/test_base_import.py
index e3357a5c3a1..6d15119ca02 100644
--- a/addons/base_import/tests/test_base_import.py
+++ b/addons/base_import/tests/test_base_import.py
@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import base64
-import csv
 import io
 import unittest
 
 from odoo.tests.common import TransactionCase, can_import
 from odoo.modules.module import get_module_resource
-from odoo.tools import mute_logger
+from odoo.tools import mute_logger, pycompat
 
 ID_FIELD = {
     'id': 'id',
@@ -501,11 +500,11 @@ class test_convert_import_data(TransactionCase):
         Ensure importing keep newlines
         """
         output = io.BytesIO()
-        writer = csv.writer(output, quoting=csv.QUOTE_ALL)
+        writer = pycompat.csv_writer(output, quoting=1)
 
-        data_row = ["\tfoo\n\tbar", " \"hello\" \n\n 'world' "]
+        data_row = [u"\tfoo\n\tbar", u" \"hello\" \n\n 'world' "]
 
-        writer.writerow(["name", "Some Value"])
+        writer.writerow([u"name", u"Some Value"])
         writer.writerow(data_row)
 
         import_wizard = self.env['base_import.import'].create({
@@ -527,16 +526,15 @@ class test_failures(TransactionCase):
         Ensure big fields (e.g. b64-encoded image data) can be imported and
         we're not hitting limits of the default CSV parser config
         """
-        import csv, io
         from PIL import Image
 
         im = Image.new('RGB', (1920, 1080))
         fout = io.BytesIO()
 
-        writer = csv.writer(fout, dialect=None)
+        writer = pycompat.csv_writer(fout, dialect=None)
         writer.writerows([
-            ['name', 'db_datas'],
-            ['foo', base64.b64encode(im.tobytes())]
+            [u'name', u'db_datas'],
+            [u'foo', base64.b64encode(im.tobytes()).decode('ascii')]
         ])
 
         import_wizard = self.env['base_import.import'].create({
diff --git a/addons/l10n_fr_fec/wizard/account_fr_fec.py b/addons/l10n_fr_fec/wizard/account_fr_fec.py
index f82df08dc99..b8c4b571124 100644
--- a/addons/l10n_fr_fec/wizard/account_fr_fec.py
+++ b/addons/l10n_fr_fec/wizard/account_fr_fec.py
@@ -4,11 +4,11 @@
 # Copyright (C) 2013-2015 Akretion (http://www.akretion.com)
 
 import base64
-import csv
 import io
 
 from odoo import api, fields, models, _
 from odoo.exceptions import Warning
+from odoo.tools import pycompat
 
 
 class AccountFrFec(models.TransientModel):
@@ -86,24 +86,24 @@ class AccountFrFec(models.TransientModel):
         # So it will be easier for the accountant to check the file before
         # sending it to the fiscal administration
         header = [
-            'JournalCode',    # 0
-            'JournalLib',     # 1
-            'EcritureNum',    # 2
-            'EcritureDate',   # 3
-            'CompteNum',      # 4
-            'CompteLib',      # 5
-            'CompAuxNum',     # 6  We use partner.id
-            'CompAuxLib',     # 7
-            'PieceRef',       # 8
-            'PieceDate',      # 9
-            'EcritureLib',    # 10
-            'Debit',          # 11
-            'Credit',         # 12
-            'EcritureLet',    # 13
-            'DateLet',        # 14
-            'ValidDate',      # 15
-            'Montantdevise',  # 16
-            'Idevise',        # 17
+            u'JournalCode',    # 0
+            u'JournalLib',     # 1
+            u'EcritureNum',    # 2
+            u'EcritureDate',   # 3
+            u'CompteNum',      # 4
+            u'CompteLib',      # 5
+            u'CompAuxNum',     # 6  We use partner.id
+            u'CompAuxLib',     # 7
+            u'PieceRef',       # 8
+            u'PieceDate',      # 9
+            u'EcritureLib',    # 10
+            u'Debit',          # 11
+            u'Credit',         # 12
+            u'EcritureLet',    # 13
+            u'DateLet',        # 14
+            u'ValidDate',      # 15
+            u'Montantdevise',  # 16
+            u'Idevise',        # 17
             ]
 
         company = self.env.user.company_id
@@ -115,7 +115,7 @@ class AccountFrFec(models.TransientModel):
                 _("FEC is for French companies only !"))
 
         fecfile = io.BytesIO()
-        w = csv.writer(fecfile, delimiter='|')
+        w = pycompat.csv_writer(fecfile, delimiter='|')
         w.writerow(header)
 
         # INITIAL BALANCE
@@ -190,7 +190,7 @@ class AccountFrFec(models.TransientModel):
                     else:
                         listrow[11] = '0,00'
                         listrow[12] = str(-listrow_amount).replace('.', ',')
-            w.writerow([s.encode("utf-8") for s in listrow])
+            w.writerow(listrow)
         #if the unaffected earnings account wasn't in the selection yet: add it manually
         if (not unaffected_earnings_line
             and unaffected_earnings_results
@@ -201,7 +201,7 @@ class AccountFrFec(models.TransientModel):
             if unaffected_earnings_account:
                 unaffected_earnings_results[4] = unaffected_earnings_account.code
                 unaffected_earnings_results[5] = unaffected_earnings_account.name
-            w.writerow([s.encode("utf-8") for s in unaffected_earnings_results])
+            w.writerow(unaffected_earnings_results)
 
         # LINES
         sql_query = '''
@@ -266,8 +266,7 @@ class AccountFrFec(models.TransientModel):
             sql_query, (self.date_from, self.date_to, company.id))
 
         for row in self._cr.fetchall():
-            listrow = list(row)
-            w.writerow([s.encode("utf-8") for s in listrow])
+            w.writerow(list(row))
 
         siren = company.vat[4:13]
         end_date = self.date_to.replace('-', '')
diff --git a/addons/l10n_lu/scripts/tax2csv.py b/addons/l10n_lu/scripts/tax2csv.py
index 013df32e3d3..f4080284064 100644
--- a/addons/l10n_lu/scripts/tax2csv.py
+++ b/addons/l10n_lu/scripts/tax2csv.py
@@ -1,17 +1,8 @@
 from collections import OrderedDict
-import csv
 
 import xlrd
 from odoo.tools import pycompat
 
-def _e(s):
-    if s is None:
-        return ''
-    if pycompat.PY2 and type(s) == pycompat.text_type:
-        return s.encode('utf8')
-    return str(s)
-
-
 def _is_true(s):
     return s not in ('F', 'False', 0, '', None, False)
 
@@ -55,9 +46,9 @@ class LuxTaxGenerator:
             yield OrderedDict(pycompat.izip(keys, row))
 
     def tax_codes_to_csv(self):
-        writer = csv.writer(open('account.tax.code.template-%s.csv' %
+        writer = pycompat.csv_writer(open('account.tax.code.template-%s.csv' %
                                  self.suffix, 'wb'))
-        tax_codes_iterator = self.iter_tax_codes
+        tax_codes_iterator = self.iter_tax_codes()
         keys = next(tax_codes_iterator)
         writer.writerow(keys)
 
@@ -68,7 +59,7 @@ class LuxTaxGenerator:
             if tax_code in tax_codes:
                 raise RuntimeError('duplicate tax code %s' % tax_code)
             tax_codes[tax_code] = row['id']
-            writer.writerow(pycompat.imap(_e, pycompat.values(row)))
+            writer.writerow([pycompat.to_text(v) for v in pycompat.values(row)])
 
         # read taxes and add leaf tax codes
         new_tax_codes = {}  # id: parent_code
@@ -144,17 +135,19 @@ class LuxTaxGenerator:
 
         for tax_code_id in sorted(new_tax_codes):
             name, parent_code = new_tax_codes[tax_code_id]
-            writer.writerow((tax_code_id,
-                             'lu_tct_m' + parent_code,
-                             tax_code_id.replace('lu_tax_code_template_', ''),
-                             '1',
-                             '',
-                             _e(name),
-                             ''))
+            writer.writerow([
+                tax_code_id,
+                u'lu_tct_m' + parent_code,
+                tax_code_id.replace('lu_tax_code_template_', u''),
+                u'1',
+                u'',
+                pycompat.to_text(name),
+                u''
+            ])
 
     def taxes_to_csv(self):
-        writer = csv.writer(open('account.tax.template-%s.csv' %
-                                 self.suffix, 'wb'))
+        writer = pycompat.csv_writer(open('account.tax.template-%s.csv' %
+                                     self.suffix, 'wb'))
         taxes_iterator = self.iter_taxes()
         keys = next(taxes_iterator)
         writer.writerow(keys[3:] + ['sequence'])
@@ -167,17 +160,20 @@ class LuxTaxGenerator:
                 cur_seq = seq + 1000
             else:
                 cur_seq = seq
-            writer.writerow(list(pycompat.imap(_e, list(pycompat.values(row))[3:])) + [cur_seq])
+            writer.writerow([
+                pycompat.to_text(v)
+                for v in list(pycompat.values(row))[3:]
+            ] + [cur_seq])
 
     def fiscal_pos_map_to_csv(self):
-        writer = csv.writer(open('account.fiscal.'
-                                 'position.tax.template-%s.csv' %
-                                 self.suffix, 'wb'))
+        writer = pycompat.csv_writer(open('account.fiscal.'
+                                     'position.tax.template-%s.csv' %
+                                     self.suffix, 'wb'))
         fiscal_pos_map_iterator = self.iter_fiscal_pos_map()
         keys = next(fiscal_pos_map_iterator)
         writer.writerow(keys)
         for row in fiscal_pos_map_iterator:
-            writer.writerow(pycompat.imap(_e, pycompat.values(row)))
+            writer.writerow([pycompat.to_text(s) for s in pycompat.values(row)])
 
 
 if __name__ == '__main__':
diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py
index 9ac7b7a2fd8..ebe9e8aa16d 100644
--- a/addons/web/controllers/main.py
+++ b/addons/web/controllers/main.py
@@ -3,7 +3,6 @@
 
 import babel.messages.pofile
 import base64
-import csv
 import datetime
 import functools
 import glob
@@ -1389,31 +1388,21 @@ class CSVExport(ExportFormat, http.Controller):
 
     def from_data(self, fields, rows):
         fp = io.BytesIO()
-        writer = csv.writer(fp, quoting=csv.QUOTE_ALL)
+        writer = pycompat.csv_writer(fp, quoting=1)
 
-        writer.writerow([name.encode('utf-8') for name in fields])
+        writer.writerow(fields)
 
         for data in rows:
             row = []
             for d in data:
-                if type(d) != str and isinstance(d, pycompat.string_types):
-                    try:
-                        d = d.encode('utf-8')
-                    except UnicodeError:
-                        pass
-                if d is False: d = None
-
                 # Spreadsheet apps tend to detect formulas on leading =, + and -
-                if type(d) is str and d.startswith(('=', '-', '+')):
+                if isinstance(d, pycompat.string_types) and d.startswith(('=', '-', '+')):
                     d = "'" + d
 
-                row.append(d)
+                row.append(pycompat.to_text(d))
             writer.writerow(row)
 
-        fp.seek(0)
-        data = fp.read()
-        fp.close()
-        return data
+        return fp.getvalue()
 
 class ExcelExport(ExportFormat, http.Controller):
     # Excel needs raw data to correctly handle numbers and date values
diff --git a/doc/python3.rst b/doc/python3.rst
index a7d143b85b7..7ca86708ced 100644
--- a/doc/python3.rst
+++ b/doc/python3.rst
@@ -452,6 +452,26 @@ conversions: text is *encoded* to bytes and bytes are *decoded* to text.
     migrated to using ``base64.b64encode`` and ``base64.b64decode``
     respectively.
 
+csv
+---
+
+``csv`` is a fairly vicious one: not only is it not a very good format, the
+Python 2 and Python 3 versions of the library are text-model incompatible in
+significant ways:
+
+* Python 2's CSV only works on *ascii-compatible byte streams* (it has no
+  encoding support at all) and extracts bytestring values
+* Python 3's CSV only works on *text streams* and extract text values
+* And ``io`` doesn't provide "native string" streaming facilities.
+
+However with respect to Odoo it turns out most or all uses of ``csv`` fit
+inside a model of *byte stream to and from text values*.
+
+The latter is thus a model implemented by cross-version wrappers
+:func:`odoo.tools.pycompat.csv_reader` and
+:func:`odoo.tools.pycompat.csv_writer`: they take a *UTF-8 byte stream* and
+read or write *text* values.
+
 .. _hash randomisation: http://bugs.python.org/issue13703
 
 .. _requests: http://docs.python-requests.org/
diff --git a/odoo/addons/test_pylint/tests/test_pylint.py b/odoo/addons/test_pylint/tests/test_pylint.py
index cc73ca5a709..92764eaeb84 100644
--- a/odoo/addons/test_pylint/tests/test_pylint.py
+++ b/odoo/addons/test_pylint/tests/test_pylint.py
@@ -79,6 +79,7 @@ class TestPyLint(TransactionCase):
     BAD_MODULES = [
         'commands',
         'cPickle',
+        'csv',
         'cStringIO',
         'md5',
         'urllib',
diff --git a/odoo/cli/server.py b/odoo/cli/server.py
index 180efcfb70f..d740dc7282a 100644
--- a/odoo/cli/server.py
+++ b/odoo/cli/server.py
@@ -12,7 +12,7 @@ GNU Public Licence.
 """
 
 import atexit
-import csv
+import csv # pylint: disable=deprecated-module
 import logging
 import os
 import signal
diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py
index 7da9f4e4412..f979df7a28c 100644
--- a/odoo/tools/convert.py
+++ b/odoo/tools/convert.py
@@ -2,7 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import base64
 import io
-import csv
 import logging
 import os.path
 import re
@@ -844,28 +843,21 @@ def convert_csv_import(cr, module, fname, csvcontent, idref=None, mode='init',
         quote: "
         delimiter: ,
         encoding: utf-8'''
-    if not idref:
-        idref={}
-    model = ('.'.join(fname.split('.')[:-1]).split('-'))[0]
-    #remove folder path from model
-    head, model = os.path.split(model)
+    filename, _ext = os.path.splitext(os.path.basename(fname))
+    model = filename.split('-')[0]
 
-    input = io.BytesIO(csvcontent) #FIXME
-    reader = csv.reader(input, quotechar='"', delimiter=',')
+    reader = pycompat.csv_reader(io.BytesIO(csvcontent), quotechar='"', delimiter=',')
     fields = next(reader)
 
     if not (mode == 'init' or 'id' in fields):
         _logger.error("Import specification does not contain 'id' and we are in init mode, Cannot continue.")
         return
 
-    datas = []
-    for line in reader:
-        if not (line and any(line)):
-            continue
-        try:
-            datas.append([ustr(v) for v in line])
-        except Exception:
-            _logger.error("Cannot import the line: %s", line)
+    # filter out empty lines (any([]) == False) and lines containing only empty cells
+    datas = [
+        line for line in reader
+        if any(line)
+    ]
 
     context = {
         'mode': mode,
diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py
index 05a5a91f356..274ae4a6d24 100644
--- a/odoo/tools/misc.py
+++ b/odoo/tools/misc.py
@@ -30,7 +30,6 @@ from lxml import etree
 
 from .which import which
 import traceback
-import csv
 from operator import itemgetter
 
 try:
@@ -403,14 +402,15 @@ def scan_languages():
     csvpath = odoo.modules.module.get_resource_path('base', 'res', 'res.lang.csv')
     try:
         # read (code, name) from languages in base/res/res.lang.csv
-        result = []
-        with open(csvpath) as csvfile:
-            reader = csv.reader(csvfile, delimiter=',', quotechar='"')
+        with open(csvpath, 'rb') as csvfile:
+            reader = pycompat.csv_reader(csvfile, delimiter=',', quotechar='"')
             fields = next(reader)
             code_index = fields.index("code")
             name_index = fields.index("name")
-            for row in reader:
-                result.append((ustr(row[code_index]), ustr(row[name_index])))
+            result = [
+                (row[code_index], row[name_index])
+                for row in reader
+            ]
     except Exception:
         _logger.error("Could not read %s", csvpath)
         result = []
diff --git a/odoo/tools/pycompat.py b/odoo/tools/pycompat.py
index e4111668d24..89bf96c3dd6 100644
--- a/odoo/tools/pycompat.py
+++ b/odoo/tools/pycompat.py
@@ -1,10 +1,16 @@
 # -*- coding: utf-8 -*-
 # to remove if we decide to add a dependency on six or future
 # very strongly inspired by https://github.com/pallets/werkzeug/blob/master/werkzeug/_compat.py
+#pylint: disable=deprecated-module
+import csv
+import collections
+import io
 import sys
 
+
 PY2 = sys.version_info[0] == 2
 
+_Writer = collections.namedtuple('_Writer', 'writerow writerows')
 if PY2:
     # pylint: disable=long-builtin,unichr-builtin,unicode-builtin
     unichr = unichr
@@ -39,6 +45,19 @@ if PY2:
         return cls
 
     exec ('def reraise(tp, value, tb=None):\n raise tp, value, tb')
+
+    def csv_reader(stream, **params):
+        for row in csv.reader(stream, **params):
+            yield [c.decode('utf-8') for c in row]
+    def csv_writer(stream, **params):
+        w = csv.writer(stream, **params)
+        return _Writer(
+            writerow=lambda r: w.writerow([c.encode('utf-8') for c in r]),
+            writerows=lambda rs: w.writerows(
+                [c.encode('utf-8') for c in r]
+                for r in rs
+            )
+        )
 else:
     # pylint: disable=bad-functions
     unichr = chr
@@ -74,6 +93,15 @@ else:
             raise value.with_traceback(tb)
         raise value
 
+    def csv_reader(stream, **params):
+        assert not isinstance(stream, io.TextIOBase),\
+            "For cross-compatibility purposes, csv_reader takes a bytes stream"
+        return csv.reader(io.TextIOWrapper(stream, encoding='utf-8'), **params)
+    def csv_writer(stream, **params):
+        assert not isinstance(stream, io.TextIOBase), \
+            "For cross-compatibility purposes, csv_writer takes a bytes stream"
+        return csv.writer(io.TextIOWrapper(stream, encoding='utf-8', line_buffering=True), **params)
+
 def to_text(source):
     """ Generates a text value (an instance of text_type) from an arbitrary 
     source.
diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py
index dbf995a5161..58bbf6bcec5 100644
--- a/odoo/tools/translate.py
+++ b/odoo/tools/translate.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import codecs
-import csv
 import fnmatch
 import inspect
 import io
@@ -121,6 +120,8 @@ _LOCALE2WIN32 = {
 ENGLISH_SMALL_WORDS = set("as at by do go if in me no of ok on or to up us we".split())
 
 
+# these direct uses of CSV are ok.
+import csv # pylint: disable=deprecated-module
 class UNIX_LINE_TERMINATOR(csv.excel):
     lineterminator = '\n'
 
@@ -657,7 +658,7 @@ def trans_export(lang, modules, buffer, format, cr):
 
     def _process(format, modules, rows, buffer, lang):
         if format == 'csv':
-            writer = csv.writer(buffer, 'UNIX')
+            writer = pycompat.csv_writer(buffer, dialect='UNIX')
             # write header first
             writer.writerow(("module","type","name","res_id","src","value","comments"))
             for module, type, name, res_id, src, trad, comments in rows:
@@ -1035,11 +1036,9 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
         # now, the serious things: we read the language file
         fileobj.seek(0)
         if fileformat == 'csv':
-            reader = csv.reader(fileobj, quotechar='"', delimiter=',')
+            reader = pycompat.csv_reader(fileobj, quotechar='"', delimiter=',')
             # read the first line of the file (it contains columns titles)
-            for row in reader:
-                fields = row
-                break
+            fields = next(reader)
 
         elif fileformat == 'po':
             reader = PoFile(fileobj)