[FIX] P3: CSV reading & writing

5 files changed, 59 insertions, 79 deletions

diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py
index f2bab8b52b3..7201b194193 100644
--- a/addons/base_import/models/base_import.py
+++ b/addons/base_import/models/base_import.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import csv
 import datetime
 import io
 import itertools
@@ -262,14 +261,13 @@ class Import(models.TransientModel):
             # csv module expect utf-8, see http://docs.python.org/2/library/csv.html
             csv_data = csv_data.decode(encoding).encode('utf-8')
 
-        csv_iterator = csv.reader(
+        csv_iterator = pycompat.csv_reader(
             io.BytesIO(csv_data),
             quotechar=str(options['quoting']),
             delimiter=str(options['separator']))
 
         return (
-            [item.decode('utf-8') for item in row]
-            for row in csv_iterator
+            row for row in csv_iterator
             if any(x for x in row if x.strip())
         )
 
diff --git a/addons/base_import/tests/test_base_import.py b/addons/base_import/tests/test_base_import.py
index e3357a5c3a1..6d15119ca02 100644
--- a/addons/base_import/tests/test_base_import.py
+++ b/addons/base_import/tests/test_base_import.py
@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import base64
-import csv
 import io
 import unittest
 
 from odoo.tests.common import TransactionCase, can_import
 from odoo.modules.module import get_module_resource
-from odoo.tools import mute_logger
+from odoo.tools import mute_logger, pycompat
 
 ID_FIELD = {
     'id': 'id',
@@ -501,11 +500,11 @@ class test_convert_import_data(TransactionCase):
         Ensure importing keep newlines
         """
         output = io.BytesIO()
-        writer = csv.writer(output, quoting=csv.QUOTE_ALL)
+        writer = pycompat.csv_writer(output, quoting=1)
 
-        data_row = ["\tfoo\n\tbar", " \"hello\" \n\n 'world' "]
+        data_row = [u"\tfoo\n\tbar", u" \"hello\" \n\n 'world' "]
 
-        writer.writerow(["name", "Some Value"])
+        writer.writerow([u"name", u"Some Value"])
         writer.writerow(data_row)
 
         import_wizard = self.env['base_import.import'].create({
@@ -527,16 +526,15 @@ class test_failures(TransactionCase):
         Ensure big fields (e.g. b64-encoded image data) can be imported and
         we're not hitting limits of the default CSV parser config
         """
-        import csv, io
         from PIL import Image
 
         im = Image.new('RGB', (1920, 1080))
         fout = io.BytesIO()
 
-        writer = csv.writer(fout, dialect=None)
+        writer = pycompat.csv_writer(fout, dialect=None)
         writer.writerows([
-            ['name', 'db_datas'],
-            ['foo', base64.b64encode(im.tobytes())]
+            [u'name', u'db_datas'],
+            [u'foo', base64.b64encode(im.tobytes()).decode('ascii')]
         ])
 
         import_wizard = self.env['base_import.import'].create({
diff --git a/addons/l10n_fr_fec/wizard/account_fr_fec.py b/addons/l10n_fr_fec/wizard/account_fr_fec.py
index f82df08dc99..b8c4b571124 100644
--- a/addons/l10n_fr_fec/wizard/account_fr_fec.py
+++ b/addons/l10n_fr_fec/wizard/account_fr_fec.py
@@ -4,11 +4,11 @@
 # Copyright (C) 2013-2015 Akretion (http://www.akretion.com)
 
 import base64
-import csv
 import io
 
 from odoo import api, fields, models, _
 from odoo.exceptions import Warning
+from odoo.tools import pycompat
 
 
 class AccountFrFec(models.TransientModel):
@@ -86,24 +86,24 @@ class AccountFrFec(models.TransientModel):
         # So it will be easier for the accountant to check the file before
         # sending it to the fiscal administration
         header = [
-            'JournalCode',    # 0
-            'JournalLib',     # 1
-            'EcritureNum',    # 2
-            'EcritureDate',   # 3
-            'CompteNum',      # 4
-            'CompteLib',      # 5
-            'CompAuxNum',     # 6  We use partner.id
-            'CompAuxLib',     # 7
-            'PieceRef',       # 8
-            'PieceDate',      # 9
-            'EcritureLib',    # 10
-            'Debit',          # 11
-            'Credit',         # 12
-            'EcritureLet',    # 13
-            'DateLet',        # 14
-            'ValidDate',      # 15
-            'Montantdevise',  # 16
-            'Idevise',        # 17
+            u'JournalCode',    # 0
+            u'JournalLib',     # 1
+            u'EcritureNum',    # 2
+            u'EcritureDate',   # 3
+            u'CompteNum',      # 4
+            u'CompteLib',      # 5
+            u'CompAuxNum',     # 6  We use partner.id
+            u'CompAuxLib',     # 7
+            u'PieceRef',       # 8
+            u'PieceDate',      # 9
+            u'EcritureLib',    # 10
+            u'Debit',          # 11
+            u'Credit',         # 12
+            u'EcritureLet',    # 13
+            u'DateLet',        # 14
+            u'ValidDate',      # 15
+            u'Montantdevise',  # 16
+            u'Idevise',        # 17
             ]
 
         company = self.env.user.company_id
@@ -115,7 +115,7 @@ class AccountFrFec(models.TransientModel):
                 _("FEC is for French companies only !"))
 
         fecfile = io.BytesIO()
-        w = csv.writer(fecfile, delimiter='|')
+        w = pycompat.csv_writer(fecfile, delimiter='|')
         w.writerow(header)
 
         # INITIAL BALANCE
@@ -190,7 +190,7 @@ class AccountFrFec(models.TransientModel):
                     else:
                         listrow[11] = '0,00'
                         listrow[12] = str(-listrow_amount).replace('.', ',')
-            w.writerow([s.encode("utf-8") for s in listrow])
+            w.writerow(listrow)
         #if the unaffected earnings account wasn't in the selection yet: add it manually
         if (not unaffected_earnings_line
             and unaffected_earnings_results
@@ -201,7 +201,7 @@ class AccountFrFec(models.TransientModel):
             if unaffected_earnings_account:
                 unaffected_earnings_results[4] = unaffected_earnings_account.code
                 unaffected_earnings_results[5] = unaffected_earnings_account.name
-            w.writerow([s.encode("utf-8") for s in unaffected_earnings_results])
+            w.writerow(unaffected_earnings_results)
 
         # LINES
         sql_query = '''
@@ -266,8 +266,7 @@ class AccountFrFec(models.TransientModel):
             sql_query, (self.date_from, self.date_to, company.id))
 
         for row in self._cr.fetchall():
-            listrow = list(row)
-            w.writerow([s.encode("utf-8") for s in listrow])
+            w.writerow(list(row))
 
         siren = company.vat[4:13]
         end_date = self.date_to.replace('-', '')
diff --git a/addons/l10n_lu/scripts/tax2csv.py b/addons/l10n_lu/scripts/tax2csv.py
index 013df32e3d3..f4080284064 100644
--- a/addons/l10n_lu/scripts/tax2csv.py
+++ b/addons/l10n_lu/scripts/tax2csv.py
@@ -1,17 +1,8 @@
 from collections import OrderedDict
-import csv
 
 import xlrd
 from odoo.tools import pycompat
 
-def _e(s):
-    if s is None:
-        return ''
-    if pycompat.PY2 and type(s) == pycompat.text_type:
-        return s.encode('utf8')
-    return str(s)
-
-
 def _is_true(s):
     return s not in ('F', 'False', 0, '', None, False)
 
@@ -55,9 +46,9 @@ class LuxTaxGenerator:
             yield OrderedDict(pycompat.izip(keys, row))
 
     def tax_codes_to_csv(self):
-        writer = csv.writer(open('account.tax.code.template-%s.csv' %
+        writer = pycompat.csv_writer(open('account.tax.code.template-%s.csv' %
                                  self.suffix, 'wb'))
-        tax_codes_iterator = self.iter_tax_codes
+        tax_codes_iterator = self.iter_tax_codes()
         keys = next(tax_codes_iterator)
         writer.writerow(keys)
 
@@ -68,7 +59,7 @@ class LuxTaxGenerator:
             if tax_code in tax_codes:
                 raise RuntimeError('duplicate tax code %s' % tax_code)
             tax_codes[tax_code] = row['id']
-            writer.writerow(pycompat.imap(_e, pycompat.values(row)))
+            writer.writerow([pycompat.to_text(v) for v in pycompat.values(row)])
 
         # read taxes and add leaf tax codes
         new_tax_codes = {}  # id: parent_code
@@ -144,17 +135,19 @@ class LuxTaxGenerator:
 
         for tax_code_id in sorted(new_tax_codes):
             name, parent_code = new_tax_codes[tax_code_id]
-            writer.writerow((tax_code_id,
-                             'lu_tct_m' + parent_code,
-                             tax_code_id.replace('lu_tax_code_template_', ''),
-                             '1',
-                             '',
-                             _e(name),
-                             ''))
+            writer.writerow([
+                tax_code_id,
+                u'lu_tct_m' + parent_code,
+                tax_code_id.replace('lu_tax_code_template_', u''),
+                u'1',
+                u'',
+                pycompat.to_text(name),
+                u''
+            ])
 
     def taxes_to_csv(self):
-        writer = csv.writer(open('account.tax.template-%s.csv' %
-                                 self.suffix, 'wb'))
+        writer = pycompat.csv_writer(open('account.tax.template-%s.csv' %
+                                     self.suffix, 'wb'))
         taxes_iterator = self.iter_taxes()
         keys = next(taxes_iterator)
         writer.writerow(keys[3:] + ['sequence'])
@@ -167,17 +160,20 @@ class LuxTaxGenerator:
                 cur_seq = seq + 1000
             else:
                 cur_seq = seq
-            writer.writerow(list(pycompat.imap(_e, list(pycompat.values(row))[3:])) + [cur_seq])
+            writer.writerow([
+                pycompat.to_text(v)
+                for v in list(pycompat.values(row))[3:]
+            ] + [cur_seq])
 
     def fiscal_pos_map_to_csv(self):
-        writer = csv.writer(open('account.fiscal.'
-                                 'position.tax.template-%s.csv' %
-                                 self.suffix, 'wb'))
+        writer = pycompat.csv_writer(open('account.fiscal.'
+                                     'position.tax.template-%s.csv' %
+                                     self.suffix, 'wb'))
         fiscal_pos_map_iterator = self.iter_fiscal_pos_map()
         keys = next(fiscal_pos_map_iterator)
         writer.writerow(keys)
         for row in fiscal_pos_map_iterator:
-            writer.writerow(pycompat.imap(_e, pycompat.values(row)))
+            writer.writerow([pycompat.to_text(s) for s in pycompat.values(row)])
 
 
 if __name__ == '__main__':
diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py
index 9ac7b7a2fd8..ebe9e8aa16d 100644
--- a/addons/web/controllers/main.py
+++ b/addons/web/controllers/main.py
@@ -3,7 +3,6 @@
 
 import babel.messages.pofile
 import base64
-import csv
 import datetime
 import functools
 import glob
@@ -1389,31 +1388,21 @@ class CSVExport(ExportFormat, http.Controller):
 
     def from_data(self, fields, rows):
         fp = io.BytesIO()
-        writer = csv.writer(fp, quoting=csv.QUOTE_ALL)
+        writer = pycompat.csv_writer(fp, quoting=1)
 
-        writer.writerow([name.encode('utf-8') for name in fields])
+        writer.writerow(fields)
 
         for data in rows:
             row = []
             for d in data:
-                if type(d) != str and isinstance(d, pycompat.string_types):
-                    try:
-                        d = d.encode('utf-8')
-                    except UnicodeError:
-                        pass
-                if d is False: d = None
-
                 # Spreadsheet apps tend to detect formulas on leading =, + and -
-                if type(d) is str and d.startswith(('=', '-', '+')):
+                if isinstance(d, pycompat.string_types) and d.startswith(('=', '-', '+')):
                     d = "'" + d
 
-                row.append(d)
+                row.append(pycompat.to_text(d))
             writer.writerow(row)
 
-        fp.seek(0)
-        data = fp.read()
-        fp.close()
-        return data
+        return fp.getvalue()
 
 class ExcelExport(ExportFormat, http.Controller):
     # Excel needs raw data to correctly handle numbers and date values