[FIX] P3: text model types

* remove references to basestring & unicode (use relevant pycompat
  helpers)
* remove some str calls (either entirely or replaced by relevant
  helper, either text or native)
* use better API to avoid unnecessary conversions
* remove some XML declarations in views

11 files changed, 103 insertions, 71 deletions

diff --git a/odoo/tools/__init__.py b/odoo/tools/__init__.py
index 13c2fd410b9..2b4eeb50c55 100644
--- a/odoo/tools/__init__.py
+++ b/odoo/tools/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import copy
+from . import pycompat
 from . import win32
 from . import appdirs
 from .config import config
diff --git a/odoo/tools/appdirs.py b/odoo/tools/appdirs.py
index 8617dabfa4f..6176e6bce75 100644
--- a/odoo/tools/appdirs.py
+++ b/odoo/tools/appdirs.py
@@ -13,6 +13,7 @@ from __future__ import print_function
 #   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
 # - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
 # - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+from . import pycompat
 
 __version_info__ = (1, 3, 0)
 __version__ = '.'.join(str(v) for v in __version_info__)
@@ -21,12 +22,6 @@ __version__ = '.'.join(str(v) for v in __version_info__)
 import sys
 import os
 
-PY3 = sys.version_info[0] == 3
-
-if PY3:
-    unicode = str
-
-
 
 def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
     r"""Return full path to the user-specific data dir for this application.
@@ -397,7 +392,7 @@ def _get_win_folder_with_pywin32(csidl_name):
     # not return unicode strings when there is unicode data in the
     # path.
     try:
-        dir = unicode(dir)
+        dir = pycompat.text_type(dir)
 
         # Downgrade to short path name if have highbit chars. See
         # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
diff --git a/odoo/tools/cache.py b/odoo/tools/cache.py
index 2554331ce9b..1226038a21c 100644
--- a/odoo/tools/cache.py
+++ b/odoo/tools/cache.py
@@ -8,7 +8,7 @@ from decorator import decorator
 from inspect import formatargspec, getargspec
 import logging
 
-from odoo.tools import pycompat
+from . import pycompat
 
 unsafe_eval = eval
 
@@ -135,7 +135,7 @@ class ormcache_multi(ormcache):
     def determine_key(self):
         """ Determine the function that computes a cache key from arguments. """
         assert self.skiparg is None, "ormcache_multi() no longer supports skiparg"
-        assert isinstance(self.multi, basestring), "ormcache_multi() parameter multi must be an argument name"
+        assert isinstance(self.multi, pycompat.string_types), "ormcache_multi() parameter multi must be an argument name"
 
         super(ormcache_multi, self).determine_key()
 
diff --git a/odoo/tools/config.py b/odoo/tools/config.py
index 97c0d2f3d09..ee6c03a7e2e 100644
--- a/odoo/tools/config.py
+++ b/odoo/tools/config.py
@@ -5,15 +5,13 @@ try:
     import configparser as ConfigParser
 except ImportError:
     import ConfigParser
-    
+
+import logging
 import optparse
 import os
 import sys
 import odoo
-import odoo.conf
-import odoo.loglevels as loglevels
-import logging
-import odoo.release as release
+from .. import release, conf, loglevels
 from . import appdirs, pycompat
 
 
@@ -398,10 +396,10 @@ class configmanager(object):
             if getattr(opt, arg):
                 self.options[arg] = getattr(opt, arg)
             # ... or keep, but cast, the config file value.
-            elif isinstance(self.options[arg], basestring) and self.casts[arg].type in optparse.Option.TYPE_CHECKER:
+            elif isinstance(self.options[arg], pycompat.string_types) and self.casts[arg].type in optparse.Option.TYPE_CHECKER:
                 self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg])
 
-        if isinstance(self.options['log_handler'], basestring):
+        if isinstance(self.options['log_handler'], pycompat.string_types):
             self.options['log_handler'] = self.options['log_handler'].split(',')
         self.options['log_handler'].extend(opt.log_handler)
 
@@ -433,7 +431,7 @@ class configmanager(object):
             if getattr(opt, arg) is not None:
                 self.options[arg] = getattr(opt, arg)
             # ... or keep, but cast, the config file value.
-            elif isinstance(self.options[arg], basestring) and self.casts[arg].type in optparse.Option.TYPE_CHECKER:
+            elif isinstance(self.options[arg], pycompat.string_types) and self.casts[arg].type in optparse.Option.TYPE_CHECKER:
                 self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg])
 
         self.options['root_path'] = os.path.abspath(os.path.expanduser(os.path.expandvars(os.path.join(os.path.dirname(__file__), '..'))))
@@ -471,9 +469,9 @@ class configmanager(object):
         if opt.save:
             self.save()
 
-        odoo.conf.addons_paths = self.options['addons_path'].split(',')
+        conf.addons_paths = self.options['addons_path'].split(',')
 
-        odoo.conf.server_wide_modules = [
+        conf.server_wide_modules = [
             m.strip() for m in self.options['server_wide_modules'].split(',') if m.strip()
         ]
 
@@ -575,7 +573,7 @@ class configmanager(object):
 
     def __setitem__(self, key, value):
         self.options[key] = value
-        if key in self.options and isinstance(self.options[key], basestring) and \
+        if key in self.options and isinstance(self.options[key], pycompat.string_types) and \
                 key in self.casts and self.casts[key].type in optparse.Option.TYPE_CHECKER:
             self.options[key] = optparse.Option.TYPE_CHECKER[self.casts[key].type](self.casts[key], key, self.options[key])
 
diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py
index 66e72c5d3ff..7da9f4e4412 100644
--- a/odoo/tools/convert.py
+++ b/odoo/tools/convert.py
@@ -16,7 +16,6 @@ import pytz
 from lxml import etree, builder
 
 import odoo
-import odoo.release
 from . import assertion_report, pycompat
 from .config import config
 from .misc import file_open, unquote, ustr, SKIPPED_ELEMENT_TYPES
@@ -683,7 +682,7 @@ form: module.record_id""" % (xml_id,)
                         f_val = int(f_val)
                     elif model._fields[f_name].type in ['float', 'monetary']:
                         f_val = float(f_val)
-                    elif model._fields[f_name].type == 'boolean' and isinstance(f_val, basestring):
+                    elif model._fields[f_name].type == 'boolean' and isinstance(f_val, pycompat.string_types):
                         f_val = str2bool(f_val)
             res[f_name] = f_val
 
@@ -893,7 +892,7 @@ def convert_xml_import(cr, module, xmlfile, idref=None, mode='init', noupdate=Fa
 
     if idref is None:
         idref={}
-    if isinstance(xmlfile, basestring):
+    if isinstance(xmlfile, pycompat.string_types):
         xml_filename = xmlfile
     else:
         xml_filename = xmlfile.name
diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py
index 8d3854aeb6d..b348e983a2e 100644
--- a/odoo/tools/misc.py
+++ b/odoo/tools/misc.py
@@ -465,7 +465,7 @@ def human_size(sz):
     if not sz:
         return False
     units = ('bytes', 'Kb', 'Mb', 'Gb')
-    if isinstance(sz,basestring):
+    if isinstance(sz,pycompat.string_types):
         sz=len(sz)
     s, i = float(sz), 0
     while s >= 1024 and i < len(units)-1:
@@ -798,7 +798,7 @@ class mute_logger(object):
 
     def __enter__(self):
         for logger in self.loggers:
-            assert isinstance(logger, basestring),\
+            assert isinstance(logger, pycompat.string_types),\
                 "A logger name must be a string, got %s" % type(logger)
             logging.getLogger(logger).addFilter(self)
 
@@ -1073,7 +1073,7 @@ def formatLang(env, value, digits=None, grouping=True, monetary=False, dp=False,
                 if not digits and digits is not 0:
                     digits = DEFAULT_DIGITS
 
-    if isinstance(value, (str, unicode)) and not value:
+    if isinstance(value, pycompat.string_types) and not value:
         return ''
 
     lang = env.user.company_id.partner_id.lang or 'en_US'
@@ -1108,7 +1108,7 @@ def format_date(env, value, lang_code=False, date_format=False):
         return ''
     if isinstance(value, datetime.datetime):
         value = value.date()
-    elif isinstance(value, basestring):
+    elif isinstance(value, pycompat.string_types):
         if len(value) < DATE_LENGTH:
             return ''
         value = value[:DATE_LENGTH]
diff --git a/odoo/tools/pycompat.py b/odoo/tools/pycompat.py
index ff229f0562c..e4111668d24 100644
--- a/odoo/tools/pycompat.py
+++ b/odoo/tools/pycompat.py
@@ -10,6 +10,15 @@ if PY2:
     unichr = unichr
     text_type = unicode
     string_types = (str, unicode)
+    def to_native(source, encoding='utf-8', falsy_empty=False):
+        if not source and falsy_empty:
+            return ''
+
+        if isinstance(source, text_type):
+            return source.encode(encoding)
+
+        return str(source)
+
     integer_types = (int, long)
 
     keys = lambda d: iter(d.iterkeys())
@@ -19,6 +28,11 @@ if PY2:
     # noinspection PyUnresolvedReferences
     from itertools import imap, izip, ifilter
 
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
+        return cls
+
     def implements_iterator(cls):
         cls.next = cls.__next__
         del cls.__next__
@@ -30,6 +44,15 @@ else:
     unichr = chr
     text_type = str
     string_types = (str,)
+    def to_native(source, encoding='utf-8', falsy_empty=False):
+        if not source and falsy_empty:
+            return ''
+
+        if isinstance(source, bytes):
+            return source.decode(encoding)
+
+        return str(source)
+
     integer_types = (int,)
 
     keys = lambda d: iter(d.keys())
@@ -40,6 +63,9 @@ else:
     izip = zip
     ifilter = filter
 
+    def implements_to_string(cls):
+        return cls
+
     def implements_iterator(cls):
         return cls
 
@@ -47,3 +73,20 @@ else:
         if value.__traceback__ != tb:
             raise value.with_traceback(tb)
         raise value
+
+def to_text(source):
+    """ Generates a text value (an instance of text_type) from an arbitrary 
+    source.
+    
+    * False and None are converted to empty strings
+    * text is passed through
+    * bytes are decoded as UTF-8
+    * rest is textified via the current version's relevant data model method
+    """
+    if source is None or source is False:
+        return u''
+
+    if isinstance(source, bytes):
+        return source.decode('utf-8')
+
+    return text_type(source)
diff --git a/odoo/tools/safe_eval.py b/odoo/tools/safe_eval.py
index de767fb7453..fec89fac087 100644
--- a/odoo/tools/safe_eval.py
+++ b/odoo/tools/safe_eval.py
@@ -26,6 +26,7 @@ import werkzeug
 
 from . import pycompat
 from .misc import ustr
+from . import pycompat
 
 import odoo
 
@@ -267,8 +268,9 @@ _BUILTINS = {
     'True': True,
     'False': False,
     'None': None,
+    'bytes': bytes,
     'str': str,
-    'unicode': unicode,
+    'unicode': pycompat.text_type,
     'bool': bool,
     'int': int,
     'float': float,
diff --git a/odoo/tools/test_reports.py b/odoo/tools/test_reports.py
index cad44431c1d..6a6401eb896 100644
--- a/odoo/tools/test_reports.py
+++ b/odoo/tools/test_reports.py
@@ -7,15 +7,14 @@
     through the code of yaml tests.
 """
 
-import odoo
-import odoo.tools as tools
 import logging
-
-from odoo.tools import pycompat
-from odoo.tools.safe_eval import safe_eval
-from subprocess import Popen, PIPE
 import os
 import tempfile
+from subprocess import Popen, PIPE
+
+from .. import api
+from . import pycompat, ustr, config
+from .safe_eval import safe_eval
 
 _logger = logging.getLogger(__name__)
 _test_logger = logging.getLogger('odoo.tests')
@@ -30,7 +29,7 @@ def try_report(cr, uid, rname, ids, data=None, context=None, our_module=None, re
         context = {}
     _test_logger.info("  - Trying %s.create(%r)", rname, ids)
 
-    env = odoo.api.Environment(cr, uid, context)
+    env = api.Environment(cr, uid, context)
 
     report_id = env['ir.actions.report'].search([('report_name', '=', rname)], limit=1)
     if not report_id:
@@ -41,12 +40,12 @@ def try_report(cr, uid, rname, ids, data=None, context=None, our_module=None, re
     if not res_data:
         raise ValueError("Report %s produced an empty result!" % rname)
 
-    if tools.config['test_report_directory']:
-        open(os.path.join(tools.config['test_report_directory'], rname+ '.'+res_format), 'wb+').write(res_data)
+    if config['test_report_directory']:
+        open(os.path.join(config['test_report_directory'], rname+ '.'+res_format), 'wb+').write(res_data)
 
     _logger.debug("Have a %s report for %s, will examine it", res_format, rname)
     if res_format == 'pdf':
-        if res_data[:5] != '%PDF-':
+        if res_data[:5] != b'%PDF-':
             raise ValueError("Report %s produced a non-pdf header, %r" % (rname, res_data[:10]))
         res_text = False
         try:
@@ -56,7 +55,7 @@ def try_report(cr, uid, rname, ids, data=None, context=None, our_module=None, re
 
             proc = Popen(['pdftotext', '-enc', 'UTF-8', '-nopgbrk', rfname, '-'], shell=False, stdout=PIPE)
             stdout, stderr = proc.communicate()
-            res_text = tools.ustr(stdout)
+            res_text = ustr(stdout)
             os.unlink(rfname)
         except Exception:
             _logger.debug("Unable to parse PDF report: install pdftotext to perform automated tests.")
@@ -92,14 +91,14 @@ def try_report_action(cr, uid, action_id, active_model=None, active_ids=None,
                 Eg. 'OK' or 'fa-print'
         :param our_module: the name of the calling module (string), like 'account'
     """
-    if not our_module and isinstance(action_id, basestring):
+    if not our_module and isinstance(action_id, pycompat.string_types):
         if '.' in action_id:
             our_module = action_id.split('.', 1)[0]
 
     context = dict(context or {})
     # TODO context fill-up
 
-    env = odoo.api.Environment(cr, uid, context)
+    env = api.Environment(cr, uid, context)
 
     def log_test(msg, *args):
         _test_logger.info("  - " + msg, *args)
@@ -113,7 +112,7 @@ def try_report_action(cr, uid, action_id, active_model=None, active_ids=None,
     if not wiz_buttons:
         wiz_buttons = []
 
-    if isinstance(action_id, basestring):
+    if isinstance(action_id, pycompat.string_types):
         if '.' in action_id:
             _, act_xmlid = action_id.split('.', 1)
         else:
@@ -138,7 +137,7 @@ def try_report_action(cr, uid, action_id, active_model=None, active_ids=None,
         if datas.get('id',False):
             context.update( {'active_id': datas.get('id',False), 'active_ids': datas.get('ids',[]), 'active_model': datas.get('model',False)})
         context1 = action.get('context', {})
-        if isinstance(context1, basestring):
+        if isinstance(context1, pycompat.string_types):
             context1 = safe_eval(context1, dict(context))
         context.update(context1)
         env = env(context=context)
diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py
index 49aeb2864a8..dbf995a5161 100644
--- a/odoo/tools/translate.py
+++ b/odoo/tools/translate.py
@@ -127,9 +127,7 @@ class UNIX_LINE_TERMINATOR(csv.excel):
 csv.register_dialect("UNIX", UNIX_LINE_TERMINATOR)
 
 
-#
-# Helper functions for translating fields
-#
+# FIXME: holy shit this whole thing needs to be cleaned up hard it's a mess
 def encode(s):
     if isinstance(s, unicode):
         return s.encode('utf8')
@@ -280,18 +278,18 @@ def translate_xml_node(node, callback, parse, serialize):
 
 
 def parse_xml(text):
-    return etree.fromstring(encode(text))
+    return etree.fromstring(text)
 
 def serialize_xml(node):
-    return etree.tostring(node, method='xml', encoding='utf8').decode('utf8')
+    return etree.tostring(node, method='xml', encoding='unicode')
 
 _HTML_PARSER = etree.HTMLParser(encoding='utf8')
 
 def parse_html(text):
-    return html.fragment_fromstring(encode(text), parser=_HTML_PARSER)
+    return html.fragment_fromstring(text, parser=_HTML_PARSER)
 
 def serialize_html(node):
-    return etree.tostring(node, method='html', encoding='utf8').decode('utf8')
+    return etree.tostring(node, method='html', encoding='unicode')
 
 
 def xml_translate(callback, value):
@@ -307,7 +305,7 @@ def xml_translate(callback, value):
         return serialize_xml(result)
     except etree.ParseError:
         # fallback for translated terms: use an HTML parser and wrap the term
-        root = parse_html("<div>%s</div>" % value)
+        root = parse_html(u"<div>%s</div>" % value)
         result = translate_xml_node(root, callback, parse_xml, serialize_xml)
         # remove tags <div> and </div> from result
         return serialize_xml(result)[5:-6]
@@ -643,14 +641,13 @@ class PoFile(object):
             # only strings in python code are python formated
             self.buffer.write(u"#, python-format\n")
 
-        if not isinstance(trad, unicode):
-            trad = unicode(trad, 'utf8')
-        if not isinstance(source, unicode):
-            source = unicode(source, 'utf8')
-
-        msg = u"msgid %s\n"      \
-              u"msgstr %s\n\n"   \
-                  % (quote(source), quote(trad))
+        msg = (
+            u"msgid %s\n"
+            u"msgstr %s\n\n"
+        ) % (
+            quote(pycompat.text_type(source)), 
+            quote(pycompat.text_type(trad))
+        )
         self.buffer.write(msg)
 
 
@@ -1109,7 +1106,7 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
                 return
 
             if isinstance(res_id, pycompat.integer_types) or \
-                    (isinstance(res_id, basestring) and res_id.isdigit()):
+                    (isinstance(res_id, pycompat.string_types) and res_id.isdigit()):
                 dic['res_id'] = int(res_id)
                 if module_name:
                     dic['module'] = module_name
diff --git a/odoo/tools/yaml_import.py b/odoo/tools/yaml_import.py
index 4cb2e7ea184..6cf4cc357eb 100644
--- a/odoo/tools/yaml_import.py
+++ b/odoo/tools/yaml_import.py
@@ -12,8 +12,7 @@ from lxml import etree
 import yaml
 
 import odoo
-from . import assertion_report
-from . import yaml_tag, pycompat
+from . import assertion_report, pycompat, yaml_tag
 from .config import config
 from .misc import file_open, DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT
 from odoo import SUPERUSER_ID
@@ -25,9 +24,6 @@ from .safe_eval import safe_eval
 
 _logger = logging.getLogger(__name__)
 
-def encode(s):
-    return s.encode('utf8') if isinstance(s, unicode) else s
-
 class YamlImportException(Exception):
     pass
 
@@ -41,7 +37,7 @@ def _is_yaml_mapping(node, tag_constructor):
     return value
 
 def is_comment(node):
-    return isinstance(node, basestring)
+    return isinstance(node, pycompat.string_types)
 
 def is_assert(node):
     return isinstance(node, yaml_tag.Assert) \
@@ -87,7 +83,7 @@ def is_ir_set(node):
     return _is_yaml_mapping(node, yaml_tag.IrSet)
 
 def is_string(node):
-    return isinstance(node, basestring)
+    return isinstance(node, pycompat.string_types)
 
 class RecordDictWrapper(dict):
     """
@@ -278,7 +274,7 @@ class YamlInterpreter(object):
         record, fields = next(pycompat.items(node))
         model = self.env[record.model]
         view_id = record.view
-        if view_id and (view_id is not True) and isinstance(view_id, basestring):
+        if view_id and (view_id is not True) and isinstance(view_id, pycompat.string_types):
             if '.' not in view_id:
                 view_id = self.module + '.' + view_id
             view_id = self.env.ref(view_id).id
@@ -358,7 +354,10 @@ class YamlInterpreter(object):
                         traverse(child, elems)
 
             elems = OrderedDict()
-            traverse(etree.fromstring(encode(view['arch'])), elems)
+            arch = view['arch']
+            if isinstance(arch, pycompat.text_type):
+                arch = arch.encode('utf-8')
+            traverse(etree.fromstring(arch), elems)
             return elems
 
         def is_readonly(field_elem):
@@ -584,7 +583,7 @@ class YamlInterpreter(object):
         assert python.model or python.id, "!python node must have attribute `model` or `id`"
         if python.id is None:
             record = self.env[python.model]
-        elif isinstance(python.id, basestring):
+        elif isinstance(python.id, pycompat.string_types):
             record = self.get_record(python.id)
         else:
             record = self.env[python.model].browse(python.id)