[FIX] P3: list -> iterable builtins (#16811)

In Python 3:

* various builtins and dict methods were changed to return
  view/iterable objects rather than lists
* and the separate Python 2 view/iterable builtins and methods were
  removed altogether

This is problematic when using these items as list (which the happens
repeatedly in Odoo), but more viciously when iterating *multiple times*
over them (which also happens, which I've messed up multiple times while
writing this, and which is a pain to debug even when you've just created
the issue).

Convert all code using these to semantics-matching cross-version
helper functions to get the LCD behaviour between P2 and P3, and
forbid the builtins via lint.

issue #8530

6 files changed, 43 insertions, 33 deletions

diff --git a/odoo/modules/db.py b/odoo/modules/db.py
index 7106ff82b69..bde759843cf 100644
--- a/odoo/modules/db.py
+++ b/odoo/modules/db.py
@@ -99,7 +99,7 @@ def create_categories(cr, categories):
     category = []
     while categories:
         category.append(categories[0])
-        xml_id = 'module_category_' + ('_'.join(map(lambda x: x.lower(), category))).replace('&', 'and').replace(' ', '_')
+        xml_id = 'module_category_' + ('_'.join(x.lower() for x in category)).replace('&', 'and').replace(' ', '_')
         # search via xml_id (because some categories are renamed)
         cr.execute("SELECT res_id FROM ir_model_data WHERE name=%s AND module=%s AND model=%s",
                    (xml_id, "base", "ir.module.category"))
diff --git a/odoo/modules/graph.py b/odoo/modules/graph.py
index d962ea78bdc..731b6f6557a 100644
--- a/odoo/modules/graph.py
+++ b/odoo/modules/graph.py
@@ -8,7 +8,7 @@ import logging
 
 import odoo
 import odoo.tools as tools
-
+from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 
@@ -36,7 +36,7 @@ class Graph(dict):
             return
         # update the graph with values from the database (if exist)
         ## First, we set the default values for each package in graph
-        additional_data = dict((key, {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None}) for key in self.keys())
+        additional_data = {key: {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None} for key in pycompat.keys(self)}
         ## Then we get the values from the database
         cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version'
                    '  FROM ir_module_module'
@@ -46,8 +46,8 @@ class Graph(dict):
         ## and we update the default values with values from the database
         additional_data.update((x['name'], x) for x in cr.dictfetchall())
 
-        for package in self.values():
-            for k, v in additional_data[package.name].items():
+        for package in pycompat.values(self):
+            for k, v in pycompat.items(additional_data[package.name]):
                 setattr(package, k, v)
 
     def add_module(self, cr, module, force=None):
@@ -94,7 +94,7 @@ class Graph(dict):
         self.update_from_db(cr)
 
         for package in later:
-            unmet_deps = filter(lambda p: p not in self, dependencies[package])
+            unmet_deps = [p for p in dependencies[package] if p not in self]
             _logger.error('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps))
 
         return len(self) - len_graph
@@ -102,9 +102,9 @@ class Graph(dict):
 
     def __iter__(self):
         level = 0
-        done = set(self.keys())
+        done = set(pycompat.keys(self))
         while done:
-            level_modules = sorted((name, module) for name, module in self.items() if module.depth==level)
+            level_modules = sorted((name, module) for name, module in pycompat.items(self) if module.depth==level)
             for name, module in level_modules:
                 done.remove(name)
                 yield module
@@ -164,7 +164,10 @@ class Node(object):
                 setattr(child, name, value + 1)
 
     def __iter__(self):
-        return itertools.chain(iter(self.children), *map(iter, self.children))
+        return itertools.chain(
+            self.children,
+            itertools.chain.from_iterable(self.children)
+        )
 
     def __str__(self):
         return self._pprint()
diff --git a/odoo/modules/loading.py b/odoo/modules/loading.py
index f8c65d40177..bd58376a7b9 100644
--- a/odoo/modules/loading.py
+++ b/odoo/modules/loading.py
@@ -20,6 +20,7 @@ import odoo.tools as tools
 
 from odoo import api, SUPERUSER_ID
 from odoo.modules.module import adapt_version, initialize_sys_path, load_openerp_module
+from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 _test_logger = logging.getLogger('odoo.tests')
@@ -295,15 +296,15 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
                 _logger.info('updating modules list')
                 Module.update_list()
 
-            _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
+            _check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
 
-            module_names = [k for k, v in tools.config['init'].items() if v]
+            module_names = [k for k, v in pycompat.items(tools.config['init']) if v]
             if module_names:
                 modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
                 if modules:
                     modules.button_install()
 
-            module_names = [k for k, v in tools.config['update'].items() if v]
+            module_names = [k for k, v in pycompat.items(tools.config['update']) if v]
             if module_names:
                 modules = Module.search([('state', '=', 'installed'), ('name', 'in', module_names)])
                 if modules:
@@ -391,7 +392,7 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
                         getattr(py_module, uninstall_hook)(cr, registry)
 
                 Module = env['ir.module.module']
-                Module.browse(modules_to_remove.values()).module_uninstall()
+                Module.browse(pycompat.values(modules_to_remove)).module_uninstall()
                 # Recursive reload, should only happen once, because there should be no
                 # modules to remove next time
                 cr.commit()
@@ -414,7 +415,7 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
             _logger.info('Modules loaded.')
 
         # STEP 8: call _register_hook on every model
-        for model in env.values():
+        for model in pycompat.values(env):
             model._register_hook()
 
         # STEP 9: save installed/updated modules for post-install tests
diff --git a/odoo/modules/migration.py b/odoo/modules/migration.py
index 5b1dcf09ab2..0fa6699cf24 100644
--- a/odoo/modules/migration.py
+++ b/odoo/modules/migration.py
@@ -13,6 +13,7 @@ from os.path import join as opj
 from odoo.modules.module import get_resource_path
 import odoo.release as release
 import odoo.tools as tools
+from odoo.tools import pycompat
 from odoo.tools.parse_version import parse_version
 
 
@@ -87,13 +88,12 @@ class MigrationManager(object):
             return "%s.%s" % (release.major_version, version)
 
         def _get_migration_versions(pkg):
-            versions = list(set(
+            versions = sorted({
                 ver
-                for lv in self.migrations[pkg.name].values()
-                for ver, lf in lv.items()
+                for lv in pycompat.values(self.migrations[pkg.name])
+                for ver, lf in pycompat.items(lv)
                 if lf
-            ))
-            versions.sort(key=lambda k: parse_version(convert_version(k)))
+            }, key=lambda k: parse_version(convert_version(k)))
             return versions
 
         def _get_migration_files(pkg, version, stage):
@@ -107,7 +107,7 @@ class MigrationManager(object):
                 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name),
             }
 
-            for x in mapping.keys():
+            for x in mapping:
                 if version in m.get(x):
                     for f in m[x][version]:
                         if not f.startswith(stage + '-'):
diff --git a/odoo/modules/module.py b/odoo/modules/module.py
index dd1485e1d8a..e6194a5fb10 100644
--- a/odoo/modules/module.py
+++ b/odoo/modules/module.py
@@ -23,6 +23,7 @@ import odoo
 import odoo.tools as tools
 import odoo.release as release
 from odoo import SUPERUSER_ID, api
+from odoo.tools import pycompat
 
 MANIFEST_NAMES = ('__manifest__.py', '__openerp__.py')
 README = ['README.rst', 'README.md', 'README.txt']
@@ -323,7 +324,7 @@ def load_information_from_description_file(module, mod_path=None):
             'sequence': 100,
             'summary': '',
         }
-        info.update(itertools.izip(
+        info.update(pycompat.izip(
             'depends data demo test init_xml update_xml demo_xml'.split(),
             iter(list, None)))
 
@@ -395,7 +396,11 @@ def get_modules():
             for mname in MANIFEST_NAMES:
                 if os.path.isfile(opj(dir, name, mname)):
                     return True
-        return map(clean, filter(is_really_module, os.listdir(dir)))
+        return [
+            clean(it)
+            for it in os.listdir(dir)
+            if is_really_module(it)
+        ]
 
     plist = []
     initialize_sys_path()
@@ -492,7 +497,7 @@ def run_unit_tests(module_name, dbname, position=runs_at_install):
     r = True
     for m in mods:
         tests = unwrap_suite(unittest.TestLoader().loadTestsFromModule(m))
-        suite = unittest.TestSuite(itertools.ifilter(position, tests))
+        suite = unittest.TestSuite(t for t in tests if position(t))
 
         if suite.countTestCases():
             t0 = time.time()
@@ -532,5 +537,5 @@ def unwrap_suite(test):
         return
 
     for item in itertools.chain.from_iterable(
-            itertools.imap(unwrap_suite, subtests)):
+            unwrap_suite(t) for t in subtests):
         yield item
diff --git a/odoo/modules/registry.py b/odoo/modules/registry.py
index e0cfb532fd1..71d7b5d9fbe 100644
--- a/odoo/modules/registry.py
+++ b/odoo/modules/registry.py
@@ -16,7 +16,8 @@ import threading
 import odoo
 from .. import SUPERUSER_ID
 from odoo.tools import (assertion_report, lazy_classproperty, config,
-                        lazy_property, table_exists, topological_sort, OrderedSet)
+                        lazy_property, table_exists, topological_sort,
+                        OrderedSet, pycompat)
 from odoo.tools.lru import LRU
 
 _logger = logging.getLogger(__name__)
@@ -156,7 +157,7 @@ class Registry(Mapping):
     def delete_all(cls):
         """ Delete all the registries. """
         with cls._lock:
-            for db_name in cls.registries.keys():
+            for db_name in list(pycompat.keys(cls.registries)):
                 cls.delete(db_name)
 
     #
@@ -192,8 +193,8 @@ class Registry(Mapping):
         # map fields on their dependents
         dependents = {
             field: set(dep for dep, _ in model._field_triggers[field] if dep != field)
-            for model in self.itervalues()
-            for field in model._fields.itervalues()
+            for model in pycompat.values(self)
+            for field in pycompat.values(model._fields)
         }
         # sort them topologically, and associate a sequence number to each field
         mapping = {
@@ -261,7 +262,7 @@ class Registry(Mapping):
             env['ir.model']._add_manual_models()
 
         # prepare the setup on all models
-        models = env.values()
+        models = list(pycompat.values(env))
         for model in models:
             model._prepare_setup()
 
@@ -310,18 +311,18 @@ class Registry(Mapping):
 
         # make sure all tables are present
         missing = [name
-                   for name, model in env.items()
+                   for name, model in pycompat.items(env)
                    if not model._abstract and not table_exists(cr, model._table)]
         if missing:
             _logger.warning("Models have no table: %s.", ", ".join(missing))
             # recreate missing tables following model dependencies
-            deps = {name: model._depends for name, model in env.items()}
+            deps = {name: model._depends for name, model in pycompat.items(env)}
             for name in topological_sort(deps):
                 if name in missing:
                     _logger.info("Recreate table of model %s.", name)
                     env[name].init()
             # check again, and log errors if tables are still missing
-            for name, model in env.items():
+            for name, model in pycompat.items(env):
                 if not model._abstract and not table_exists(cr, model._table):
                     _logger.error("Model %s has no table.", name)
 
@@ -340,7 +341,7 @@ class Registry(Mapping):
         """ Clear the caches associated to methods decorated with
         ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models.
         """
-        for model in self.models.itervalues():
+        for model in pycompat.values(self.models):
             model.clear_caches()
 
     def setup_signaling(self):