[MERGE] forward port branch 11.0 up to 36d96e01503

4 files changed, 203 insertions, 61 deletions

diff --git a/odoo/addons/base/tests/__init__.py b/odoo/addons/base/tests/__init__.py
index 069afe0cec1..a0099045a80 100644
--- a/odoo/addons/base/tests/__init__.py
+++ b/odoo/addons/base/tests/__init__.py
@@ -2,6 +2,7 @@ from . import test_acl
 from . import test_api
 from . import test_base
 from . import test_basecase
+from . import test_cache
 from . import test_db_cursor
 from . import test_expression
 from . import test_float
diff --git a/odoo/addons/base/tests/test_cache.py b/odoo/addons/base/tests/test_cache.py
new file mode 100644
index 00000000000..3c8fbd9f603
--- /dev/null
+++ b/odoo/addons/base/tests/test_cache.py
@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+import os
+import psutil
+
+from odoo.tests.common import TransactionCase
+
+
+class TestRecordCache(TransactionCase):
+
+    def test_cache(self):
+        """ Check the record cache object. """
+        Model = self.env['res.partner']
+        name = type(Model).name
+        ref = type(Model).ref
+
+        cache = self.env.cache
+
+        def check1(record, field, value):
+            # value is None means no value in cache
+            self.assertEqual(cache.contains(record, field), value is not None)
+            self.assertEqual(cache.contains_value(record, field), value is not None)
+            self.assertEqual(cache.get_value(record, field), value)
+            try:
+                self.assertEqual(cache.get(record, field), value)
+                self.assertIsNotNone(value)
+            except KeyError:
+                self.assertIsNone(value)
+            self.assertIsNone(cache.get_special(record, field))
+            self.assertEqual(field in cache.get_fields(record), value is not None)
+            self.assertEqual(record in cache.get_records(record, field), value is not None)
+
+        def check(record, name_val, ref_val):
+            """ check the values of fields 'name' and 'ref' on record. """
+            check1(record, name, name_val)
+            check1(record, ref, ref_val)
+
+        foo1, bar1 = Model.browse([1, 2])
+        foo2, bar2 = Model.sudo(self.env.ref('base.user_demo')).browse([1, 2])
+        self.assertNotEqual(foo1.env.uid, foo2.env.uid)
+
+        # cache is empty
+        cache.invalidate()
+        check(foo1, None, None)
+        check(foo2, None, None)
+        check(bar1, None, None)
+        check(bar2, None, None)
+        self.assertCountEqual(cache.get_missing_ids(foo1 + bar1, name), [1, 2])
+        self.assertCountEqual(cache.get_missing_ids(foo2 + bar2, name), [1, 2])
+
+        # set values in one environment only
+        for rec in [foo1, bar1]:
+            cache.set(rec, name, 'NAME1')
+            cache.set(rec, ref, 'REF1')
+        check(foo1, 'NAME1', 'REF1')
+        check(foo2, None, None)
+        check(bar1, 'NAME1', 'REF1')
+        check(bar2, None, None)
+        self.assertCountEqual(cache.get_missing_ids(foo1 + bar1, name), [])
+        self.assertCountEqual(cache.get_missing_ids(foo2 + bar2, name), [1, 2])
+
+        # set values in both environments
+        for rec in [foo2, bar2]:
+            cache.set(rec, name, 'NAME2')
+            cache.set(rec, ref, 'REF2')
+        check(foo1, 'NAME1', 'REF1')
+        check(foo2, 'NAME2', 'REF2')
+        check(bar1, 'NAME1', 'REF1')
+        check(bar2, 'NAME2', 'REF2')
+        self.assertCountEqual(cache.get_missing_ids(foo1 + bar1, name), [])
+        self.assertCountEqual(cache.get_missing_ids(foo2 + bar2, name), [])
+
+        # remove value in one environment
+        cache.remove(foo1, name)
+        check(foo1, None, 'REF1')
+        check(foo2, 'NAME2', 'REF2')
+        check(bar1, 'NAME1', 'REF1')
+        check(bar2, 'NAME2', 'REF2')
+        self.assertCountEqual(cache.get_missing_ids(foo1 + bar1, name), [1])
+        self.assertCountEqual(cache.get_missing_ids(foo2 + bar2, name), [])
+
+        # partial invalidation
+        cache.invalidate([(name, None), (ref, foo1.ids)])
+        check(foo1, None, None)
+        check(foo2, None, None)
+        check(bar1, None, 'REF1')
+        check(bar2, None, 'REF2')
+
+        # total invalidation
+        cache.invalidate()
+        check(foo1, None, None)
+        check(foo2, None, None)
+        check(bar1, None, None)
+        check(bar2, None, None)
+
+        # set a special value
+        cache.set_special(foo1, name, lambda: '42')
+        self.assertTrue(cache.contains(foo1, name))
+        self.assertFalse(cache.contains_value(foo1, name))
+        self.assertEqual(cache.get(foo1, name), '42')
+        self.assertIsNone(cache.get_value(foo1, name))
+        self.assertIsNotNone(cache.get_special(foo1, name))
+
+        # copy cache
+        for rec in [foo1, bar1]:
+            cache.set(rec, name, 'NAME1')
+            cache.set(rec, ref, 'REF1')
+        check(foo1, 'NAME1', 'REF1')
+        check(foo2, None, None)
+        check(bar1, 'NAME1', 'REF1')
+        check(bar2, None, None)
+
+        cache.copy(foo1 + bar1, foo2.env)
+        check(foo1, 'NAME1', 'REF1')
+        check(foo2, 'NAME1', 'REF1')
+        check(bar1, 'NAME1', 'REF1')
+        check(bar2, 'NAME1', 'REF1')
+
+    def test_memory(self):
+        """ Check memory consumption of the cache. """
+        NB_RECORDS = 100000
+        MAX_MEMORY = 100
+
+        cache = self.env.cache
+        model = self.env['res.partner']
+        records = [model.new() for index in range(NB_RECORDS)]
+
+        process = psutil.Process(os.getpid())
+        rss0 = process.memory_info().rss
+
+        char_names = [
+            'name', 'display_name', 'email', 'website', 'phone', 'mobile',
+            'street', 'street2', 'city', 'zip', 'vat', 'ref',
+        ]
+        for name in char_names:
+            field = model._fields[name]
+            for record in records:
+                cache.set(record, field, 'test')
+
+        mem_usage = process.memory_info().rss - rss0
+        self.assertLess(
+            mem_usage, MAX_MEMORY * 1024 * 1024,
+            "Caching %s records must take less than %sMB of memory" % (NB_RECORDS, MAX_MEMORY),
+        )
diff --git a/odoo/api.py b/odoo/api.py
index 075d9ac3cab..9b904c497f3 100644
--- a/odoo/api.py
+++ b/odoo/api.py
@@ -741,6 +741,7 @@ class Environment(Mapping):
         self.cr, self.uid, self.context = self.args = (cr, uid, frozendict(context))
         self.registry = Registry(cr.dbname)
         self.cache = envs.cache
+        self._cache_key = (cr, uid)
         self._protected = StackMap()                # {field: ids, ...}
         self.dirty = defaultdict(set)               # {record: set(field_name), ...}
         self.all = envs
@@ -925,6 +926,12 @@ class Environment(Mapping):
         finally:
             self.all.recompute = tmp
 
+    def cache_key(self, field):
+        """ Return the key to store the value of ``field`` in cache, the full
+            cache key being ``(key, field, record.id)``.
+        """
+        return self if field.context_dependent else self._cache_key
+
 
 class Environments(object):
     """ A common object for all environments in a request. """
@@ -947,52 +954,52 @@ class Environments(object):
 class Cache(object):
     """ Implementation of the cache of records. """
     def __init__(self):
-        # {field: {record_id: {key: value}}}
+        # {key: {field: {record_id: value}}}
         self._data = defaultdict(lambda: defaultdict(dict))
 
     def contains(self, record, field):
         """ Return whether ``record`` has a value for ``field``. """
-        key = field.cache_key(record)
-        return key in self._data[field].get(record.id, ())
+        key = record.env.cache_key(field)
+        return record.id in self._data[key].get(field, ())
 
     def get(self, record, field):
         """ Return the value of ``field`` for ``record``. """
-        key = field.cache_key(record)
-        value = self._data[field][record.id][key]
+        key = record.env.cache_key(field)
+        value = self._data[key][field][record.id]
         return value.get() if isinstance(value, SpecialValue) else value
 
     def set(self, record, field, value):
         """ Set the value of ``field`` for ``record``. """
-        key = field.cache_key(record)
-        self._data[field][record.id][key] = value
+        key = record.env.cache_key(field)
+        self._data[key][field][record.id] = value
 
     def remove(self, record, field):
         """ Remove the value of ``field`` for ``record``. """
-        key = field.cache_key(record)
-        del self._data[field][record.id][key]
+        key = record.env.cache_key(field)
+        del self._data[key][field][record.id]
 
     def contains_value(self, record, field):
         """ Return whether ``record`` has a regular value for ``field``. """
-        key = field.cache_key(record)
-        value = self._data[field][record.id].get(key, SpecialValue(None))
+        key = record.env.cache_key(field)
+        value = self._data[key][field].get(record.id, SpecialValue(None))
         return not isinstance(value, SpecialValue)
 
     def get_value(self, record, field, default=None):
         """ Return the regular value of ``field`` for ``record``. """
-        key = field.cache_key(record)
-        value = self._data[field][record.id].get(key, SpecialValue(None))
+        key = record.env.cache_key(field)
+        value = self._data[key][field].get(record.id, SpecialValue(None))
         return default if isinstance(value, SpecialValue) else value
 
     def get_special(self, record, field, default=None):
         """ Return the special value of ``field`` for ``record``. """
-        key = field.cache_key(record)
-        value = self._data[field][record.id].get(key)
+        key = record.env.cache_key(field)
+        value = self._data[key][field].get(record.id)
         return value.get if isinstance(value, SpecialValue) else default
 
     def set_special(self, record, field, getter):
         """ Set the value of ``field`` for ``record`` to return ``getter()``. """
-        key = field.cache_key(record)
-        self._data[field][record.id][key] = SpecialValue(getter)
+        key = record.env.cache_key(field)
+        self._data[key][field][record.id] = SpecialValue(getter)
 
     def set_failed(self, records, fields, exception):
         """ Mark ``fields`` on ``records`` with the given exception. """
@@ -1005,67 +1012,66 @@ class Cache(object):
     def get_fields(self, record):
         """ Return the fields with a value for ``record``. """
         for name, field in record._fields.items():
-            key = field.cache_key(record)
-            if name != 'id' and key in self._data[field].get(record.id, ()):
+            key = record.env.cache_key(field)
+            if name != 'id' and record.id in self._data[key].get(field, ()):
                 yield field
 
     def get_records(self, model, field):
         """ Return the records of ``model`` that have a value for ``field``. """
-        key = field.cache_key(model)
-        # optimization: do not field.cache_key(record) for each record in cache
-        ids = [
-            record_id
-            for record_id, field_record_cache in self._data[field].items()
-            if key in field_record_cache
-        ]
+        key = model.env.cache_key(field)
+        ids = list(self._data[key][field])
         return model.browse(ids)
 
     def get_missing_ids(self, records, field):
         """ Return the ids of ``records`` that have no value for ``field``. """
-        key = field.cache_key(records)
-        field_cache = self._data[field]
+        key = records.env.cache_key(field)
+        field_cache = self._data[key][field]
         for record_id in records._ids:
-            if key not in field_cache.get(record_id, ()):
+            if record_id not in field_cache:
                 yield record_id
 
     def copy(self, records, env):
         """ Copy the cache of ``records`` to ``env``. """
-        src = records
-        dst = records.with_env(env)
-        for field, field_cache in self._data.items():
-            src_key = field.cache_key(src)
-            dst_key = field.cache_key(dst)
-            for record_cache in field_cache.values():
-                if src_key in record_cache and not isinstance(record_cache[src_key], SpecialValue):
-                    # But not if it's a SpecialValue, which often is an access error
-                    # because the other environment (eg. sudo()) is well expected to have access.
-                    record_cache[dst_key] = record_cache[src_key]
+        src, dst = records.env, env
+        for src_key, dst_key in [(src, dst), (src._cache_key, dst._cache_key)]:
+            if src_key == dst_key:
+                break
+            src_cache = self._data[src_key]
+            dst_cache = self._data[dst_key]
+            for field, src_field_cache in src_cache.items():
+                dst_field_cache = dst_cache[field]
+                for record_id, value in src_field_cache.items():
+                    if not isinstance(value, SpecialValue):
+                        # But not if it's a SpecialValue, which often is an access error
+                        # because the other environment (eg. sudo()) is well expected to have access.
+                        dst_field_cache[record_id] = value
 
     def invalidate(self, spec=None):
         """ Invalidate the cache, partially or totally depending on ``spec``. """
         if spec is None:
             self._data.clear()
         elif spec:
-            data = self._data
             for field, ids in spec:
                 if ids is None:
-                    data.pop(field, None)
+                    for data in self._data.values():
+                        data.pop(field, None)
                 else:
-                    field_cache = data[field]
-                    for id in ids:
-                        field_cache.pop(id, None)
+                    for data in self._data.values():
+                        field_cache = data.get(field)
+                        if field_cache:
+                            for id in ids:
+                                field_cache.pop(id, None)
 
     def check(self, env):
         """ Check the consistency of the cache for the given environment. """
         # make a full copy of the cache, and invalidate it
         dump = defaultdict(dict)
-        for field, field_cache in self._data.items():
-            browse = env[field.model_name].browse
-            for record_id, field_record_cache in field_cache.items():
-                if record_id:
-                    key = field.cache_key(browse(record_id))
-                    if key in field_record_cache:
-                        dump[field][record_id] = field_record_cache[key]
+        for key in [env, env._cache_key]:
+            key_cache = self._data[key]
+            for field, field_cache in key_cache.items():
+                for record_id, value in field_cache.items():
+                    if record_id:
+                        dump[field][record_id] = value
 
         self.invalidate()
 
diff --git a/odoo/fields.py b/odoo/fields.py
index d39fa6e35fa..36d815215ff 100644
--- a/odoo/fields.py
+++ b/odoo/fields.py
@@ -769,16 +769,6 @@ class Field(MetaField('DummyField', (object,), {})):
     # Conversion of values
     #
 
-    def cache_key(self, record):
-        """ Return the key to get/set the value of ``self`` on ``record`` in
-            cache, the full cache key being ``(self, record.id, key)``.
-        """
-        env = record.env
-        # IMPORTANT: odoo.api.Cache.get_records() depends on the fact that the
-        # result does not depend on record.id. If you ever make the following
-        # dependent on record.id, don't forget to fix the other method!
-        return env if self.context_dependent else (env.cr, env.uid)
-
     def null(self, record):
         """ Return the null value for this field in the record format. """
         return False