[REF] move support for sparse fields into a separate module (#15070)

This removal has led to a necessary refactoring:
- make the setup of field attributes extensible;
- make the instantiation of custom fields extensible;
- delegate model and field reflection to `ir.model` and `ir.model.fields`;
- move the implementation of sparse and serialized fields to module `base_sparse_field`;

8 files changed, 262 insertions, 0 deletions

diff --git a/addons/base_sparse_field/__init__.py b/addons/base_sparse_field/__init__.py
new file mode 100644
index 00000000000..edbaa61c052
--- /dev/null
+++ b/addons/base_sparse_field/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+from . import models
+from . import tests
+
+def _update_database_schema(cr):
+    # The reflection of fields is done before updating the model's schema,
+    # therefore we first update the schema.
+    cr.execute("""
+        ALTER TABLE ir_model_fields
+        ADD COLUMN serialization_field_id integer REFERENCES ir_model_fields ON DELETE cascade
+    """)
diff --git a/addons/base_sparse_field/__manifest__.py b/addons/base_sparse_field/__manifest__.py
new file mode 100644
index 00000000000..031a6c179de
--- /dev/null
+++ b/addons/base_sparse_field/__manifest__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+{
+    'name': "Sparse Fields",
+    'summary': """Implementation of sparse fields.""",
+    'description': """
+        The purpose of this module is to implement "sparse" fields, i.e., fields
+        that are mostly null. This implementation circumvents the PostgreSQL
+        limitation on the number of columns in a table. The values of all sparse
+        fields are stored in a "serialized" field in the form of a JSON mapping.
+    """,
+    'category': 'Technical Settings',
+    'version': '1.0',
+    'depends': ['base'],
+    'data': [
+        'views/views.xml',
+    ],
+    'pre_init_hook': '_update_database_schema',
+}
diff --git a/addons/base_sparse_field/models/__init__.py b/addons/base_sparse_field/models/__init__.py
new file mode 100644
index 00000000000..37c0f9345cc
--- /dev/null
+++ b/addons/base_sparse_field/models/__init__.py
@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+from . import fields
+from . import models
diff --git a/addons/base_sparse_field/models/fields.py b/addons/base_sparse_field/models/fields.py
new file mode 100644
index 00000000000..cb1e1981193
--- /dev/null
+++ b/addons/base_sparse_field/models/fields.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+import json
+
+from odoo import fields
+
+
+def monkey_patch(cls):
+    """ Return a method decorator to monkey-patch the given class. """
+    def decorate(func):
+        name = func.__name__
+        func.super = getattr(cls, name, None)
+        setattr(cls, name, func)
+        return func
+    return decorate
+
+
+#
+# Implement sparse fields by monkey-patching fields.Field
+#
+
+fields.Field.__doc__ += """
+
+        .. _field-sparse:
+
+        .. rubric:: Sparse fields
+
+        Sparse fields have a very small probability of being not null. Therefore
+        many such fields can be serialized compactly into a common location, the
+        latter being a so-called "serialized" field.
+
+        :param sparse: the name of the field where the value of this field must
+            be stored.
+"""
+
+@monkey_patch(fields.Field)
+def _get_attrs(self, model, name):
+    attrs = _get_attrs.super(self, model, name)
+    if attrs.get('sparse'):
+        # by default, sparse fields are not stored and not copied
+        attrs['store'] = False
+        attrs['copy'] = attrs.get('copy', False)
+        attrs['compute'] = self._compute_sparse
+        if not attrs.get('readonly'):
+            attrs['inverse'] = self._inverse_sparse
+    return attrs
+
+@monkey_patch(fields.Field)
+def _compute_sparse(self, records):
+    for record in records:
+        values = record[self.sparse]
+        record[self.name] = values.get(self.name)
+    if self.relational:
+        for record in records:
+            record[self.name] = record[self.name].exists()
+
+@monkey_patch(fields.Field)
+def _inverse_sparse(self, records):
+    for record in records:
+        values = record[self.sparse]
+        value = self.convert_to_read(record[self.name], record, use_name_get=False)
+        if value:
+            if values.get(self.name) != value:
+                values[self.name] = value
+                record[self.sparse] = values
+        else:
+            if self.name in values:
+                values.pop(self.name)
+                record[self.sparse] = values
+
+
+#
+# Definition and implementation of serialized fields
+#
+
+class Serialized(fields.Field):
+    """ Serialized fields provide the storage for sparse fields. """
+    type = 'serialized'
+    _slots = {
+        'prefetch': False,              # not prefetched by default
+    }
+    column_type = ('text', 'text')
+
+    def convert_to_column(self, value, record):
+        return json.dumps(value)
+
+    def convert_to_cache(self, value, record, validate=True):
+        # cache format: dict
+        value = value or {}
+        return value if isinstance(value, dict) else json.loads(value)
+
+fields.Serialized = Serialized
diff --git a/addons/base_sparse_field/models/models.py b/addons/base_sparse_field/models/models.py
new file mode 100644
index 00000000000..b0ea825b7d3
--- /dev/null
+++ b/addons/base_sparse_field/models/models.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from odoo import models, fields, api
+
+
+class IrModelFields(models.Model):
+    _inherit = 'ir.model.fields'
+
+    ttype = fields.Selection(selection_add=[('serialized', 'serialized')])
+    serialization_field_id = fields.Many2one('ir.model.fields', string='Serialization Field',
+        ondelete='cascade', domain="[('ttype','=','serialized')]",
+        help="If set, this field will be stored in the sparse structure of the "
+             "serialization field, instead of having its own database column. "
+             "This cannot be changed after creation.",
+    )
+
+    @api.multi
+    def write(self, vals):
+        # Limitation: renaming a sparse field or changing the storing system is
+        # currently not allowed
+        if 'serialization_field_id' in vals or 'name' in vals:
+            for field in self:
+                if 'serialization_field_id' in vals and field.serialization_field_id.id != vals['serialization_field_id']:
+                    raise UserError(_('Changing the storing system for field "%s" is not allowed.') % field.name)
+                if field.serialization_field_id and (field.name != vals['name']):
+                    raise UserError(_('Renaming sparse field "%s" is not allowed') % field.name)
+
+        return super(IrModelFields, self).write(vals)
+
+    def _reflect_values(self, field, existing):
+        vals = super(IrModelFields, self)._reflect_values(field, existing)
+
+        vals['serialization_field_id'] = None
+        if getattr(field, 'sparse', None):
+            model = self.env[field.model_name]
+            serialization_field = model._fields.get(field.sparse)
+            if serialization_field is None:
+                raise UserError(_("Serialization field `%s` not found for sparse field `%s`!") % (field.sparse, field.name))
+            serialization_record = self._reflect(serialization_field, existing)
+            vals['serialization_field_id'] = serialization_record.id
+
+        return vals
+
+    def _instanciate_attrs(self, field_data, partial):
+        attrs = super(IrModelFields, self)._instanciate_attrs(field_data, partial)
+        if field_data['serialization_field_id']:
+            serialization_record = self.browse(field_data['serialization_field_id'])
+            attrs['sparse'] = serialization_record.name
+        return attrs
+
+
+class TestSparse(models.TransientModel):
+    _name = 'sparse_fields.test'
+
+    data = fields.Serialized()
+    boolean = fields.Boolean(sparse='data')
+    integer = fields.Integer(sparse='data')
+    float = fields.Float(sparse='data')
+    char = fields.Char(sparse='data')
+    selection = fields.Selection([('one', 'One'), ('two', 'Two')], sparse='data')
+    partner = fields.Many2one('res.partner', sparse='data')
diff --git a/addons/base_sparse_field/tests/__init__.py b/addons/base_sparse_field/tests/__init__.py
new file mode 100644
index 00000000000..1dc570533d5
--- /dev/null
+++ b/addons/base_sparse_field/tests/__init__.py
@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import test_sparse_fields
diff --git a/addons/base_sparse_field/tests/test_sparse_fields.py b/addons/base_sparse_field/tests/test_sparse_fields.py
new file mode 100644
index 00000000000..7c7e8264d34
--- /dev/null
+++ b/addons/base_sparse_field/tests/test_sparse_fields.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+from odoo.tests import common
+
+
+class TestSparseFields(common.TransactionCase):
+
+    def test_sparse(self):
+        """ test sparse fields. """
+        record = self.env['sparse_fields.test'].create({})
+        self.assertFalse(record.data)
+
+        partner = self.env.ref('base.main_partner')
+        values = [
+            ('boolean', True),
+            ('integer', 42),
+            ('float', 3.14),
+            ('char', 'John'),
+            ('selection', 'two'),
+            ('partner', partner.id),
+        ]
+        for n, (key, val) in enumerate(values):
+            record.write({key: val})
+            self.assertEqual(record.data, dict(values[:n+1]))
+
+        for key, val in values[:-1]:
+            self.assertEqual(record[key], val)
+        self.assertEqual(record.partner, partner)
+
+        for n, (key, val) in enumerate(values):
+            record.write({key: False})
+            self.assertEqual(record.data, dict(values[n+1:]))
+
+        # check reflection of sparse fields in 'ir.model.fields'
+        names = [name for name, _ in values]
+        domain = [('model', '=', 'sparse_fields.test'), ('name', 'in', names)]
+        fields = self.env['ir.model.fields'].search(domain)
+        self.assertEqual(len(fields), len(names))
+        for field in fields:
+            self.assertEqual(field.serialization_field_id.name, 'data')
diff --git a/addons/base_sparse_field/views/views.xml b/addons/base_sparse_field/views/views.xml
new file mode 100644
index 00000000000..95c3d23634a
--- /dev/null
+++ b/addons/base_sparse_field/views/views.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <data>
+
+        <!-- Add 'serialization_field_id' in ir.model form view -->
+        <record model="ir.ui.view" id="model_form_view">
+            <field name="model">ir.model</field>
+            <field name="inherit_id" ref="base.view_model_form"/>
+            <field name="arch" type="xml">
+                <field name="related" position="before">
+                    <field name="serialization_field_id"
+                        domain="[('ttype','=','serialized'), ('model_id','=',parent.model)]"
+                        attrs="{'readonly': [('state','=','base')]}"/>
+                </field>
+            </field>
+        </record>
+
+        <!-- Add 'serialization_field_id' in ir.model.fields form view -->
+        <record model="ir.ui.view" id="field_form_view">
+            <field name="model">ir.model.fields</field>
+            <field name="inherit_id" ref="base.view_model_fields_form"/>
+            <field name="arch" type="xml">
+                <field name="related" position="before">
+                    <field name="serialization_field_id"
+                        domain="[('ttype','=','serialized'), ('model_id','=',model_id)]"
+                        attrs="{'readonly': [('state','=','base')]}"/>
+                </field>
+            </field>
+        </record>
+
+    </data>
+</odoo>