[REM] *: ability to put raw modules in evaluation contexts

Co-authored-by: Xavier Morel <xmo@odoo.com>

17 files changed, 149 insertions, 127 deletions

diff --git a/addons/base_automation/models/base_automation.py b/addons/base_automation/models/base_automation.py
index e5ef6cf2f16..efab143c67d 100644
--- a/addons/base_automation/models/base_automation.py
+++ b/addons/base_automation/models/base_automation.py
@@ -3,17 +3,14 @@
 
 import datetime
 import logging
-import time
 import traceback
 from collections import defaultdict
 
-import dateutil
 from dateutil.relativedelta import relativedelta
 
 from odoo import _, api, fields, models, SUPERUSER_ID
-from odoo.modules.registry import Registry
 from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools import safe_eval
 
 _logger = logging.getLogger(__name__)
 
@@ -172,9 +169,9 @@ class BaseAutomation(models.Model):
             :returns: dict -- evaluation context given to safe_eval
         """
         return {
-            'datetime': datetime,
-            'dateutil': dateutil,
-            'time': time,
+            'datetime': safe_eval.datetime,
+            'dateutil': safe_eval.dateutil,
+            'time': safe_eval.time,
             'uid': self.env.uid,
             'user': self.env.user,
         }
@@ -198,7 +195,7 @@ class BaseAutomation(models.Model):
     def _filter_pre(self, records):
         """ Filter the records that satisfy the precondition of action ``self``. """
         if self.filter_pre_domain and records:
-            domain = [('id', 'in', records.ids)] + safe_eval(self.filter_pre_domain, self._get_eval_context())
+            domain = [('id', 'in', records.ids)] + safe_eval.safe_eval(self.filter_pre_domain, self._get_eval_context())
             return records.sudo().search(domain).with_env(records.env)
         else:
             return records
@@ -209,7 +206,7 @@ class BaseAutomation(models.Model):
     def _filter_post_export_domain(self, records):
         """ Filter the records that satisfy the postcondition of action ``self``. """
         if self.filter_domain and records:
-            domain = [('id', 'in', records.ids)] + safe_eval(self.filter_domain, self._get_eval_context())
+            domain = [('id', 'in', records.ids)] + safe_eval.safe_eval(self.filter_domain, self._get_eval_context())
             return records.sudo().search(domain).with_env(records.env), domain
         else:
             return records, None
diff --git a/addons/gamification/models/goal.py b/addons/gamification/models/goal.py
index 19e2c9a095d..ef48542a067 100644
--- a/addons/gamification/models/goal.py
+++ b/addons/gamification/models/goal.py
@@ -3,12 +3,11 @@
 
 import ast
 import logging
-import time
 from datetime import date, datetime, timedelta
 
 from odoo import api, fields, models, _, exceptions
 from odoo.osv import expression
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools.safe_eval import safe_eval, time
 
 _logger = logging.getLogger(__name__)
 
diff --git a/addons/mail/models/mail_render_mixin.py b/addons/mail/models/mail_render_mixin.py
index d35fbf5dddf..1d90284b8aa 100644
--- a/addons/mail/models/mail_render_mixin.py
+++ b/addons/mail/models/mail_render_mixin.py
@@ -3,16 +3,16 @@
 
 import babel
 import copy
-import datetime
-import dateutil.relativedelta as relativedelta
 import functools
 import logging
 import re
 
+import dateutil.relativedelta as relativedelta
 from werkzeug import urls
 
 from odoo import _, api, fields, models, tools
 from odoo.exceptions import UserError
+from odoo.tools import safe_eval
 
 _logger = logging.getLogger(__name__)
 
@@ -55,7 +55,7 @@ try:
         'str': str,
         'quote': urls.url_quote,
         'urlencode': urls.url_encode,
-        'datetime': datetime,
+        'datetime': safe_eval.datetime,
         'len': len,
         'abs': abs,
         'min': min,
@@ -326,6 +326,7 @@ class MailRenderMixin(models.AbstractModel):
         variables = self._render_jinja_eval_context()
         if add_context:
             variables.update(**add_context)
+        safe_eval.check_values(variables)
 
         # TDE CHECKME
         # records = self.env[model].browse(it for it in res_ids if it)  # filter to avoid browsing [None]
diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py
index 061c1c57926..6cac30fea6a 100644
--- a/addons/web/controllers/main.py
+++ b/addons/web/controllers/main.py
@@ -18,7 +18,6 @@ import os
 import re
 import sys
 import tempfile
-import time
 
 import werkzeug
 import werkzeug.exceptions
@@ -39,7 +38,7 @@ from odoo.tools import image_process, topological_sort, html_escape, pycompat, u
 from odoo.tools.mimetypes import guess_mimetype
 from odoo.tools.translate import _
 from odoo.tools.misc import str2bool, xlsxwriter, file_open
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools.safe_eval import safe_eval, time
 from odoo import http, tools
 from odoo.http import content_disposition, dispatch_rpc, request, serialize_exception as _serialize_exception, Response
 from odoo.exceptions import AccessError, UserError, AccessDenied
diff --git a/odoo/addons/base/models/ir_actions.py b/odoo/addons/base/models/ir_actions.py
index 929478426d1..aa047379b73 100644
--- a/odoo/addons/base/models/ir_actions.py
+++ b/odoo/addons/base/models/ir_actions.py
@@ -6,30 +6,16 @@ from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.exceptions import MissingError, UserError, ValidationError, AccessError
 from odoo.osv import expression
 from odoo.tools.safe_eval import safe_eval, test_python_expr
-from odoo.tools import wrap_module
-from odoo.http import request
 
 import base64
 from collections import defaultdict
-import datetime
 import functools
 import logging
-import time
 
 from pytz import timezone
 
 _logger = logging.getLogger(__name__)
 
-# build dateutil helper, starting with the relevant *lazy* imports
-import dateutil
-import dateutil.parser
-import dateutil.relativedelta
-import dateutil.rrule
-import dateutil.tz
-mods = {'parser', 'relativedelta', 'rrule', 'tz'}
-attribs = {atr for m in mods for atr in getattr(dateutil, m).__all__}
-dateutil = wrap_module(dateutil, mods | attribs)
-
 
 class IrActions(models.Model):
     _name = 'ir.actions.actions'
@@ -84,9 +70,9 @@ class IrActions(models.Model):
         return {
             'uid': self._uid,
             'user': self.env.user,
-            'time': time,
-            'datetime': datetime,
-            'dateutil': dateutil,
+            'time': tools.safe_eval.time,
+            'datetime': tools.safe_eval.datetime,
+            'dateutil': tools.safe_eval.dateutil,
             'timezone': timezone,
             'b64encode': base64.b64encode,
             'b64decode': base64.b64decode,
diff --git a/odoo/addons/base/models/ir_actions_report.py b/odoo/addons/base/models/ir_actions_report.py
index 9770089aba5..095f7de1b73 100644
--- a/odoo/addons/base/models/ir_actions_report.py
+++ b/odoo/addons/base/models/ir_actions_report.py
@@ -2,14 +2,13 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.exceptions import UserError, AccessError
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools.safe_eval import safe_eval, time
 from odoo.tools.misc import find_in_path
 from odoo.tools import config
 from odoo.sql_db import TestCursor
 from odoo.http import request
 from odoo.osv.expression import NEGATIVE_TERM_OPERATORS, FALSE_DOMAIN
 
-import time
 import base64
 import io
 import logging
diff --git a/odoo/addons/base/models/ir_filters.py b/odoo/addons/base/models/ir_filters.py
index 0c948d6c0f2..7a5d9c391ae 100644
--- a/odoo/addons/base/models/ir_filters.py
+++ b/odoo/addons/base/models/ir_filters.py
@@ -1,11 +1,9 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import datetime
-
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import UserError
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools.safe_eval import safe_eval, datetime
 
 
 class IrFilters(models.Model):
diff --git a/odoo/addons/base/models/ir_model.py b/odoo/addons/base/models/ir_model.py
index 0509f7eecfd..b9330a3339e 100644
--- a/odoo/addons/base/models/ir_model.py
+++ b/odoo/addons/base/models/ir_model.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-import datetime
 import itertools
 import logging
 import re
@@ -10,14 +9,13 @@ from collections import defaultdict
 from collections.abc import Mapping
 from operator import itemgetter
 
-import dateutil
 from psycopg2 import sql
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import AccessError, UserError, ValidationError
 from odoo.osv import expression
 from odoo.tools import pycompat, unique
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools.safe_eval import safe_eval, datetime, dateutil
 
 _logger = logging.getLogger(__name__)
 
diff --git a/odoo/addons/base/models/ir_rule.py b/odoo/addons/base/models/ir_rule.py
index 025f65437d6..7bbf03058ff 100644
--- a/odoo/addons/base/models/ir_rule.py
+++ b/odoo/addons/base/models/ir_rule.py
@@ -1,14 +1,13 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import logging
-import time
 import warnings
 
 from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.exceptions import AccessError, ValidationError
 from odoo.osv import expression
 from odoo.tools import config
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools.safe_eval import safe_eval, time
 
 _logger = logging.getLogger(__name__)
 class IrRule(models.Model):
diff --git a/odoo/addons/base/models/ir_ui_view.py b/odoo/addons/base/models/ir_ui_view.py
index ca2d69699cc..9f89b7af687 100644
--- a/odoo/addons/base/models/ir_ui_view.py
+++ b/odoo/addons/base/models/ir_ui_view.py
@@ -27,7 +27,7 @@ from odoo.modules.module import get_resource_from_path, get_resource_path
 from odoo.tools import config, ConstantMapping, get_diff, pycompat, apply_inheritance_specs, locate_node
 from odoo.tools.convert import _fix_multiple_roots
 from odoo.tools.json import scriptsafe as json_scriptsafe
-from odoo.tools.safe_eval import safe_eval
+from odoo.tools import safe_eval
 from odoo.tools.view_validation import valid_view, get_variable_names, get_domain_identifiers, get_dict_asts
 from odoo.tools.translate import xml_translate, TRANSLATED_ATTRS
 from odoo.tools.image import image_data_uri
@@ -74,7 +74,7 @@ def transfer_node_to_modifiers(node, modifiers, context=None, current_node_path=
 
     for a in ('invisible', 'readonly', 'required'):
         if node.get(a):
-            v = bool(safe_eval(node.get(a), {'context': context or {}}))
+            v = bool(safe_eval.safe_eval(node.get(a), {'context': context or {}}))
             node_path = current_node_path or ()
             if 'tree' in node_path and 'header' not in node_path and a == 'invisible':
                 # Invisible in a tree view has a specific meaning, make it a
@@ -1091,7 +1091,7 @@ actual arch.
         for attribute in ('invisible', 'readonly', 'required'):
             val = node.get(attribute)
             if val:
-                res = safe_eval(val, {'context': self._context})
+                res = safe_eval.safe_eval(val, {'context': self._context})
                 if res not in (1, 0, True, False, None):
                     msg = _(
                         'Attribute %(attribute)s evaluation expects a boolean, got %(value)s',
@@ -1716,8 +1716,8 @@ actual arch.
             test_mode_enabled=bool(config['test_enable'] or config['test_file']),
             json=json_scriptsafe,
             quote_plus=werkzeug.urls.url_quote_plus,
-            time=time,
-            datetime=datetime,
+            time=safe_eval.time,
+            datetime=safe_eval.datetime,
             relativedelta=relativedelta,
             xmlid=self.sudo().key,
             viewid=self.id,
diff --git a/odoo/addons/base/models/qweb.py b/odoo/addons/base/models/qweb.py
index 335808bb07d..34ce76277d3 100644
--- a/odoo/addons/base/models/qweb.py
+++ b/odoo/addons/base/models/qweb.py
@@ -18,6 +18,7 @@ import werkzeug
 from werkzeug.utils import escape as _escape
 
 from odoo.tools import pycompat, freehash
+from odoo.tools.safe_eval import check_values
 
 import builtins
 builtin_defaults = {name: getattr(builtins, name) for name in dir(builtins)}
@@ -325,6 +326,7 @@ class QWeb(object):
             log = {'last_path_node': None}
             new = self.default_values()
             new.update(values)
+            check_values(new)
             try:
                 return compiled(self, append, new, options, log)
             except (QWebException, TransactionRollbackError) as e:
diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py
index 94a954bbf73..0f01ec55cc0 100644
--- a/odoo/tools/convert.py
+++ b/odoo/tools/convert.py
@@ -1,13 +1,17 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+__all__ = [
+    'convert_file', 'convert_sql_import',
+    'convert_csv_import', 'convert_xml_import'
+]
+
 import base64
 import io
 import logging
 import os.path
 import re
 import subprocess
-import sys
-import time
 import warnings
 
 from datetime import datetime, timedelta
@@ -29,7 +33,7 @@ from odoo import SUPERUSER_ID, api
 
 _logger = logging.getLogger(__name__)
 
-from .safe_eval import safe_eval as s_eval
+from .safe_eval import safe_eval as s_eval, pytz, time
 safe_eval = lambda expr, ctx={}: s_eval(expr, ctx, nocopy=True)
 
 class ParseError(Exception):
diff --git a/odoo/tools/json.py b/odoo/tools/json.py
new file mode 100644
index 00000000000..ba9a5aa04f7
--- /dev/null
+++ b/odoo/tools/json.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
+import json as json_
+import re
+
+JSON_SCRIPTSAFE_MAPPER = {
+    '&': r'\u0026',
+    '<': r'\u003c',
+    '>': r'\u003e',
+    '\u2028': r'\u2028',
+    '\u2029': r'\u2029'
+}
+class JSON:
+    def loads(self, *args, **kwargs):
+        return json_.loads(*args, **kwargs)
+    def dumps(self, *args, **kwargs):
+        """ JSON used as JS in HTML (script tags) is problematic: <script>
+        tags are a special context which only waits for </script> but doesn't
+        interpret anything else, this means standard htmlescaping does not
+        work (it breaks double quotes, and e.g. `<` will become `&lt;` *in
+        the resulting JSON/JS* not just inside the page).
+
+        However, failing to escape embedded json means the json strings could
+        contains `</script>` and thus become XSS vector.
+
+        The solution turns out to be very simple: use JSON-level unicode
+        escapes for HTML-unsafe characters (e.g. "<" -> "\u003C". This removes
+        the XSS issue without breaking the json, and there is no difference to
+        the end result once it's been parsed back from JSON. So it will work
+        properly even for HTML attributes or raw text.
+
+        Also handle U+2028 and U+2029 the same way just in case as these are
+        interpreted as newlines in javascript but not in JSON, which could
+        lead to oddities and issues.
+
+        .. warning::
+
+            except inside <script> elements, this should be escaped following
+            the normal rules of the containing format
+
+        Cf https://code.djangoproject.com/ticket/17419#comment:27
+        """
+        # replacement can be done straight in the serialised JSON as the
+        # problematic characters are not JSON metacharacters (and can thus
+        # only occur in strings)
+        return re.sub(
+            r'[<>&\u2028\u2029]',
+            lambda m: JSON_SCRIPTSAFE_MAPPER[m[0]],
+            json_.dumps(*args, **kwargs),
+        )
+scriptsafe = JSON()
diff --git a/odoo/tools/json/__init__.py b/odoo/tools/json/__init__.py
deleted file mode 100644
index 40a96afc6ff..00000000000
--- a/odoo/tools/json/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# -*- coding: utf-8 -*-
diff --git a/odoo/tools/json/scriptsafe.py b/odoo/tools/json/scriptsafe.py
deleted file mode 100644
index f1147b02883..00000000000
--- a/odoo/tools/json/scriptsafe.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-import json as json_
-import re
-from json import *
-
-__all__ = ['dumps', 'loads']
-
-JSON_SCRIPTSAFE_MAPPER = {
-    '&': r'\u0026',
-    '<': r'\u003c',
-    '>': r'\u003e',
-    '\u2028': r'\u2028',
-    '\u2029': r'\u2029'
-}
-
-def dumps(*args, **kwargs):
-    """ JSON used as JS in HTML (script tags) is problematic: <script>
-    tags are a special context which only waits for </script> but doesn't
-    interpret anything else, this means standard htmlescaping does not
-    work (it breaks double quotes, and e.g. `<` will become `&lt;` *in
-    the resulting JSON/JS* not just inside the page).
-
-    However, failing to escape embedded json means the json strings could
-    contains `</script>` and thus become XSS vector.
-
-    The solution turns out to be very simple: use JSON-level unicode
-    escapes for HTML-unsafe characters (e.g. "<" -> "\u003C". This removes
-    the XSS issue without breaking the json, and there is no difference to
-    the end result once it's been parsed back from JSON. So it will work
-    properly even for HTML attributes or raw text.
-
-    Also handle U+2028 and U+2029 the same way just in case as these are
-    interpreted as newlines in javascript but not in JSON, which could
-    lead to oddities and issues.
-
-    .. warning::
-
-        except inside <script> elements, this should be escaped following
-        the normal rules of the containing format
-
-    Cf https://code.djangoproject.com/ticket/17419#comment:27
-    """
-    # replacement can be done straight in the serialised JSON as the
-    # problematic characters are not JSON metacharacters (and can thus
-    # only occur in strings)
-    return re.sub(
-        r'[<>&\u2028\u2029]',
-        lambda m: JSON_SCRIPTSAFE_MAPPER[m[0]],
-        json_.dumps(*args, **kwargs),
-    )
diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py
index 4ecdb6015ef..1017d3e0633 100644
--- a/odoo/tools/misc.py
+++ b/odoo/tools/misc.py
@@ -23,7 +23,7 @@ import traceback
 import types
 import unicodedata
 import zipfile
-from collections import defaultdict, OrderedDict
+from collections import OrderedDict
 from collections.abc import Iterable, Mapping, MutableMapping, MutableSet
 from contextlib import contextmanager
 from difflib import HtmlDiff
@@ -1500,26 +1500,6 @@ pickle.loads = lambda text, encoding='ASCII': _pickle_load(io.BytesIO(text), enc
 pickle.dump = pickle_.dump
 pickle.dumps = pickle_.dumps
 
-def wrap_module(module, attr_list):
-    """Helper for wrapping a package/module to expose selected attributes
-
-       :param Module module: the actual package/module to wrap, as returned by ``import <module>``
-       :param iterable attr_list: a global list of attributes to expose, usually the top-level
-            attributes and their own main attributes. No support for hiding attributes in case
-            of name collision at different levels.
-    """
-    attr_list = set(attr_list)
-    class WrappedModule(object):
-        def __getattr__(self, attrib):
-            if attrib in attr_list:
-                target = getattr(module, attrib)
-                if isinstance(target, types.ModuleType):
-                    return wrap_module(target, attr_list)
-                return target
-            raise AttributeError(attrib)
-    # module and attr_list are in the closure
-    return WrappedModule()
-
 
 class DotDict(dict):
     """Helper for dot.notation access to dictionary attributes
diff --git a/odoo/tools/safe_eval.py b/odoo/tools/safe_eval.py
index c9043f17dc3..828b42063cc 100644
--- a/odoo/tools/safe_eval.py
+++ b/odoo/tools/safe_eval.py
@@ -15,13 +15,14 @@ condition/math builtins.
 #  - safe_eval in lp:~xrg/openobject-server/optimize-5.0
 #  - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad
 import dis
-from opcode import HAVE_ARGUMENT, opmap, opname
-
 import functools
-from psycopg2 import OperationalError
-from types import CodeType
 import logging
+import types
+from opcode import HAVE_ARGUMENT, opmap, opname
+from types import CodeType
+
 import werkzeug
+from psycopg2 import OperationalError
 
 from .misc import ustr
 
@@ -306,6 +307,9 @@ def safe_eval(expr, globals_dict=None, locals_dict=None, mode="eval", nocopy=Fal
         if locals_dict is not None:
             locals_dict = dict(locals_dict)
 
+    check_values(globals_dict)
+    check_values(locals_dict)
+
     if globals_dict is None:
         globals_dict = {}
 
@@ -350,3 +354,59 @@ def test_python_expr(expr, mode="eval"):
             msg = ustr(err)
         return msg
     return False
+
+
+def check_values(d):
+    if not d:
+        return d
+    for v in d.values():
+        if isinstance(v, types.ModuleType):
+            raise TypeError(f"""Module {v} can not be used in evaluation contexts
+
+Prefer providing only the items necessary for your intended use.
+
+If a "module" is necessary for backwards compatibility, use
+`odoo.tools.safe_eval.wrap_module` to generate a wrapper recursively
+whitelisting allowed attributes.
+
+Pre-wrapped modules are provided as attributes of `odoo.tools.safe_eval`.
+""")
+    return d
+
+class wrap_module:
+    def __init__(self, module, attributes):
+        """Helper for wrapping a package/module to expose selected attributes
+
+        :param module: the actual package/module to wrap, as returned by ``import <module>``
+        :param iterable attributes: attributes to expose / whitelist. If a dict,
+                                    the keys are the attributes and the values
+                                    are used as an ``attributes`` in case the
+                                    corresponding item is a submodule
+        """
+        # builtin modules don't have a __file__ at all
+        modfile = getattr(module, '__file__', '(built-in)')
+        self._repr = f"<wrapped {module.__name__!r} ({modfile})>"
+        for attrib in attributes:
+            target = getattr(module, attrib)
+            if isinstance(target, types.ModuleType):
+                target = wrap_module(target, attributes[attrib])
+            setattr(self, attrib, target)
+
+    def __repr__(self):
+        return self._repr
+
+# dateutil submodules are lazy so need to import them for them to "exist"
+import dateutil
+mods = ['parser', 'relativedelta', 'rrule', 'tz']
+for mod in mods:
+    __import__('dateutil.%s' % mod)
+datetime = wrap_module(__import__('datetime'), ['date', 'datetime', 'time', 'timedelta', 'timezone', 'tzinfo', 'MAXYEAR', 'MINYEAR'])
+dateutil = wrap_module(dateutil, {
+    mod: getattr(dateutil, mod).__all__
+    for mod in mods
+})
+json = wrap_module(__import__('json'), ['loads', 'dumps'])
+time = wrap_module(__import__('time'), ['time', 'strptime', 'strftime'])
+pytz = wrap_module(__import__('pytz'), [
+    'utc', 'UTC', 'timezone',
+])