[IMP] test_mail_full: add test for SMS mailing default domain and optout

Purpose is to test specific behavior of opt-out for SMS marketing. This is
currently not tested and working a bit strangely, as they are flagged as
blacklist. This will soon be updated in another task but at least current
behavior is logged somewhere.

We also update some test models in order to reflect potential strange and
non-default behavior, notably partner_id field used as contact field. It
will be used soon when cleaning some opt-out behavior.

Tests are added to check phone sanitized is correctly taken into account
in default domains when inheriting from the phone blacklist mixin.

Some counters notes are also updated next to some last updates.

Task ID-2431217
COM PR odoo/odoo#71140

9 files changed, 155 insertions, 11 deletions

diff --git a/addons/test_mail_full/models/__init__.py b/addons/test_mail_full/models/__init__.py
index 26ed2919633..0fe839f20a6 100644
--- a/addons/test_mail_full/models/__init__.py
+++ b/addons/test_mail_full/models/__init__.py
@@ -1,3 +1,4 @@
 # -*- coding: utf-8 -*-
 
 from . import test_mail_models
+from . import mailing_mailing
diff --git a/addons/test_mail_full/models/mailing_mailing.py b/addons/test_mail_full/models/mailing_mailing.py
new file mode 100644
index 00000000000..a97c41e9047
--- /dev/null
+++ b/addons/test_mail_full/models/mailing_mailing.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+import logging
+
+from odoo import models
+
+_logger = logging.getLogger(__name__)
+
+
+class Mailing(models.Model):
+    _inherit = 'mailing.mailing'
+
+    def _get_opt_out_list_sms(self):
+        """Returns a set of emails opted-out in target model"""
+        self.ensure_one()
+        if self.mailing_model_real in ('mail.test.sms.bl.optout',
+                                       'mail.test.sms.partner'):
+            res_ids = self._get_recipients()
+            opt_out_contacts = set(self.env[self.mailing_model_real].search([
+                ('id', 'in', res_ids),
+                ('opt_out', '=', True)
+            ]).ids)
+            _logger.info(
+                "Mass-mailing %s targets %s, optout: %s emails",
+                self, self.mailing_model_real, len(opt_out_contacts))
+            return opt_out_contacts
+        return super(Mailing, self)._get_opt_out_list_sms()
diff --git a/addons/test_mail_full/models/test_mail_models.py b/addons/test_mail_full/models/test_mail_models.py
index 1d05eab7123..9ee84a0b3a2 100644
--- a/addons/test_mail_full/models/test_mail_models.py
+++ b/addons/test_mail_full/models/test_mail_models.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-from odoo import api, fields, models
+from odoo import fields, models
 
 
 class MailTestSMS(models.Model):
@@ -27,9 +27,9 @@ class MailTestSMS(models.Model):
 
 
 class MailTestSMSBL(models.Model):
-    """ A model inheriting from mail.thread with some fields used for SMS
-    gateway, like a partner, a specific mobile phone, ... """
-    _description = 'Chatter Model for SMS Gateway'
+    """ A model inheriting from mail.thread.phone allowing to test auto formatting
+    of phone numbers, blacklist, ... """
+    _description = 'SMS Mailing Blacklist Enabled'
     _name = 'mail.test.sms.bl'
     _inherit = ['mail.thread.phone']
     _order = 'name asc, id asc'
@@ -49,7 +49,31 @@ class MailTestSMSBL(models.Model):
         return ['phone_nbr', 'mobile_nbr']
 
 
-class MailTestSMSSoLike(models.Model):
+class MailTestSMSOptout(models.Model):
+    """ Model using blacklist mechanism and a hijacked opt-out mechanism for
+    mass mailing features. """
+    _description = 'SMS Mailing Blacklist / Optout Enabled'
+    _name = 'mail.test.sms.bl.optout'
+    _inherit = ['mail.thread.phone']
+    _order = 'name asc, id asc'
+
+    name = fields.Char()
+    subject = fields.Char()
+    email_from = fields.Char()
+    phone_nbr = fields.Char()
+    mobile_nbr = fields.Char()
+    customer_id = fields.Many2one('res.partner', 'Customer')
+    opt_out = fields.Boolean()
+
+    def _sms_get_partner_fields(self):
+        return ['customer_id']
+
+    def _sms_get_number_fields(self):
+        # TDE note: should override _phone_get_number_fields but ok as sms in dependencies
+        return ['phone_nbr', 'mobile_nbr']
+
+
+class MailTestSMSPartner(models.Model):
     """ A model like sale order having only a customer, not specific phone
     or mobile fields. """
     _description = 'Chatter Model for SMS Gateway (Partner only)'
@@ -57,4 +81,11 @@ class MailTestSMSSoLike(models.Model):
     _inherit = ['mail.thread']
 
     name = fields.Char()
-    partner_id = fields.Many2one('res.partner', 'Customer')
+    customer_id = fields.Many2one('res.partner', 'Customer')
+    opt_out = fields.Boolean()
+
+    def _sms_get_partner_fields(self):
+        return ['customer_id']
+
+    def _sms_get_number_fields(self):
+        return []
diff --git a/addons/test_mail_full/security/ir.model.access.csv b/addons/test_mail_full/security/ir.model.access.csv
index 408a47fdb96..b65c78635ce 100644
--- a/addons/test_mail_full/security/ir.model.access.csv
+++ b/addons/test_mail_full/security/ir.model.access.csv
@@ -3,5 +3,7 @@ access_mail_test_sms_all,mail.test.sms.all,model_mail_test_sms,,0,0,0,0
 access_mail_test_sms_user,mail.test.sms.user,model_mail_test_sms,base.group_user,1,1,1,1
 access_mail_test_sms_bl_all,mail.test.sms.bl.all,model_mail_test_sms_bl,,0,0,0,0
 access_mail_test_sms_bl_user,mail.test.sms.bl.user,model_mail_test_sms_bl,base.group_user,1,1,1,1
+access_mail_test_sms_bl_optout_all,mail.test.sms.bl.optout.all,model_mail_test_sms_bl_optout,,0,0,0,0
+access_mail_test_sms_bl_optout_user,mail.test.sms.bl.optout.user,model_mail_test_sms_bl_optout,base.group_user,1,1,1,1
 access_mail_test_sms_partner_all,mail.test.sms.partner.all,model_mail_test_sms_partner,,0,0,0,0
 access_mail_test_sms_partner_user,mail.test.sms.partner.user,model_mail_test_sms_partner,base.group_user,1,1,1,1
diff --git a/addons/test_mail_full/tests/common.py b/addons/test_mail_full/tests/common.py
index 579ff6cb7ca..e41d48806d1 100644
--- a/addons/test_mail_full/tests/common.py
+++ b/addons/test_mail_full/tests/common.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo.addons.phone_validation.tools import phone_validation
 from odoo.addons.mass_mailing_sms.tests.common import MassSMSCommon
@@ -12,6 +13,37 @@ class TestMailFullCommon(TestMassMailCommon, MassSMSCommon):
     def setUpClass(cls):
         super(TestMailFullCommon, cls).setUpClass()
 
+        cls.mailing_sms = cls.env['mailing.mailing'].with_user(cls.user_marketing).create({
+            'name': 'XMas SMS',
+            'subject': 'Xmas SMS for {object.name}',
+            'mailing_model_id': cls.env['ir.model']._get('mail.test.sms').id,
+            'mailing_type': 'sms',
+            'mailing_domain': '%s' % repr([('name', 'ilike', 'SMSTest')]),
+            'body_plaintext': 'Dear ${object.display_name} this is a mass SMS with two links http://www.odoo.com/smstest and http://www.odoo.com/smstest/${object.id}',
+            'sms_force_send': True,
+            'sms_allow_unsubscribe': True,
+        })
+
+    @classmethod
+    def _create_mailing_sms_test_records(cls, model='mail.test.sms', partners=None, count=1):
+        """ Helper to create data. Currently simple, to be improved. """
+        Model = cls.env[model]
+        phone_field = 'phone_nbr' if 'phone_nbr' in Model else 'phone'
+        partner_field = 'customer_id' if 'customer_id' in Model else 'partner_id'
+
+        vals_list = []
+        for idx in range(count):
+            vals = {
+                'name': 'SMSTestRecord_%02d' % idx,
+                phone_field: '045600%02d%02d' % (idx, idx)
+            }
+            if partners:
+                vals[partner_field] = partners[idx % len(partners)]
+
+            vals_list.append(vals)
+
+        return cls.env[model].create(vals_list)
+
 
 class TestRecipients(TestRecipients):
 
diff --git a/addons/test_mail_full/tests/test_mass_mailing.py b/addons/test_mail_full/tests/test_mass_mailing.py
index 135c0e0f751..a01804df49c 100644
--- a/addons/test_mail_full/tests/test_mass_mailing.py
+++ b/addons/test_mail_full/tests/test_mass_mailing.py
@@ -43,7 +43,7 @@ class TestMassMailing(TestMailFullCommon):
         mailing.write({'mailing_domain': [('id', 'in', recipients_all.ids)]})
         mailing.action_put_in_queue()
         with self.mock_mail_gateway(mail_unlink_sent=False):
-            mailing._process_mass_mailing_queue()
+            mailing.action_send_mail()
 
         for recipient in recipients_all:
             recipient_info = {
diff --git a/addons/test_mail_full/tests/test_mass_sms.py b/addons/test_mail_full/tests/test_mass_sms.py
index 33cb90e0028..9dc48232ab3 100644
--- a/addons/test_mail_full/tests/test_mass_sms.py
+++ b/addons/test_mail_full/tests/test_mass_sms.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+from ast import literal_eval
 
 from odoo.addons.phone_validation.tools import phone_validation
 from odoo.addons.test_mail_full.tests.common import TestMailFullCommon
@@ -63,6 +64,25 @@ class TestMassSMSCommon(TestMailFullCommon):
 @tagged('mass_mailing')
 class TestMassSMSInternals(TestMassSMSCommon):
 
+    @users('user_marketing')
+    def test_mass_sms_domain(self):
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'Xmas Spam',
+            'subject': 'Xmas Spam',
+            'mailing_model_id': self.env['ir.model']._get('mail.test.sms').id,
+            'mailing_type': 'sms',
+        })
+        self.assertEqual(literal_eval(mailing.mailing_domain), [])
+
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'Xmas Spam',
+            'subject': 'Xmas Spam',
+            'mailing_model_id': self.env['ir.model']._get('mail.test.sms.bl').id,
+            'mailing_type': 'sms',
+        })
+        self.assertEqual(literal_eval(mailing.mailing_domain), [('phone_sanitized_blacklisted', '=', False)])
+
+    @users('user_marketing')
     def test_mass_sms_internals(self):
         with self.with_user('user_marketing'):
             mailing = self.env['mailing.mailing'].create({
@@ -122,7 +142,7 @@ class TestMassSMSInternals(TestMassSMSCommon):
         })
         self.env['phone.blacklist'].create({'number': '0456110011'})
         # new customer, number already on record -> should be ignored
-        country_be_id = self.env.ref('base.be').id,
+        country_be_id = self.env.ref('base.be').id
         nr2_partner = self.env['res.partner'].create({
             'name': 'Partner_nr2',
             'country_id': country_be_id,
@@ -272,3 +292,33 @@ class TestMassSMS(TestMassSMSCommon):
                 ('unsubscribe', False, {}),
             ] for record in self.records],
         )
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mass_sms_w_opt_out(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_sms.ids)
+        recipients = self._create_mailing_sms_test_records(model='mail.test.sms.bl.optout', count=5)
+
+        # optout records 0 and 1
+        (recipients[0] | recipients[1]).write({'opt_out': True})
+        # blacklist records 4
+        # TDE FIXME: sudo should not be necessary
+        self.env['phone.blacklist'].sudo().create({'number': recipients[4].phone_nbr})
+
+        mailing.write({
+            'mailing_model_id': self.env['ir.model']._get('mail.test.sms.bl.optout'),
+            'mailing_domain': [('id', 'in', recipients.ids)],
+        })
+
+        with self.mockSMSGateway():
+            mailing.action_send_sms()
+
+        self.assertSMSTraces(
+            [{'number': '+32456000000', 'state': 'ignored', 'failure_type': 'sms_blacklist'},  # TDE FIXME: should be opt_out
+             {'number': '+32456000101', 'state': 'ignored', 'failure_type': 'sms_blacklist'},  # TDE FIXME: should be opt_out
+             {'number': '+32456000202', 'state': 'sent'},
+             {'number': '+32456000303', 'state': 'sent'},
+             {'number': '+32456000404', 'state': 'ignored', 'failure_type': 'sms_blacklist'}],
+            mailing, recipients
+        )
+        self.assertEqual(mailing.ignored, 3)
diff --git a/addons/test_mail_full/tests/test_sms_post.py b/addons/test_mail_full/tests/test_sms_post.py
index 549dd5e2cd0..6851974fd88 100644
--- a/addons/test_mail_full/tests/test_sms_post.py
+++ b/addons/test_mail_full/tests/test_sms_post.py
@@ -91,7 +91,7 @@ class TestSMSPost(TestMailFullCommon, TestRecipients):
 
     def test_message_sms_model_w_partner_only(self):
         with self.with_user('employee'):
-            record = self.env['mail.test.sms.partner'].create({'partner_id': self.partner_1.id})
+            record = self.env['mail.test.sms.partner'].create({'customer_id': self.partner_1.id})
 
             with self.mockSMSGateway():
                 messages = record._message_sms(self._test_body)
@@ -100,7 +100,7 @@ class TestSMSPost(TestMailFullCommon, TestRecipients):
 
     def test_message_sms_model_w_partner_only_void(self):
         with self.with_user('employee'):
-            record = self.env['mail.test.sms.partner'].create({'partner_id': False})
+            record = self.env['mail.test.sms.partner'].create({'customer_id': False})
 
             with self.mockSMSGateway():
                 messages = record._message_sms(self._test_body)
diff --git a/addons/test_mass_mailing/tests/test_performance.py b/addons/test_mass_mailing/tests/test_performance.py
index 9f9e26776d0..27fda6e9b1b 100644
--- a/addons/test_mass_mailing/tests/test_performance.py
+++ b/addons/test_mass_mailing/tests/test_performance.py
@@ -90,7 +90,7 @@ class TestMassMailBlPerformance(TestMassMailPerformanceBase):
             'mailing_domain': [('id', 'in', self.mm_recs.ids)],
         })
 
-        # runbot needs +70 compared to local
+        # runbot needs +63 compared to local
         with self.assertQueryCount(__system__=1992, marketing=1993):  # test_mass_mailing only: 1929 - 1930
             mailing.action_send_mail()