[IMP] mail, (test)_mass_mailing, test_mail_full: improve email-based mailing tests

PURPOSE

Purpose is to have more tests when sending mail mailings, notably about
canceled or failed mails or sms as well as jinja and links rendering.

SPECIFICATIONS

In this commit we improve Email Marketing mailing tests. We notably

  * add tests for void and invalid email and numbers. It allows to check they
    correctly update their trace status;
  * add tests for unsubscribe and view links embedded in mass mailing emails;
  * add tests to simulate a click on links sent through mass mailing and
    ensure click statistics are effectively updated;
  * add tests to simulate bounce emails coming back to the mail gateway;

Default content of mailings used in test is updated to ensure jinja is
correctly rendered, including links and some corner cases. This will also
helps ensuring behavior is kept when converting to QWeb.

Various docstrings are added to helpers and custom asserts.

LINKS

Task ID-2508643
Followup of odoo/odoo#68874 (improve mail tests)
Prepares Task ID-27033 (support QWeb in templates)
Prepares Task ID-2377974 (clean trace and status management in mass mailing)
COM PR odoo/odoo#69461
ENT PR odoo/enterprise#17780

9 files changed, 381 insertions, 107 deletions

diff --git a/addons/mail/tests/common.py b/addons/mail/tests/common.py
index 4bb38d819c8..4a688535886 100644
--- a/addons/mail/tests/common.py
+++ b/addons/mail/tests/common.py
@@ -125,6 +125,10 @@ class MockEmail(common.BaseCase):
         return self.env[target_model].search([(target_field, '=', subject)])
 
     def gateway_reply_wrecord(self, template, record, use_in_reply_to=True):
+        """ Deprecated, remove in 14.4 """
+        return self.gateway_mail_reply_wrecord(template, record, use_in_reply_to=use_in_reply_to)
+
+    def gateway_mail_reply_wrecord(self, template, record, use_in_reply_to=True):
         """ Simulate a reply through the mail gateway. Usage: giving a record,
         find an email sent to him and use its message-ID to simulate a reply.
 
@@ -176,7 +180,11 @@ class MockEmail(common.BaseCase):
         """ Find a sent email with a given list of recipients. Email should match
         exactly the recipients.
 
-        :return email: a dictionary mapping values given to ``build_email``;
+        :param email-to: a list of emails that will be compared to email_to
+          of sent emails (also a list of emails);
+
+        :return email: an email which is a dictionary mapping values given to
+          ``build_email``;
         """
         for sent_email in self._mails:
             if set(sent_email['email_to']) == set([email_to]):
@@ -185,6 +193,20 @@ class MockEmail(common.BaseCase):
             raise AssertionError('sent mail not found for email_to %s' % (email_to))
         return sent_email
 
+    def _find_mail_mail_wid(self, mail_id):
+        """ Find a ``mail.mail`` record based on a given ID (used notably when having
+        mail ID in mailing traces).
+
+        :return mail: a ``mail.mail`` record generated during the mock and matching
+          given ID;
+        """
+        for mail in self._new_mails:
+            if mail.id == mail_id:
+                break
+        else:
+            raise AssertionError('mail.mail not found for ID %s' % (mail_id))
+        return mail
+
     def _find_mail_mail_wpartners(self, recipients, status, mail_message=None, author=None):
         """ Find a mail.mail record based on various parameters, notably a list
         of recipients (partners).
@@ -271,7 +293,7 @@ class MockEmail(common.BaseCase):
         :param status: mail.mail state used to filter mails. If ``sent`` this method
           also check that emails have been sent trough gateway;
         :param mail_message: see ``_find_mail_mail_wpartners``;
-        :param author: see ``_find_mail_mail_wpartners``;;
+        :param author: see ``_find_mail_mail_wpartners``;
         :param content: if given, check it is contained within mail html body;
         :param fields_values: if given, should be a dictionary of field names /
           values allowing to check ``mail.mail`` additional values (subject,
@@ -329,7 +351,33 @@ class MockEmail(common.BaseCase):
             for email_to in emails:
                 self.assertSentEmail(email_values['email_from'] if email_values and email_values.get('email_from') else author, [email_to], **(email_values or {}))
 
+    def assertMailMailWId(self, mail_id, status,
+                          content=None, fields_values=None):
+        """ Assert mail.mail records are created and maybe sent as emails. Allow
+        asserting their content. Records to check are the one generated when
+        using mock (mail.mail and outgoing emails). This method takes partners
+        as source of record fetch and assert.
+
+        :param mail_id: a ``mail.mail`` DB ID. See ``_find_mail_mail_wid``;
+        :param status: mail.mail state to check upon found mail;
+        :param content: if given, check it is contained within mail html body;
+        :param fields_values: if given, should be a dictionary of field names /
+          values allowing to check ``mail.mail`` additional values (subject,
+          reply_to, ...);
+        """
+        found_mail = self._find_mail_mail_wid(mail_id)
+        self.assertTrue(bool(found_mail))
+        if status:
+            self.assertEqual(found_mail.state, status)
+        if content:
+            self.assertIn(content, found_mail.body_html)
+        for fname, fvalue in (fields_values or {}).items():
+            self.assertEqual(
+                found_mail[fname], fvalue,
+                'Mail: expected %s for %s, got %s' % (fvalue, fname, found_mail[fname]))
+
     def assertNoMail(self, recipients, mail_message=None, author=None):
+        """ Check no mail.mail and email was generated during gateway mock. """
         try:
             self._find_mail_mail_wpartners(recipients, False, mail_message=mail_message, author=author)
         except AssertionError:
@@ -340,6 +388,7 @@ class MockEmail(common.BaseCase):
             self.assertNotSentEmail()
 
     def assertNotSentEmail(self):
+        """ Check no email was generated during gateway mock. """
         self.assertEqual(len(self._mails), 0)
 
     def assertSentEmail(self, author, recipients, **values):
diff --git a/addons/mass_mailing/models/mailing.py b/addons/mass_mailing/models/mailing.py
index 995ab5d6582..1fecd2a87a6 100644
--- a/addons/mass_mailing/models/mailing.py
+++ b/addons/mass_mailing/models/mailing.py
@@ -176,6 +176,8 @@ class MassMailing(models.Model):
             self[key] = False
         if not self.ids:
             return
+        # ensure traces are sent to db
+        self.flush()
         self.env.cr.execute("""
             SELECT
                 m.id as mailing_id,
diff --git a/addons/mass_mailing/tests/common.py b/addons/mass_mailing/tests/common.py
index b4a2fa1d5e3..1baf6c7d75d 100644
--- a/addons/mass_mailing/tests/common.py
+++ b/addons/mass_mailing/tests/common.py
@@ -1,32 +1,78 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import datetime
+import random
+import re
+import werkzeug
+
 from odoo.addons.link_tracker.tests.common import MockLinkTracker
 from odoo.addons.mail.tests.common import MailCase, MailCommon, mail_new_test_user
-
+from odoo import tools
 
 class MassMailCase(MailCase, MockLinkTracker):
 
+    # ------------------------------------------------------------
+    # ASSERTS
+    # ------------------------------------------------------------
+
+    def assertMailingStatistics(self, mailing, **kwargs):
+        """ Helper to assert mailing statistics fields. As we have many of them
+        it helps lessening test asserts. """
+        if not kwargs.get('expected'):
+            kwargs['expected'] = len(mailing.mailing_trace_ids)
+        if not kwargs.get('delivered'):
+            kwargs['delivered'] = len(mailing.mailing_trace_ids)
+        for fname in ['scheduled', 'expected', 'sent', 'delivered',
+                      'opened', 'replied', 'clicked',
+                      'ignored', 'failed', 'bounced']:
+            self.assertEqual(
+                mailing[fname], kwargs.get(fname, 0),
+                'Mailing %s statistics failed: got %s instead of %s' % (fname, mailing[fname], kwargs.get(fname, 0))
+            )
+
     def assertMailTraces(self, recipients_info, mailing, records,
-                         check_mail=True, author=None):
-        """ Check content of traces.
+                         check_mail=True, sent_unlink=False, author=None,
+                         mail_links_info=None):
+        """ Check content of traces. Traces are fetched based on a given mailing
+        and records. Their content is compared to recipients_info structure that
+        holds expected information. Links content may be checked, notably to
+        assert shortening or unsubscribe links. Mail.mail records may optionally
+        be checked.
 
         :param recipients_info: list[{
+            # TRACE
             'partner': res.partner record (may be empty),
             'email': email used when sending email (may be empty, computed based on partner),
-            'state': outgoing / sent / open / reply / error / cancel (sent by default),
+            'state': outgoing / sent / ignored / bounced / exception / opened (sent by default),
             'record: linked record,
-            'content': UDPATE ME
-            'failure_type': optional: UPDATE ME
+            # MAIL.MAIL
+            'content': optional content that should be present in mail.mail body_html;
+            'failure_type': optional failure reason;
             }, { ... }]
+
+        :param mailing: a mailing.mailing record from which traces have been
+          generated;
+        :param records: records given to mailing that generated traces. It is
+          used notably to find traces using their IDs;
+        :param check_mail: if True, also check mail.mail records that should be
+          linked to traces;
+        :param sent_unlink: it True, sent mail.mail are deleted and we check gateway
+          output result instead of actual mail.mail records;
+        :param mail_links_info: if given, should follow order of ``recipients_info``
+          and give details about links. See ``assertLinkShortenedHtml`` helper for
+          more details about content to give;
+        :param author: author of sent mail.mail;
         """
         # map trace state to email state
         state_mapping = {
             'sent': 'sent',
-            'replied': 'sent',  # replied imply something has been sent
+            'opened': 'sent',  # opened implies something has been sent
+            'replied': 'sent',  # replied implies something has been sent
             'ignored': 'cancel',
             'exception': 'exception',
-            'canceled': 'canceled',
+            'canceled': 'cancel',
+            'bounced': 'cancel',
         }
 
         traces = self.env['mailing.trace'].search([
@@ -39,11 +85,13 @@ class MassMailCase(MailCase, MockLinkTracker):
         self.assertEqual(set(s.res_id for s in traces), set(records.ids))
 
         # check each traces
-        for recipient_info in recipients_info:
+        if not mail_links_info:
+            mail_links_info = [None] * len(recipients_info)
+        for recipient_info, link_info, record in zip(recipients_info, mail_links_info, records):
             partner = recipient_info.get('partner', self.env['res.partner'])
             email = recipient_info.get('email')
             state = recipient_info.get('state', 'sent')
-            record = recipient_info.get('record')
+            record = record or recipient_info.get('record')
             content = recipient_info.get('content')
             if email is None and partner:
                 email = partner.email_normalized
@@ -55,6 +103,7 @@ class MassMailCase(MailCase, MockLinkTracker):
                 len(recipient_trace) == 1,
                 'MailTrace: email %s (recipient %s, state: %s, record: %s): found %s records (1 expected)' % (email, partner, state, record, len(recipient_trace))
             )
+            self.assertTrue(bool(recipient_trace.mail_mail_id_int))
 
             if check_mail:
                 if author is None:
@@ -64,11 +113,103 @@ class MassMailCase(MailCase, MockLinkTracker):
                 if 'failure_type' in recipient_info:
                     fields_values['failure_type'] = recipient_info['failure_type']
 
+                # specific for partner: email_formatted is used
                 if partner:
-                    self.assertMailMail(partner, state_mapping[state], author=author, content=content, fields_values=fields_values)
+                    if state == 'sent' and sent_unlink:
+                        self.assertSentEmail(author, [partner])
+                    else:
+                        self.assertMailMail(partner, state_mapping[state], author=author, content=content, fields_values=fields_values)
+                # specific if email is False -> could have troubles finding it if several falsy traces
+                elif not email and state in ('ignored', 'canceled', 'bounced'):
+                    self.assertMailMailWId(recipient_trace.mail_mail_id_int, state_mapping[state], content=content, fields_values=fields_values)
                 else:
                     self.assertMailMailWEmails([email], state_mapping[state], author=author, content=content, fields_values=fields_values)
 
+            if link_info:
+                trace_mail = self._find_mail_mail_wrecord(record)
+                for (anchor_id, url, is_shortened, add_link_params) in link_info:
+                    link_params = {'utm_medium': 'Email', 'utm_source': mailing.name}
+                    if add_link_params:
+                        link_params.update(**add_link_params)
+                    self.assertLinkShortenedHtml(
+                        trace_mail.body_html,
+                        (anchor_id, url, is_shortened),
+                        link_params=link_params,
+                    )
+
+    # ------------------------------------------------------------
+    # TOOLS
+    # ------------------------------------------------------------
+
+    def gateway_mail_bounce(self, mailing, record, bounce_base_values=None):
+        """ Generate a bounce at mailgateway level.
+
+        :param mailing: a ``mailing.mailing`` record on which we find a trace
+          to bounce;
+        :param record: record which should bounce;
+        :param bounce_base_values: optional values given to routing;
+        """
+        trace = mailing.mailing_trace_ids.filtered(lambda t: t.model == record._name and t.res_id == record.id)
+
+        parsed_bounce_values = {
+            'email_from': 'some.email@external.example.com',  # TDE check: email_from -> trace email ?
+            'to': 'bounce@test.example.com',  # TDE check: bounce alias ?
+            'message_id': tools.generate_tracking_message_id('MailTest'),
+            'bounced_partner': self.env['res.partner'].sudo(),
+            'bounced_message': self.env['mail.message'].sudo()
+        }
+        if bounce_base_values:
+            parsed_bounce_values.update(bounce_base_values)
+        parsed_bounce_values.update({
+            'bounced_email': trace.email,
+            'bounced_msg_id': [trace.message_id],
+        })
+        self.env['mail.thread']._routing_handle_bounce(False, parsed_bounce_values)
+
+    def gateway_mail_click(self, mailing, record, click_label):
+        """ Simulate a click on a sent email. """
+        trace = mailing.mailing_trace_ids.filtered(lambda t: t.model == record._name and t.res_id == record.id)
+        email = self._find_sent_mail_wemail(trace.email)
+        self.assertTrue(bool(email))
+        for (_url_href, link_url, _dummy, label) in re.findall(tools.HTML_TAG_URL_REGEX, email['body']):
+            if label == click_label and '/r/' in link_url:  # shortened link, like 'http://localhost:8069/r/LBG/m/53'
+                parsed_url = werkzeug.urls.url_parse(link_url)
+                path_items = parsed_url.path.split('/')
+                code, trace_id = path_items[2], int(path_items[4])
+                self.assertEqual(trace.id, trace_id)
+
+                self.env['link.tracker.click'].sudo().add_click(
+                    code,
+                    ip='100.200.300.%3f' % random.random(),
+                    country_code='BE',
+                    mailing_trace_id=trace.id
+                )
+                break
+        else:
+            raise AssertionError('url %s not found in mailing %s for record %s' % (click_label, mailing, record))
+
+    @classmethod
+    def _create_bounce_trace(cls, mailing, record, dt=None):
+        if 'email_normalized' in record:
+            trace_email = record.email_normalized
+        elif 'email_from' in record:
+            trace_email = record.email_from
+        else:
+            trace_email = record.email
+        if dt is None:
+            dt = datetime.datetime.now() - datetime.timedelta(days=1)
+        randomized = random.random()
+        trace = cls.env['mailing.trace'].create({
+            'mass_mailing_id': mailing.id,
+            'model': record._name,
+            'res_id': record.id,
+            'bounced': dt,
+            # TDE FIXME: improve this with a mail-enabled heuristics
+            'email': trace_email,
+            'message_id': '<%5f@gilbert.boitempomils>' % randomized,
+        })
+        return trace
+
 
 class MassMailCommon(MailCommon, MassMailCase):
 
diff --git a/addons/mass_mailing/tests/test_mailing_internals.py b/addons/mass_mailing/tests/test_mailing_internals.py
index fbb61f3f0fd..df26c19ce35 100644
--- a/addons/mass_mailing/tests/test_mailing_internals.py
+++ b/addons/mass_mailing/tests/test_mailing_internals.py
@@ -202,7 +202,7 @@ class TestMassMailFeatures(MassMailCommon):
         self.assertMailTraces(
             [{'partner': partner_a},
              {'partner': partner_b, 'state': 'ignored'}],
-            mailing, partner_a | partner_b, check_mail=True
+            mailing, partner_a + partner_b, check_mail=True
         )
 
     @users('user_marketing')
diff --git a/addons/test_mail_full/tests/test_mass_mailing.py b/addons/test_mail_full/tests/test_mass_mailing.py
index 693022572fa..ff0f43b7f90 100644
--- a/addons/test_mail_full/tests/test_mass_mailing.py
+++ b/addons/test_mail_full/tests/test_mass_mailing.py
@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import werkzeug
+
 from odoo.addons.test_mail_full.tests.common import TestMailFullCommon
 from odoo.tests.common import users
-from odoo.tools import config, mute_logger
+from odoo.tools import mute_logger
 from odoo.tests import tagged
 
 
@@ -17,49 +19,55 @@ class TestMassMailing(TestMailFullCommon):
     @users('user_marketing')
     @mute_logger('odoo.addons.mail.models.mail_mail')
     def test_mailing_w_blacklist_opt_out(self):
-        # TDE FIXME: better URLs check for unsubscribe / view (res_id + email + correct parse of url)
-        mailing = self.mailing_bl.with_user(self.env.user)
-
-        jinja_html = '''
-<div>
-    <p>Hello <span class="text-muted">${object.name}</span></p>
-    <p>
-        Here are your personal links
-        <a href="http://www.example.com">External link</a>
-        <a href="/event/dummy-event-0">Internal link</a>
-        <a role="button" href="/unsubscribe_from_list" class="btn btn-link">Unsubscribe link</a>
-        <a href="/view">
-            View link
-        </a>
-    </p>
-</div>'''
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
 
-        mailing.write({
-            'preview': 'Hi ${object.name} :)',
-            'body_html': jinja_html,
-            'mailing_model_id': self.env['ir.model']._get('mailing.test.optout').id,
-        })
-        recipients = self._create_test_blacklist_records(model='mailing.test.optout', count=10)
+        mailing.write({'mailing_model_id': self.env['ir.model']._get('mailing.test.optout').id})
+        recipients = self._create_mailing_test_records(model='mailing.test.optout', count=10)
 
         # optout records 1 and 2
         (recipients[1] | recipients[2]).write({'opt_out': True})
         # blacklist records 3 and 4
         self.env['mail.blacklist'].create({'email': recipients[3].email_normalized})
         self.env['mail.blacklist'].create({'email': recipients[4].email_normalized})
+        # have a duplicate email for 9
+        recipient_dup_1 = recipients[9].copy()
+        # have a void mail
+        recipient_void_1 = self.env['mailing.test.optout'].create({'name': 'TestRecord_void_1'})
+        # have a falsy mail
+        recipient_falsy_1 = self.env['mailing.test.optout'].create({
+            'name': 'TestRecord_falsy_1',
+            'email_from': 'falsymail'
+        })
+        recipients_all = recipients + recipient_dup_1 + recipient_void_1 + recipient_falsy_1
 
-        mailing.write({'mailing_domain': [('id', 'in', recipients.ids)]})
+        mailing.write({'mailing_domain': [('id', 'in', recipients_all.ids)]})
         mailing.action_put_in_queue()
         with self.mock_mail_gateway(mail_unlink_sent=False):
             mailing._process_mass_mailing_queue()
 
-        for recipient in recipients:
+        for recipient in recipients_all:
             recipient_info = {
                 'email': recipient.email_normalized,
-                'content': 'Hello <span class="text-muted">%s</span' % recipient.name}
+                'content': 'Hello %s' % recipient.name}
+            # opt-out: ignored (cancel mail)
             if recipient in recipients[1] | recipients[2]:
                 recipient_info['state'] = 'ignored'
+            # blacklisted: ignored (cancel mail)
             elif recipient in recipients[3] | recipients[4]:
                 recipient_info['state'] = 'ignored'
+            # duplicates: ignored (cancel mail)
+            elif recipient == recipient_dup_1:
+                recipient_info['state'] = 'ignored'
+            # void: ignored (cancel mail)
+            elif recipient == recipient_void_1:
+                recipient_info['state'] = 'ignored'
+            # falsy: ignored (cancel mail)
+            elif recipient == recipient_falsy_1:
+                # TDE FIXME: currently setting False as email
+                # recipient_info['state'] = 'ignored'
+                # recipient_info['email'] = recipient.email_from  # normalized is False but email should be falsymail
+                recipient_info['state'] = 'ignored'
+                recipient_info['email'] = False
             else:
                 email = self._find_sent_mail_wemail(recipient.email_normalized)
                 # preview correctly integrated rendered jinja
@@ -68,13 +76,47 @@ class TestMassMailing(TestMailFullCommon):
                     email['body'])
                 # rendered unsubscribe
                 self.assertIn(
-                    'http://localhost:%s/mail/mailing/%s/unsubscribe' % (config['http_port'], mailing.id),
+                    '%s/mail/mailing/%s/unsubscribe' % (mailing.get_base_url(), mailing.id),
                     email['body'])
+                unsubscribe_href = self._get_href_from_anchor_id(email['body'], "url6")
+                unsubscribe_url = werkzeug.urls.url_parse(unsubscribe_href)
+                unsubscribe_params = unsubscribe_url.decode_query().to_dict(flat=True)
+                self.assertEqual(int(unsubscribe_params['res_id']), recipient.id)
+                self.assertEqual(unsubscribe_params['email'], recipient.email_normalized)
+                self.assertEqual(
+                    mailing._unsubscribe_token(unsubscribe_params['res_id'], (unsubscribe_params['email'])),
+                    unsubscribe_params['token']
+                )
                 # rendered view
                 self.assertIn(
-                    'http://localhost:%s/mailing/%s/view' % (config['http_port'], mailing.id),
+                    '%s/mailing/%s/view' % (mailing.get_base_url(), mailing.id),
                     email['body'])
+                view_href = self._get_href_from_anchor_id(email['body'], "url6")
+                view_url = werkzeug.urls.url_parse(view_href)
+                view_params = view_url.decode_query().to_dict(flat=True)
+                self.assertEqual(int(view_params['res_id']), recipient.id)
+                self.assertEqual(view_params['email'], recipient.email_normalized)
+                self.assertEqual(
+                    mailing._unsubscribe_token(view_params['res_id'], (view_params['email'])),
+                    view_params['token']
+                )
 
-            self.assertMailTraces([recipient_info], mailing, recipient, check_mail=True, author=self.env.user.partner_id)
+            self.assertMailTraces(
+                [recipient_info], mailing, recipient,
+                mail_links_info=[[
+                    ('url0', 'https://www.odoo.tz/my/%s' % recipient.name, True, {}),
+                    ('url1', 'https://www.odoo.be', True, {}),
+                    ('url2', 'https://www.odoo.com', True, {}),
+                    ('url3', 'https://www.odoo.eu', True, {}),
+                    ('url4', 'https://www.example.com/foo/bar?baz=qux', True, {'baz': 'qux'}),
+                    ('url5', '%s/event/dummy-event-0' % mailing.get_base_url(), True, {}),
+                    # view is not shortened and parsed at sending
+                    ('url6', '%s/view' % mailing.get_base_url(), False, {}),
+                    ('url7', 'mailto:test@odoo.com', False, {}),
+                    # unsubscribe is not shortened and parsed at sending
+                    ('url8', '%s/unsubscribe_from_list' % mailing.get_base_url(), False, {}),
+                ]],
+                check_mail=True,)
 
-        self.assertEqual(mailing.ignored, 4)
+        # sent: 13, 2 bl, 2 opt-out, 3 invalid -> 6 remaining
+        self.assertMailingStatistics(mailing, expected=6, delivered=6, sent=6, ignored=7)
diff --git a/addons/test_mass_mailing/tests/common.py b/addons/test_mass_mailing/tests/common.py
index 2a550878b12..43b1ad27ea1 100644
--- a/addons/test_mass_mailing/tests/common.py
+++ b/addons/test_mass_mailing/tests/common.py
@@ -1,9 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import datetime
-import random
-
 from odoo.addons.mass_mailing.tests.common import MassMailCommon
 from odoo.addons.test_mail.tests.common import TestMailCommon
 
@@ -19,40 +16,45 @@ class TestMassMailCommon(MassMailCommon, TestMailCommon):
         cls.mailing_bl = cls.env['mailing.mailing'].with_user(cls.user_marketing).create({
             'name': 'SourceName',
             'subject': 'MailingSubject',
-            'body_html': '<p>Hello ${object.name}</p>',
+            'preview': 'Hi ${object.name} :)',
+            'body_html': """<div><p>Hello ${object.name}</p>,
+% set url = "www.odoo.com"
+% set httpurl = "https://www.odoo.eu"
+<span>Website0: <a id="url0" href="https://www.odoo.tz/my/${object.name}">https://www.odoo.tz/my/${object.name}</a></span>
+<span>Website1: <a id="url1" href="https://www.odoo.be">https://www.odoo.be</a></span>
+<span>Website2: <a id="url2" href="https://${url}">https://${url}</a></span>
+<span>Website3: <a id="url3" href="${httpurl}">${httpurl}</a></span>
+<span>External1: <a id="url4" href="https://www.example.com/foo/bar?baz=qux">Youpie</a></span>
+<span>Internal1: <a id="url5" href="/event/dummy-event-0">Internal link</a></span>
+<span>Internal2: <a id="url6" href="/view"/>View link</a></span>
+<span>Email: <a id="url7" href="mailto:test@odoo.com">test@odoo.com</a></span>
+<p>Stop spam ? <a id="url8" role="button" href="/unsubscribe_from_list">Ok</a></p>
+</div>""",
             'mailing_type': 'mail',
             'mailing_model_id': cls.env['ir.model']._get('mailing.test.blacklist').id,
         })
 
     @classmethod
     def _create_test_blacklist_records(cls, model='mailing.test.blacklist', count=1):
+        """ Deprecated, remove in 14.4 """
+        return cls.__create_mailing_test_records(model=model, count=count)
+
+    @classmethod
+    def _create_mailing_test_records(cls, model='mailing.test.blacklist', partners=None, count=1):
         """ Helper to create data. Currently simple, to be improved. """
         Model = cls.env[model]
         email_field = 'email' if 'email' in Model else 'email_from'
+        partner_field = 'customer_id' if 'customer_id' in Model else 'partner_id'
 
-        records = cls.env[model].create([{
-            'name': 'TestRecord_%02d' % x,
-            email_field: '"TestCustomer %02d" <test.record.%02d@test.example.com>' % (x, x),
-        } for x in range(0, count)])
-        return records
+        vals_list = []
+        for x in range(0, count):
+            vals = {
+                'name': 'TestRecord_%02d' % x,
+                email_field: '"TestCustomer %02d" <test.record.%02d@test.example.com>' % (x, x),
+            }
+            if partners:
+                vals[partner_field] = partners[x % len(partners)]
 
-    @classmethod
-    def _create_bounce_trace(cls, record, dt=None):
-        if dt is None:
-            dt = datetime.datetime.now() - datetime.timedelta(days=1)
-        randomized = random.random()
-        if 'email_normalized' in record:
-            trace_email = record.email_normalized
-        elif 'email_from' in record:
-            trace_email = record.email_from
-        else:
-            trace_email = record.email
-        trace = cls.env['mailing.trace'].create({
-            'model': record._name,
-            'res_id': record.id,
-            'bounced': dt,
-            # TDE FIXME: improve this with a mail-enabled heuristics
-            'email': trace_email,
-            'message_id': '<%5f@gilbert.boitempomils>' % randomized,
-        })
-        return trace
+            vals_list.append(vals)
+
+        return cls.env[model].create(vals_list)
diff --git a/addons/test_mass_mailing/tests/test_blacklist_behavior.py b/addons/test_mass_mailing/tests/test_blacklist_behavior.py
index e07bfff760c..a031e9784c7 100644
--- a/addons/test_mass_mailing/tests/test_blacklist_behavior.py
+++ b/addons/test_mass_mailing/tests/test_blacklist_behavior.py
@@ -11,24 +11,17 @@ class TestAutoBlacklist(common.TestMassMailCommon):
 
     @users('user_marketing')
     def test_mailing_bounce_w_auto_bl(self):
-        mailing = self.mailing_bl.with_user(self.env.user)
-        base_parsed_values = {
-            'email_from': 'toto@yaourth.com', 'to': 'tata@yaourth.com', 'message_id': '<123.321@yaourth.com>',
-            'bounced_partner': self.env['res.partner'].sudo(), 'bounced_message': self.env['mail.message'].sudo()
-        }
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        target = self._create_mailing_test_records()[0]
+        mailing.write({'mailing_domain': [('id', 'in', target.ids)]})
 
-        target = self._create_test_blacklist_records()[0]
         # create bounced history of 4 statistics
         for idx in range(4):
-            trace = self._create_bounce_trace(target, dt=datetime.datetime.now() - datetime.timedelta(weeks=idx+2))
-            base_parsed_values.update({
-                'bounced_email': target.email_normalized,
-                'bounced_msg_id': [trace.message_id],
-            })
-            self.env['mail.thread']._routing_handle_bounce(False, base_parsed_values)
+            new_mailing = mailing.copy()
+            self._create_bounce_trace(new_mailing, target, dt=datetime.datetime.now() - datetime.timedelta(weeks=idx+2))
+            self.gateway_mail_bounce(new_mailing, target)
 
         # mass mail record: ok, not blacklisted yet
-        mailing.write({'mailing_domain': [('id', 'in', target.ids)]})
         mailing.action_put_in_queue()
         with self.mock_mail_gateway(mail_unlink_sent=False):
             mailing._process_mass_mailing_queue()
@@ -40,12 +33,7 @@ class TestAutoBlacklist(common.TestMassMailCommon):
         )
 
         # call bounced
-        trace = self._create_bounce_trace(target, dt=datetime.datetime.now())
-        base_parsed_values.update({
-            'bounced_email': target.email_normalized,
-            'bounced_msg_id': [trace.message_id],
-        })
-        self.env['mail.thread']._routing_handle_bounce(False, base_parsed_values)
+        self.gateway_mail_bounce(mailing, target)
 
         # check blacklist
         blacklist_record = self.env['mail.blacklist'].sudo().search([('email', '=', target.email_normalized)])
diff --git a/addons/test_mass_mailing/tests/test_mailing_internals.py b/addons/test_mass_mailing/tests/test_mailing_internals.py
index 5d284ef6d09..555578e492d 100644
--- a/addons/test_mass_mailing/tests/test_mailing_internals.py
+++ b/addons/test_mass_mailing/tests/test_mailing_internals.py
@@ -45,7 +45,7 @@ class TestMailingInternals(TestMassMailCommon):
         # Test if bad jinja in the subject raises an error
         mailing.write({'subject': 'Subject ${object.name_id.id}'})
         with self.mock_mail_gateway(), self.assertRaises(Exception):
-                mailing_test.send_mail_test()
+            mailing_test.send_mail_test()
 
         # Test if bad jinja in the body raises an error
         mailing.write({
@@ -53,15 +53,15 @@ class TestMailingInternals(TestMassMailCommon):
             'body_html': '<p>Hello ${object.name_id.id}</p>',
         })
         with self.mock_mail_gateway(), self.assertRaises(Exception):
-                mailing_test.send_mail_test()
-        
+            mailing_test.send_mail_test()
+
         # Test if bad jinja in the preview raises an error
         mailing.write({
             'body_html': '<p>Hello ${object.name}</p>',
             'preview': 'Preview ${object.name_id.id}',
         })
         with self.mock_mail_gateway(), self.assertRaises(Exception):
-                mailing_test.send_mail_test()
+            mailing_test.send_mail_test()
 
     def test_mailing_trace_update(self):
         customers = self.env['res.partner']
@@ -85,8 +85,8 @@ class TestMailingInternals(TestMassMailCommon):
         with self.mock_mail_gateway(mail_unlink_sent=False):
             mailing.action_send_mail()
 
-        self.gateway_reply_wrecord(MAIL_TEMPLATE, customers[0], use_in_reply_to=True)
-        self.gateway_reply_wrecord(MAIL_TEMPLATE, customers[1], use_in_reply_to=False)
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, customers[0], use_in_reply_to=True)
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, customers[1], use_in_reply_to=False)
 
         # customer2 looses headers
         mail_mail = self._find_mail_mail_wrecord(customers[2])
@@ -152,8 +152,8 @@ class TestMailingInternals(TestMassMailCommon):
         self.assertEqual(len(traces), 3)
 
         # simulate response to mailing
-        self.gateway_reply_wrecord(MAIL_TEMPLATE, self.mailing_list_1.contact_ids[0], use_in_reply_to=True)
-        self.gateway_reply_wrecord(MAIL_TEMPLATE, self.mailing_list_1.contact_ids[1], use_in_reply_to=False)
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, self.mailing_list_1.contact_ids[0], use_in_reply_to=True)
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, self.mailing_list_1.contact_ids[1], use_in_reply_to=False)
 
         mailing_test_utms = self.env['mailing.test.utm'].search([('name', '=', 'Re: %s' % subject)])
         self.assertEqual(len(mailing_test_utms), 2)
diff --git a/addons/test_mass_mailing/tests/test_mass_mailing.py b/addons/test_mass_mailing/tests/test_mass_mailing.py
index 478f9ce061d..0a2f45f0e17 100644
--- a/addons/test_mass_mailing/tests/test_mass_mailing.py
+++ b/addons/test_mass_mailing/tests/test_mass_mailing.py
@@ -2,10 +2,12 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo.addons.test_mass_mailing.tests.common import TestMassMailCommon
+from odoo.tests import tagged
 from odoo.tests.common import users
 from odoo.tools import mute_logger
 
 
+@tagged('mass_mailing')
 class TestMassMailing(TestMassMailCommon):
 
     @classmethod
@@ -14,9 +16,57 @@ class TestMassMailing(TestMassMailCommon):
 
     @users('user_marketing')
     @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_gateway_update(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(model='mailing.test.optout', count=5)
+        self.assertEqual(len(recipients), 5)
+
+        mailing.write({
+            'mailing_model_id': self.env['ir.model']._get('mailing.test.optout'),
+            'mailing_domain': [('id', 'in', recipients.ids)]
+        })
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        self.assertMailTraces(
+            [{'email': record.email_normalized}
+             for record in recipients],
+            mailing, recipients,
+            mail_links_info=[[
+                ('url0', 'https://www.odoo.tz/my/%s' % record.name, True, {}),
+                ('url1', 'https://www.odoo.be', True, {}),
+                ('url2', 'https://www.odoo.com', True, {}),
+                ('url3', 'https://www.odoo.eu', True, {}),
+                ('url4', 'https://www.example.com/foo/bar?baz=qux', True, {'baz': 'qux'}),
+                ('url5', '%s/event/dummy-event-0' % mailing.get_base_url(), True, {}),
+                # view is not shortened and parsed at sending
+                ('url6', '%s/view' % mailing.get_base_url(), False, {}),
+                ('url7', 'mailto:test@odoo.com', False, {}),
+                # unsubscribe is not shortened and parsed at sending
+                ('url8', '%s/unsubscribe_from_list' % mailing.get_base_url(), False, {}),
+            ] for record in recipients],
+            check_mail=True
+        )
+        self.assertMailingStatistics(mailing, expected=5, delivered=5, sent=5)
+
+        # simulate a click
+        self.gateway_mail_click(mailing, recipients[0], 'https://www.odoo.be')
+        mailing.invalidate_cache()
+        self.assertMailingStatistics(mailing, expected=5, delivered=5, sent=5, opened=1, clicked=1)
+
+        # simulate a bounce
+        self.assertEqual(recipients[1].message_bounce, 0)
+        self.gateway_mail_bounce(mailing, recipients[1])
+        mailing.invalidate_cache()
+        self.assertMailingStatistics(mailing, expected=5, delivered=4, sent=5, opened=1, clicked=1, bounced=1)
+        self.assertEqual(recipients[1].message_bounce, 1)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
     def test_mailing_w_blacklist(self):
-        mailing = self.mailing_bl.with_user(self.env.user)
-        recipients = self._create_test_blacklist_records(count=5)
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(count=5)
 
         # blacklist records 3 and 4
         self.env['mail.blacklist'].create({'email': recipients[3].email_normalized})
@@ -40,8 +90,8 @@ class TestMassMailing(TestMassMailCommon):
     @users('user_marketing')
     @mute_logger('odoo.addons.mail.models.mail_mail')
     def test_mailing_w_opt_out(self):
-        mailing = self.mailing_bl.with_user(self.env.user)
-        recipients = self._create_test_blacklist_records(model='mailing.test.optout', count=5)
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(model='mailing.test.optout', count=5)
 
         # optout records 0 and 1
         (recipients[0] | recipients[1]).write({'opt_out': True})
@@ -114,13 +164,13 @@ class TestMassMailing(TestMassMailCommon):
             mailing._process_mass_mailing_queue()
 
         self.assertMailTraces(
-            [{'email': 'test@test.example.com', 'state': 'ignored'},
-             {'email': 'test@test.example.com', 'state': 'sent'},
+            [{'email': 'test@test.example.com', 'state': 'sent'},
+             {'email': 'test@test.example.com', 'state': 'ignored'},
              {'email': 'test3@test.example.com'},
              {'email': 'test4@test.example.com'},
              {'email': 'test5@test.example.com', 'state': 'ignored'}],
             mailing,
-            mailing_contact_1 | mailing_contact_2 | mailing_contact_3 | mailing_contact_4 | mailing_contact_5,
+            mailing_contact_1 + mailing_contact_2 + mailing_contact_3 + mailing_contact_4 + mailing_contact_5,
             check_mail=True
         )
         self.assertEqual(mailing.ignored, 2)