[IMP] mail: tests: improve / add common classes and data for mail tests

This commit improves common classes and add helpers for mail tests.
Future commits will use this new stuff to improve the tests.

 * improve existing models and add some of them. There are now several
   models

  * mail.test.simple: chatter-enabled record without any really advanced
    feature. Can be used everywhere standard chatter and communication
    tests are required;
  * mail.test.track: chatter-enabled record with basic value tracking
    and responsible tracking;
  * mail.test.activity: chatter-enabled record using activities;
  * mail.test.full: complete chatter-enabled record with advanced
    tracking, automatic subscription and automatic email send based on
    templates and tracking;
  * mail.test: chatter-enabled record having an alias. It can be used
    where umbrella record like projects or teams has to be used in
    tests;

 * add MockEmails test class: this class mocks the mail gateway so that
   tests can work on emails without actually sending them. It is taken
   out of TestMail class to clearly separate features from class;
 * improve created data in the various classes and prepare work to
   lessen created data during tests;
 * add helpers on base test class, notably

  * sudoAs: context manager to run a code snippet using a given user;
  * assertNotifications: assert notification content, aka Inbox or email
    notification and their content; may be subject to change depending on
    mail evolution;
  * assertEmails: check content of sent emails;

4 files changed, 286 insertions, 67 deletions

diff --git a/addons/mail/security/ir.model.access.csv b/addons/mail/security/ir.model.access.csv
index 0a2e9713c4f..c0ef5639d28 100644
--- a/addons/mail/security/ir.model.access.csv
+++ b/addons/mail/security/ir.model.access.csv
@@ -33,9 +33,16 @@ access_mail_template,mail.template,model_mail_template,base.group_user,1,1,1,0
 access_mail_template_system,mail.template_system,model_mail_template,base.group_system,1,1,1,1

 access_mail_shortcode,mail.shortcode,model_mail_shortcode,base.group_user,1,1,1,1

 access_mail_shortcode_portal,mail.shortcode.portal,model_mail_shortcode,base.group_portal,1,0,0,0

-access_mail_test_user,mail.test.all,model_mail_test,base.group_user,1,1,1,1

-access_mail_test_portal,mail.test.all,model_mail_test,base.group_portal,1,1,0,0

-access_mail_test_simple,mail.test.simple.all,model_mail_test_simple,base.group_user,1,1,1,1

+access_mail_test_simple_portal,mail.test.simple.portal,model_mail_test_simple,base.group_portal,1,0,0,0

+access_mail_test_simple_user,mail.test.simple.user,model_mail_test_simple,base.group_user,1,1,1,1

+access_mail_test_track_portal,mail.test.track.portal,model_mail_test_track,base.group_portal,0,0,0,0

+access_mail_test_track_user,mail.test.track.user.employee,model_mail_test_track,base.group_user,1,1,1,1

+access_mail_test_activity_portal,mail.test.activity.portal,model_mail_test_activity,base.group_portal,1,0,0,0

+access_mail_test_activity_user,mail.test.activity.user,model_mail_test_activity,base.group_user,1,1,1,1

+access_mail_test_full_portal,mail.test.full.portal,model_mail_test_full,base.group_portal,0,0,0,0

+access_mail_test_full_user,mail.test.full.user,model_mail_test_full,base.group_user,1,1,1,1

+access_mail_test_portal,mail.test.portal,model_mail_test,base.group_portal,1,1,0,0

+access_mail_test_user,mail.test.user,model_mail_test,base.group_user,1,1,1,1

 access_mail_activity_all,mail.activity.all,model_mail_activity,,0,0,0,0

 access_mail_activity_user,mail.activity.user,model_mail_activity,base.group_user,1,1,1,1

 access_mail_activity_type_all,mail.activity.type.all,model_mail_activity_type,,0,0,0,0

diff --git a/addons/mail/tests/common.py b/addons/mail/tests/common.py
index 3c03fc07401..893d687b8c7 100644
--- a/addons/mail/tests/common.py
+++ b/addons/mail/tests/common.py
@@ -1,75 +1,132 @@
 # -*- coding: utf-8 -*-
 
+from contextlib import contextmanager
+from email.utils import formataddr
+
 from odoo import api
 from odoo.tests import common
 
 
 class BaseFunctionalTest(common.SavepointCase):
+
     @classmethod
     def setUpClass(cls):
         super(BaseFunctionalTest, cls).setUpClass()
+        cls._quick_create_ctx = {
+            'mail_create_nolog': True,
+            'mail_create_nosubscribe': True,
+            'mail_notrack': True,
+        }
+        cls._quick_create_user_ctx = dict(cls._quick_create_ctx, no_reset_password=True)
 
-        # User groups
         user_group_employee = cls.env.ref('base.group_user')
-        user_group_portal = cls.env.ref('base.group_portal')
-        user_group_public = cls.env.ref('base.group_public')
-
-        # User Data: employee, noone
-        Users = cls.env['res.users'].with_context({'no_reset_password': True, 'mail_create_nosubscribe': True})
-        cls.user_employee = Users.create({
+        cls.user_employee = cls.env['res.users'].with_context(cls._quick_create_user_ctx).create({
             'name': 'Ernest Employee',
             'login': 'ernest',
             'email': 'e.e@example.com',
             'signature': '--\nErnest',
             'notification_type': 'email',
             'groups_id': [(6, 0, [user_group_employee.id])]})
-        cls.user_public = Users.create({
-            'name': 'Bert Tartignole',
-            'login': 'bert',
-            'email': 'b.t@example.com',
-            'signature': 'SignBert',
-            'notification_type': 'email',
-            'groups_id': [(6, 0, [user_group_public.id])]})
-        cls.user_portal = Users.create({
-            'name': 'Chell Gladys',
-            'login': 'chell',
-            'email': 'chell@gladys.portal',
-            'signature': 'SignChell',
-            'notification_type': 'email',
-            'groups_id': [(6, 0, [user_group_portal.id])]})
+        cls.partner_employee = cls.user_employee.partner_id
         cls.user_admin = cls.env.user
+        cls.partner_admin = cls.user_admin.partner_id
 
-        # Listener channel
-        cls.channel_listen = cls.env['mail.channel'].with_context({
-            'mail_create_nolog': True,
-            'mail_create_nosubscribe': True
-        }).create({'name': 'Listener'})
+        cls.channel_listen = cls.env['mail.channel'].with_context(cls._quick_create_ctx).create({'name': 'Listener'})
+
+        cls.test_record = cls.env['mail.test.simple'].with_context(cls._quick_create_ctx).create({'name': 'Test', 'email_from': 'ignasse@example.com'})
+
+    @contextmanager
+    def assertNotifications(self, **counters):
+        """ Counters: 'partner_attribute': 'inbox' or 'email' """
+        try:
+            init_messages = self.test_record.message_ids
+            init = {}
+            partners = self.env['res.partner']
+            for partner_attribute in counters.keys():
+                partner = getattr(self, partner_attribute)
+                partners |= partner
+                if partner.user_ids:
+                    init[partner] = {
+                        'na_counter': self.test_record.sudo(partner.user_ids[0]).message_needaction_counter,
+                    }
+            yield
+        finally:
+            new_messages = self.test_record.sudo().message_ids - init_messages
+            new_notifications = self.env['mail.notification'].search([
+                ('res_partner_id', 'in', partners.ids),
+                ('mail_message_id', 'in', new_messages.ids)
+            ])
+
+            for partner_attribute in counters.keys():
+                counter, notif_type, notif_read = counters[partner_attribute]
+                partner = getattr(self, partner_attribute)
+                partner_notif = new_notifications.filtered(lambda n: n.res_partner_id == partner)
+
+                self.assertEqual(len(partner_notif), counter)
+
+                if partner.user_ids:
+                    expected = init[partner]['na_counter'] + counter if notif_read == 'unread' else init[partner]['na_counter']
+                    self.assertEqual(expected, self.test_record.sudo(partner.user_ids[0]).message_needaction_counter,
+                                     'Invalid number of notification for %s: %s instead of %s' %
+                                     (partner.name, expected, self.test_record.sudo(partner.user_ids[0]).message_needaction_counter))
+                if partner_notif:
+                    self.assertTrue(all(n.is_email == (notif_type == 'email') for n in partner_notif))
+                    self.assertTrue(all(n.is_read == (notif_read == 'read') for n in partner_notif),
+                                    'Invalid read status for %s' % partner.name)
+
+            # for simplification, limitate to single message asserts
+            if hasattr(self, 'assertEmails') and len(new_messages) == 1:
+                self.assertEmails(new_messages.author_id, new_notifications.filtered(lambda n: n.is_email).mapped('res_partner_id'))
 
+    @contextmanager
+    def sudoAs(self, login):
+        old_uid = self.uid
+        try:
+            user = self.env['res.users'].sudo().search([('login', '=', login)])
+            self.test_record_old = self.test_record
+            # switch user
+            self.uid = user.id
+            self.env = self.env(user=self.uid)
+            self.test_record = self.test_record.sudo(self.uid)
+            yield
+        finally:
+            # back
+            self.uid = old_uid
+            self.env = self.env(user=self.uid)
+            self.test_record = self.test_record_old
 
-class TestMail(BaseFunctionalTest):
+
+class TestRecipients(common.SavepointCase):
 
     @classmethod
-    def _init_mock_build_email(cls):
-        cls._mails_args = []
-        cls._mails = []
+    def setUpClass(cls):
+        super(TestRecipients, cls).setUpClass()
+        Partner = cls.env['res.partner'].with_context({
+            'mail_create_nolog': True,
+            'mail_create_nosubscribe': True,
+            'mail_notrack': True,
+            'no_reset_password': True,
+        })
+        cls.partner_1 = Partner.create({
+            'name': 'Valid Lelitre',
+            'email': 'valid.lelitre@agrolait.com'})
+        cls.partner_2 = Partner.create({
+            'name': 'Valid Poilvache',
+            'email': 'valid.other@gmail.com'})
 
-    def format_and_process(self, template, to='groups@example.com, other@gmail.com', subject='Frogs',
-                           extra='', email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>',
-                           cc='', msg_id='<1198923581.41972151344608186760.JavaMail@agrolait.com>',
-                           model=None, target_model='mail.test', target_field='name'):
-        self.assertFalse(self.env[target_model].search([(target_field, '=', subject)]))
-        mail = template.format(to=to, subject=subject, cc=cc, extra=extra, email_from=email_from, msg_id=msg_id)
-        self.env['mail.thread'].with_context(mail_channel_noautofollow=True).message_process(model, mail)
-        return self.env[target_model].search([(target_field, '=', subject)])
+
+class MockEmails(common.SingleTransactionCase):
 
     def setUp(self):
-        super(TestMail, self).setUp()
+        super(MockEmails, self).setUp()
         self._mails_args[:] = []
         self._mails[:] = []
 
     @classmethod
     def setUpClass(cls):
-        super(TestMail, cls).setUpClass()
+        super(MockEmails, cls).setUpClass()
+        cls._mails_args = []
+        cls._mails = []
 
         def build_email(self, *args, **kwargs):
             cls._mails_args.append(args)
@@ -83,18 +140,122 @@ class TestMail(BaseFunctionalTest):
         cls.env['ir.mail_server']._patch_method('build_email', build_email)
         cls.env['ir.mail_server']._patch_method('send_email', send_email)
 
-        # Test Data for Partners
-        cls.partner_1 = cls.env['res.partner'].create({
+    def assertEmails(self, partner_from, recipients, **values):
+        """ Tools method to ease the check of send emails """
+        expected_email_values = []
+        for partners in recipients:
+            expected = {
+                'email_from': formataddr((partner_from.name, partner_from.email)),
+                'email_to': [formataddr((partner.name, partner.email)) for partner in partners]
+            }
+            if 'reply_to' in values:
+                expected['reply_to'] = values['reply_to']
+            if 'subject' in values:
+                expected['subject'] = values['subject']
+            if 'attachments' in values:
+                expected['attachments'] = values['attachments']
+            if 'body' in values:
+                expected['body'] = values['body']
+            if 'body_content' in values:
+                expected['body_content'] = values['body_content']
+            if 'body_alt_content' in values:
+                expected['body_alternative_content'] = values['body_alt_content']
+            if 'references' in values:
+                expected['references'] = values['references']
+            if 'ref_content' in values:
+                expected['references_content'] = values['ref_content']
+            expected_email_values.append(expected)
+
+        self.assertEqual(len(self._mails), len(expected_email_values))
+        for expected in expected_email_values:
+            sent_mail = next((mail for mail in self._mails if set(mail['email_to']) == set(expected['email_to'])), False)
+            self.assertTrue(bool(sent_mail), 'Expected mail to %s not found' % expected['email_to'])
+            for val in ['email_from', 'reply_to', 'subject', 'body', 'references', 'attachments']:
+                if val in expected:
+                    self.assertEqual(expected[val], sent_mail[val], 'Value for %s: expected %s, received %s' % (val, expected[val], sent_mail[val]))
+            for val in ['body_content', 'body_alternative', 'references_content']:
+                if val in expected:
+                    self.assertIn(expected[val], sent_mail[val[:-8]], 'Value for %s: %s does not contain %s' % (val, sent_mail[val[:-8]], expected[val]))
+
+    def assertHtmlEqual(self, value, expected, message=None):
+        from lxml import html
+
+        tree = html.fragment_fromstring(value, parser=html.HTMLParser(encoding='utf-8'), create_parent='body')
+
+        # mass mailing: add base tag we have to remove
+        for base_node in tree.xpath('//base'):
+            base_node.getparent().remove(base_node)
+
+        # chatter: read more / read less TODO
+
+        # mass mailing: add base tag we have to remove
+        expected_node = html.fragment_fromstring(expected, create_parent='body')
+
+        if message:
+            self.assertEqual(tree, expected_node, message)
+        else:
+            self.assertEqual(tree, expected_node)
+
+    @classmethod
+    def tearDownClass(cls):
+        # Remove mocks
+        cls.env['ir.mail_server']._revert_method('build_email')
+        cls.env['ir.mail_server']._revert_method('send_email')
+        super(MockEmails, cls).tearDownClass()
+
+    def _init_mock_build_email(self):
+        self._mails_args[:] = []
+        self._mails[:] = []
+
+    def format(self, template, to='groups@example.com, other@gmail.com', subject='Frogs',
+               extra='', email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>',
+               cc='', msg_id='<1198923581.41972151344608186760.JavaMail@agrolait.com>'):
+        return template.format(to=to, subject=subject, cc=cc, extra=extra, email_from=email_from, msg_id=msg_id)
+
+    def format_and_process(self, template, to='groups@example.com, other@gmail.com', subject='Frogs',
+                           extra='', email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>',
+                           cc='', msg_id='<1198923581.41972151344608186760.JavaMail@agrolait.com>',
+                           model=None, target_model='mail.test', target_field='name'):
+        self.assertFalse(self.env[target_model].search([(target_field, '=', subject)]))
+        mail = self.format(template, to=to, subject=subject, cc=cc, extra=extra, email_from=email_from, msg_id=msg_id)
+        self.env['mail.thread'].with_context(mail_channel_noautofollow=True).message_process(model, mail)
+        return self.env[target_model].search([(target_field, '=', subject)])
+
+
+
+class TestMail(BaseFunctionalTest, MockEmails):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestMail, cls).setUpClass()
+
+        user_group_portal = cls.env.ref('base.group_portal')
+        user_group_public = cls.env.ref('base.group_public')
+
+        cls.partner_1 = cls.env['res.partner'].with_context(cls._quick_create_ctx).create({
             'name': 'Valid Lelitre',
             'email': 'valid.lelitre@agrolait.com'})
-        cls.partner_2 = cls.env['res.partner'].create({
+        cls.partner_2 = cls.env['res.partner'].with_context(cls._quick_create_ctx).create({
             'name': 'Valid Poilvache',
             'email': 'valid.other@gmail.com'})
 
-        TestModel = cls.env['mail.test'].with_context({
-            'mail_create_nolog': True,
-            'mail_create_nosubscribe': True,
-        })
+        Users = cls.env['res.users'].with_context(cls._quick_create_ctx)
+        cls.user_public = Users.create({
+            'name': 'Bert Tartignole',
+            'login': 'bert',
+            'email': 'b.t@example.com',
+            'signature': 'SignBert',
+            'notification_type': 'email',
+            'groups_id': [(6, 0, [user_group_public.id])]})
+        cls.user_portal = Users.create({
+            'name': 'Chell Gladys',
+            'login': 'chell',
+            'email': 'chell@gladys.portal',
+            'signature': 'SignChell',
+            'notification_type': 'email',
+            'groups_id': [(6, 0, [user_group_portal.id])]})
+
+        TestModel = cls.env['mail.test'].with_context(cls._quick_create_ctx)
         cls.test_pigs = TestModel.create({
             'name': 'Pigs',
             'description': 'Fans of Pigs, unite !',
@@ -111,12 +272,3 @@ class TestMail(BaseFunctionalTest):
         cls.env['mail.followers'].search([
             ('res_model', '=', 'mail.test'),
             ('res_id', 'in', (cls.test_public | cls.test_pigs).ids)]).unlink()
-
-        cls._init_mock_build_email()
-
-    @classmethod
-    def tearDownClass(cls):
-        # Remove mocks
-        cls.env['ir.mail_server']._revert_method('build_email')
-        cls.env['ir.mail_server']._revert_method('send_email')
-        super(TestMail, cls).tearDownClass()
diff --git a/addons/mail/tests/models/test_mail_models.py b/addons/mail/tests/models/test_mail_models.py
index 578c779aad5..a45602bf6d5 100644
--- a/addons/mail/tests/models/test_mail_models.py
+++ b/addons/mail/tests/models/test_mail_models.py
@@ -5,17 +5,77 @@ from odoo import fields, models
 
 
 class MailTestSimple(models.Model):
-    _description = 'Test Simple Chatter Record'
+    """ A very simple model only inheriting from mail.thread when only
+    communication history is necessary. """
+    _description = 'Simple Chatter Model'
     _name = 'mail.test.simple'
     _inherit = ['mail.thread']
 
     name = fields.Char()
     email_from = fields.Char()
-    description = fields.Text()
 
 
-class MailTest(models.Model):
-    _description = 'Test Mail Model'
+class MailTestStandard(models.Model):
+    """ This model can be used in tests when automatic subscription and simple
+    tracking is necessary. Most features are present in a simple way. """
+    _description = 'Standard Chatter Model'
+    _name = 'mail.test.track'
+    _inherit = ['mail.thread']
+
+    name = fields.Char()
+    email_from = fields.Char()
+    user_id = fields.Many2one('res.users', 'Responsible', track_visibility='onchange')
+    umbrella_id = fields.Many2one('mail.test', track_visibility='onchange')
+
+
+class MailTestActivity(models.Model):
+    """ This model can be used to test activities in addition to simple chatter
+    features. """
+    _description = 'Activity Model'
+    _name = 'mail.test.activity'
+    _inherit = ['mail.thread', 'mail.activity.mixin']
+
+    name = fields.Char()
+    email_from = fields.Char()
+
+
+class MailTestFull(models.Model):
+    """ This model can be used in tests when complex chatter features are
+    required like modeling tasks or tickets. """
+    _description = 'Full Chatter Model'
+    _name = 'mail.test.full'
+    _inherit = ['mail.thread']
+
+    name = fields.Char()
+    email_from = fields.Char(track_visibility='always')
+    count = fields.Integer(default=1)
+    datetime = fields.Datetime(default=fields.Datetime.now)
+    mail_template = fields.Many2one('mail.template', 'Template')
+    customer_id = fields.Many2one('res.partner', 'Customer', track_visibility='onchange')
+    user_id = fields.Many2one('res.users', 'Responsible', track_visibility='onchange')
+    umbrella_id = fields.Many2one('mail.test', track_visibility='onchange')
+
+    def _track_template(self, tracking):
+        res = super(MailTestFull, self)._track_template(tracking)
+        record = self[0]
+        changes, tracking_value_ids = tracking[record.id]
+        if 'customer_id' in changes and record.mail_template:
+            res['customer_id'] = (record.mail_template, {'composition_mode': 'mass_mail'})
+        elif 'datetime' in changes:
+            res['datetime'] = ('mail.track_template', {'composition_mode': 'mass_mail'})
+        return res
+
+    def _track_subtype(self, init_values):
+        self.ensure_one()
+        if 'umbrella_id' in init_values and self.umbrella_id:
+            return 'mail.track_subtype_umbrella'
+        return super(MailTestFull, self)._track_subtype(init_values)
+
+
+class MailTestAlias(models.Model):
+    """ This model can be used in tests when umbrella records like projects
+    or teams are required. """
+    _description = 'Alias Chatter Model'
     _name = 'mail.test'
     _mail_post_access = 'read'
     _inherit = ['mail.thread', 'mail.alias.mixin']
@@ -31,7 +91,7 @@ class MailTest(models.Model):
 
     def get_alias_values(self):
         self.ensure_one()
-        res = super(MailTest, self).get_alias_values()
+        res = super(MailTestAlias, self).get_alias_values()
         res['alias_force_thread_id'] = self.id
         res['alias_parent_thread_id'] = self.id
         return res
diff --git a/addons/mail/tests/test_mail_gateway.py b/addons/mail/tests/test_mail_gateway.py
index bc8160f4d9a..9435448db5b 100644
--- a/addons/mail/tests/test_mail_gateway.py
+++ b/addons/mail/tests/test_mail_gateway.py
@@ -495,8 +495,8 @@ class TestMailgateway(TestMail):
         self.assertTrue(msg_fw.res_id == channel.id)
 
         # tmp
-        from odoo.addons.mail.tests.models.test_mail_models import MailTest
-        MailTest._mail_flat_thread = False
+        from odoo.addons.mail.tests.models.test_mail_models import MailTestAlias
+        MailTestAlias._mail_flat_thread = False
 
         fw_msg_id = '<THIS.IS.A.FW.MESSAGE.2@bert.fr>'
         fw_message = MAIL_TEMPLATE.format(to='public@example.com',
@@ -515,4 +515,4 @@ class TestMailgateway(TestMail):
         self.assertTrue(msg_fw.res_id == self.test_public.id)
 
         # re-tmp
-        MailTest._mail_flat_thread = True
+        MailTestAlias._mail_flat_thread = True