[REF] mail: perform some code cleaning in mail gateway about parsing email values

PURPOSE

Add some improvements in mail gateway: remove private discussion, improve
bounce management, allow resetting bounce counters, improve automatic set or
reset of blacklists and ease mass mailing inheritance.

SPECIFICATIONS

Parsing email values: make message_parse effectively prepare all required
values for later processing

  * update message_parse to add some more data directly in parsed dictionary
    and avoid having to parse value again in later processing and computation.

    * notably recipients computation is now done directly in parsing. It
      includes email check and reconstruction using tools methods. That way
      routing just has to handle those values and not compute part of them;
    * from and cc are correctly managed with parsed and cleaned (sanitized)
      versions in dictionary;
    * remove all code decoding message header and replace them by fetching
      the parsed value instead;

  * filter parsed values before calling message_post to ensure right values
    are given to it instead of having to manage them in message_post. Indeed
    caller has to give the right input;
  * make parsing sub-methods (extract payload, post process payload) have
    a dictionary as input and output to always manage a dictionary of values
    through the parsing methods and update their namespace;

Routing: remove deprecated code parts

  * remove use of email_references regex. Indeed emails are routed using their
    messageId and not the regex anymore since several versions [1];
  * clean some unnecessary variables;
  * simplify variable type input, like message_parse accepts only valid
    email.message instances and to avoid several decode / encode. Tests are
    updated accordingly;

Update docstrings of those methods.

LINKS

Related to task 1893155
Linked to PR #33340

[1] See df037e295971650e8843b2ee668501e7aa3e7df6 and 0028c42136f02fcb7a9a81ed5763396290ffdcb8

6 files changed, 135 insertions, 145 deletions

diff --git a/addons/crm/tests/test_crm_lead.py b/addons/crm/tests/test_crm_lead.py
index 8bb95851860..737ae351671 100644
--- a/addons/crm/tests/test_crm_lead.py
+++ b/addons/crm/tests/test_crm_lead.py
@@ -52,14 +52,14 @@ class TestCRMLead(TestCrmCases):
         self.env['mail.thread'].with_user(self.crm_salesman).message_process('crm.lead', request_message)
 
         # After getting the mail, I check details of new lead of that customer
-        lead = self.env['crm.lead'].with_user(self.crm_salesman).search([('email_from', '=', 'Mr. John Right <info@customer.com>')], limit=1)
+        lead = self.env['crm.lead'].with_user(self.crm_salesman).search([('email_normalized', '=', 'info@customer.com')], limit=1)
         self.assertTrue(lead.ids, 'Fail to create merge opportunity wizard')
         self.assertFalse(lead.partner_id, 'Customer should be a new one')
         self.assertEqual(lead.name, 'Fournir votre devis avec le meilleur prix.', 'Subject does not match')
 
         # I reply his request with welcome message.
         # TODO revert mail.mail to mail.compose.message (conversion to customer should be automatic).
-        lead = self.env['crm.lead'].search([('email_from', '=', 'Mr. John Right <info@customer.com>')], limit=1)
+        lead = self.env['crm.lead'].search([('email_normalized', '=', 'info@customer.com')], limit=1)
         mail = self.env['mail.compose.message'].with_context(active_model='crm.lead', active_id=lead.id).create({
             'body': "Merci de votre intérêt pour notre produit, nous vous contacterons bientôt. Bien à vous",
             'email_from': 'sales@mycompany.com'
@@ -70,7 +70,7 @@ class TestCRMLead(TestCrmCases):
             pass
 
         # Now, I convert him into customer and put him into regular customer list
-        lead = self.env['crm.lead'].search([('email_from', '=', 'Mr. John Right <info@customer.com>')], limit=1)
+        lead = self.env['crm.lead'].search([('email_normalized', '=', 'info@customer.com')], limit=1)
         lead.handle_partner_assignation()
 
     def test_crm_lead_merge(self):
diff --git a/addons/hr_recruitment/tests/test_recruitment_process.py b/addons/hr_recruitment/tests/test_recruitment_process.py
index 1a20c9c6bf5..03f411bb371 100644
--- a/addons/hr_recruitment/tests/test_recruitment_process.py
+++ b/addons/hr_recruitment/tests/test_recruitment_process.py
@@ -27,7 +27,7 @@ class TestRecruitmentProcess(common.TransactionCase):
             'hr.applicant', request_message, custom_values={"job_id": self.env.ref('hr.job_developer').id})
 
         # After getting the mail, I check the details of the new applicant.
-        applicant = self.env['hr.applicant'].search([('email_from', '=', 'Mr. Richard Anderson <Richard_Anderson@yahoo.com>')], limit=1)
+        applicant = self.env['hr.applicant'].search([('email_from', 'ilike', 'Richard_Anderson@yahoo.com')], limit=1)
         self.assertTrue(applicant, "Applicant is not created after getting the mail")
         resume_ids = self.env['ir.attachment'].search([
             ('name', '=', 'resume.pdf'),
diff --git a/addons/l10n_it_edi/models/ir_mail_server.py b/addons/l10n_it_edi/models/ir_mail_server.py
index f36ef683dee..176b89f68d7 100644
--- a/addons/l10n_it_edi/models/ir_mail_server.py
+++ b/addons/l10n_it_edi/models/ir_mail_server.py
@@ -96,7 +96,8 @@ class FetchmailServer(models.Model):
         return super(FetchmailServer, self.filtered(lambda s: not s.l10n_it_is_pec)).fetch_mail()
 
     def _attachment_invoice(self, msg_txt):
-        body, attachments = self.env['mail.thread']._message_extract_payload(msg_txt)
+        parsed_values = self.env['mail.thread']._message_parse_extract_payload(msg_txt)
+        body, attachments = parsed_values['body'], parsed_values['attachments']
         from_address = tools.decode_smtp_header(msg_txt.get('from'))
         for attachment in attachments:
             split_attachment = attachment.fname.rpartition('.')
diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py
index 2b28b9044dd..7a8788f19bc 100644
--- a/addons/mail/models/mail_thread.py
+++ b/addons/mail/models/mail_thread.py
@@ -701,8 +701,8 @@ class MailThread(models.AbstractModel):
         assert isinstance(route, (list, tuple)), 'A route should be a list or a tuple'
         assert len(route) == 5, 'A route should contain 5 elements: model, thread_id, custom_values, uid, alias record'
 
-        message_id = message.get('Message-Id')
-        email_from = tools.decode_message_header(message, 'From')
+        message_id = message_dict['message_id']
+        email_from = message_dict['email_from']
         author_id = message_dict.get('author_id')
         model, thread_id, alias = route[0], route[1], route[4]
         record_set = None
@@ -711,7 +711,7 @@ class MailThread(models.AbstractModel):
 <p>Hello,</p>
 <p>The following email sent to %s cannot be accepted because this is a private email address.
    Only allowed people can contact us at this address.</p>
-</div><blockquote>%s</blockquote>""" % (message.get('to'), message_dict.get('body'))
+</div><blockquote>%s</blockquote>""" % (message_dict.get('to'), message_dict.get('body'))
 
         # Wrong model
         if not model:
@@ -773,15 +773,11 @@ class MailThread(models.AbstractModel):
          * if the message replies to an existing thread by having a Message-Id
            that matches an existing mail_message.message_id, we take the original
            message model/thread_id pair and ignore custom_value as no creation will
-           take place
-         * if the message replies to an existing thread by having In-Reply-To or
-           References matching odoo model/thread_id Message-Id and if this thread
-           has messages without message_id, take this model/thread_id pair and
-           ignore custom_value as no creation will take place (6.1 compatibility)
+           take place;
          * look for a mail.alias entry matching the message recipients and use the
            corresponding model, thread_id, custom_values and user_id. This could
-           lead to a thread update or creation depending on the alias
-         * fallback on provided ``model``, ``thread_id`` and ``custom_values``
+           lead to a thread update or creation depending on the alias;
+         * fallback on provided ``model``, ``thread_id`` and ``custom_values``;
          * raise an exception as no route has been found
 
         :param string message: an email.message instance
@@ -802,37 +798,29 @@ class MailThread(models.AbstractModel):
         """
         if not isinstance(message, Message):
             raise TypeError('message must be an email.message.Message at this point')
-        MailMessage = self.env['mail.message']
-        Alias, dest_aliases = self.env['mail.alias'], self.env['mail.alias']
         catchall_alias = self.env['ir.config_parameter'].sudo().get_param("mail.catchall.alias")
         bounce_alias = self.env['ir.config_parameter'].sudo().get_param("mail.bounce.alias")
         fallback_model = model
 
         # get email.message.Message variables for future processing
         local_hostname = socket.gethostname()
-        message_id = message.get('Message-Id')
+        message_id = message_dict['message_id']
 
         # compute references to find if message is a reply to an existing thread
-        references = tools.decode_message_header(message, 'References')
-        in_reply_to = tools.decode_message_header(message, 'In-Reply-To').strip()
-        thread_references = references or in_reply_to
-        reply_match, reply_model, reply_thread_id, reply_hostname, reply_private = tools.email_references(thread_references)
+        thread_references = message_dict['references'] or message_dict['in_reply_to']
+        msg_references = [ref for ref in tools.mail_header_msgid_re.findall(thread_references) if 'reply_to' not in ref]
+        mail_messages = self.env['mail.message'].sudo().search([('message_id', 'in', msg_references)], limit=1)
+        is_a_reply = bool(mail_messages)
+        reply_model, reply_thread_id = mail_messages.model, mail_messages.res_id
 
         # author and recipients
-        email_from = tools.decode_message_header(message, 'From')
+        email_from = message_dict['email_from']
         email_from_localpart = (tools.email_split(email_from) or [''])[0].split('@', 1)[0].lower()
-        email_to = tools.decode_message_header(message, 'To')
+        email_to = message_dict['to']
         email_to_localpart = (tools.email_split(email_to) or [''])[0].split('@', 1)[0].lower()
-
         # Delivered-To is a safe bet in most modern MTAs, but we have to fallback on To + Cc values
         # for all the odd MTAs out there, as there is no standard header for the envelope's `rcpt_to` value.
-        rcpt_tos = ','.join([
-            tools.decode_message_header(message, 'Delivered-To'),
-            tools.decode_message_header(message, 'To'),
-            tools.decode_message_header(message, 'Cc'),
-            tools.decode_message_header(message, 'Resent-To'),
-            tools.decode_message_header(message, 'Resent-Cc')])
-        rcpt_tos_localparts = [e.split('@')[0].lower() for e in tools.email_split(rcpt_tos)]
+        rcpt_tos_localparts = [e.split('@')[0].lower() for e in tools.email_split(message_dict['recipients'])]
 
         # 0. Verify whether this is a bounced email and use it to collect bounce data and update notifications for customers
         if bounce_alias and bounce_alias in email_to_localpart:
@@ -861,7 +849,7 @@ class MailThread(models.AbstractModel):
                 if email_part:
                     email = email_part.get_payload()[0]
                     bounced_message_id = tools.mail_header_msgid_re.findall(tools.decode_message_header(email, 'Message-Id'))
-                    mail_message = MailMessage.sudo().search([('message_id', 'in', bounced_message_id)])
+                    mail_message = self.env['mail.message'].sudo().search([('message_id', 'in', bounced_message_id)])
 
                 if partners and mail_message:
                     notifications = self.env['mail.notification'].sudo().search([
@@ -887,38 +875,33 @@ class MailThread(models.AbstractModel):
                          message_id, email_from, email_to)
             return []
 
-        # 1. Check if message is a reply on a thread
-        msg_references = [ref for ref in tools.mail_header_msgid_re.findall(thread_references) if 'reply_to' not in ref]
-        mail_messages = MailMessage.sudo().search([('message_id', 'in', msg_references)], limit=1)
-        is_a_reply = bool(mail_messages)
-
-        # 1.1 Handle forward to an alias with a different model: do not consider it as a reply
+        # 1. Handle reply
+        #    if destination = alias with different model -> consider it is a forward and not a reply
+        #    if destination = alias with same model -> check contact settings as they still apply
         if reply_model and reply_thread_id:
-            other_alias = Alias.search([
+            other_alias = self.env['mail.alias'].search([
                 '&',
                 ('alias_name', '!=', False),
                 ('alias_name', '=', email_to_localpart)
             ])
             if other_alias and other_alias.alias_model_id.model != reply_model:
                 is_a_reply = False
-
         if is_a_reply:
-            model, thread_id = mail_messages.model, mail_messages.res_id
-            dest_aliases = Alias.search([('alias_name', 'in', rcpt_tos_localparts)], limit=1)
+            dest_aliases = self.env['mail.alias'].search([('alias_name', 'in', rcpt_tos_localparts)], limit=1)
 
             route = self._routing_check_route(
                 message, message_dict,
-                (model, thread_id, custom_values, self._uid, dest_aliases),
+                (reply_model, reply_thread_id, custom_values, self._uid, dest_aliases),
                 raise_exception=False)
             if route:
                 _logger.info(
                     'Routing mail from %s to %s with Message-Id %s: direct reply to msg: model: %s, thread_id: %s, custom_values: %s, uid: %s',
-                    email_from, email_to, message_id, model, thread_id, custom_values, self._uid)
+                    email_from, email_to, message_id, reply_model, reply_thread_id, custom_values, self._uid)
                 return [route]
             elif route is False:
                 return []
 
-        # 2. Look for a matching mail.alias entry
+        # 2. Handle new incoming email by checking aliases and applying their settings
         if rcpt_tos_localparts:
             # no route found for a matching reference (or reply), so parent is invalid
             message_dict.pop('parent_id', None)
@@ -932,7 +915,7 @@ class MailThread(models.AbstractModel):
                 self._routing_create_bounce_email(email_from, body, message, reply_to=self.env.company.email)
                 return []
 
-            dest_aliases = Alias.search([('alias_name', 'in', rcpt_tos_localparts)])
+            dest_aliases = self.env['mail.alias'].search([('alias_name', 'in', rcpt_tos_localparts)])
             if dest_aliases:
                 routes = []
                 for alias in dest_aliases:
@@ -954,7 +937,7 @@ class MailThread(models.AbstractModel):
                         routes.append(route)
                 return routes
 
-        # 5. Fallback to the provided parameters, if they work
+        # 3. Fallback to the provided parameters, if they work
         if fallback_model:
             # no route found for a matching reference (or reply), so parent is invalid
             message_dict.pop('parent_id', None)
@@ -1019,6 +1002,9 @@ class MailThread(models.AbstractModel):
                     subtype_id = self.env['ir.model.data'].xmlid_to_res_id('mail.mt_comment')
 
             post_params = dict(subtype_id=subtype_id, partner_ids=partner_ids, **message_dict)
+            # remove computational values not stored on mail.message and avoid warnings when creating it
+            for x in ('from', 'to', 'cc', 'recipients', 'references', 'in_reply_to'):
+                post_params.pop(x, None)
             new_msg = False
             if thread._name == 'mail.thread':  # message with parent_id not linked to record
                 new_msg = thread.message_notify(**post_params)
@@ -1068,23 +1054,22 @@ class MailThread(models.AbstractModel):
             message = bytes(message.data)
         if isinstance(message, str):
             message = message.encode('utf-8')
-        msg_txt = email.message_from_bytes(message)
+        message = email.message_from_bytes(message)
 
         # parse the message, verify we are not in a loop by checking message_id is not duplicated
-        msg = self.message_parse(msg_txt, save_original=save_original)
+        msg_dict = self.message_parse(message, save_original=save_original)
         if strip_attachments:
-            msg.pop('attachments', None)
+            msg_dict.pop('attachments', None)
 
-        if msg.get('message_id'):   # should always be True as message_parse generate one if missing
-            existing_msg_ids = self.env['mail.message'].search([('message_id', '=', msg.get('message_id'))])
-            if existing_msg_ids:
-                _logger.info('Ignored mail from %s to %s with Message-Id %s: found duplicated Message-Id during processing',
-                                msg.get('from'), msg.get('to'), msg.get('message_id'))
-                return False
+        existing_msg_ids = self.env['mail.message'].search([('message_id', '=', msg_dict['message_id'])], limit=1)
+        if existing_msg_ids:
+            _logger.info('Ignored mail from %s to %s with Message-Id %s: found duplicated Message-Id during processing',
+                         msg_dict.get('email_from'), msg_dict.get('to'), msg_dict.get('message_id'))
+            return False
 
         # find possible routes for the message
-        routes = self.message_route(msg_txt, msg, model, thread_id, custom_values)
-        thread_id = self.message_route_process(msg_txt, msg, routes)
+        routes = self.message_route(message, msg_dict, model, thread_id, custom_values)
+        thread_id = self.message_route_process(message, msg_dict, routes)
         return thread_id
 
     @api.model
@@ -1149,14 +1134,15 @@ class MailThread(models.AbstractModel):
         """
         pass
 
-    def _message_extract_payload_postprocess(self, message, body, attachments):
+    def _message_parse_extract_payload_postprocess(self, message, payload_dict):
         """ Perform some cleaning / postprocess in the body and attachments
         extracted from the email. Note that this processing is specific to the
         mail module, and should not contain security or generic html cleaning.
         Indeed those aspects should be covered by the html_sanitize method
         located in tools. """
+        body, attachments = payload_dict['body'], payload_dict['attachments']
         if not body:
-            return body, attachments
+            return payload_dict
         try:
             root = lxml.html.fromstring(body)
         except ValueError:
@@ -1182,9 +1168,9 @@ class MailThread(models.AbstractModel):
             node.getparent().remove(node)
         if postprocessed:
             body = etree.tostring(root, pretty_print=False, encoding='unicode')
-        return body, attachments
+        return {'body': body, 'attachments': attachments}
 
-    def _message_extract_payload(self, message, save_original=False):
+    def _message_parse_extract_payload(self, message, save_original=False):
         """Extract body as HTML and attachments from the mail message"""
         attachments = []
         body = u''
@@ -1259,43 +1245,43 @@ class MailThread(models.AbstractModel):
                 else:
                     attachments.append(self._Attachment(filename or 'attachment', part.get_payload(decode=True), {}))
 
-        body, attachments = self._message_extract_payload_postprocess(message, body, attachments)
-        return body, attachments
+        return self._message_parse_extract_payload_postprocess(message, {'body': body, 'attachments': attachments})
 
     @api.model
     def message_parse(self, message, save_original=False):
-        """Parses a string or email.message.Message representing an
-           RFC-2822 email, and returns a generic dict holding the
-           message details.
-
-           :param message: the message to parse
-           :type message: email.message.Message | string | unicode
-           :param bool save_original: whether the returned dict
-               should include an ``original`` attachment containing
-               the source of the message
-           :rtype: dict
-           :return: A dict with the following structure, where each
-                    field may not be present if missing in original
-                    message::
-                    { 'message_id': msg_id,
-                      'subject': subject,
-                      'from': from,
-                      'to': to,
-                      'cc': cc,
-                      'body': unified_body,
-                      'attachments': [('file1', 'bytes'),
-                                      ('file2', 'bytes')}
-                    }
+        """ Parses an email.message.Message representing an RFC-2822 email
+        and returns a generic dict holding the message details.
+
+        :param message: email to parse
+        :type message: email.message.Message
+        :param bool save_original: whether the returned dict should include
+            an ``original`` attachment containing the source of the message
+        :rtype: dict
+        :return: A dict with the following structure, where each field may not
+            be present if missing in original message::
+
+            { 'message_id': msg_id,
+              'subject': subject,
+              'email_from': from,
+              'to': to + delivered-to,
+              'cc': cc,
+              'recipients': delivered-to + to + cc + resent-to + resent-cc,
+              'partner_ids': partners found based on recipients emails,
+              'body': unified_body,
+              'references': references,
+              'in_reply_to': in-reply-to,
+              'parent_id': parent mail.message based on in_reply_to or references,
+              'internal': answer to an internal message (note),
+              'date': date,
+              'attachments': [('file1', 'bytes'),
+                              ('file2', 'bytes')}
+            }
         """
-        msg_dict = {
-            'message_type': 'email',
-        }
         if not isinstance(message, Message):
-            # message_from_string works on a native str
-            message = pycompat.to_text(message)
-            message = email.message_from_string(message)
+            raise ValueError(_('Message should be a valid Message instance'))
+        msg_dict = {'message_type': 'email'}
 
-        message_id = message['message-id']
+        message_id = message.get('Message-Id')
         if not message_id:
             # Very unusual situation, be we should be fault-tolerant here
             message_id = "<%s@localhost>" % time.time()
@@ -1303,19 +1289,42 @@ class MailThread(models.AbstractModel):
         msg_dict['message_id'] = message_id
 
         if message.get('Subject'):
-            msg_dict['subject'] = tools.decode_smtp_header(message.get('Subject'))
-
-        # Envelope fields not stored in mail.message but made available for message_new()
-        msg_dict['from'] = tools.decode_smtp_header(message.get('from'))
-        msg_dict['to'] = tools.decode_smtp_header(message.get('to'))
-        msg_dict['cc'] = tools.decode_smtp_header(message.get('cc'))
-        msg_dict['email_from'] = tools.decode_smtp_header(message.get('from'))
-        recipient_emails = ', '.join([tools.decode_smtp_header(message.get(h)) for h in ['To', 'Cc'] if message.get(h)])
-        partner_ids = [x.id for x in self._mail_find_partner_from_emails(tools.email_split(recipient_emails), records=self) if x]
+            msg_dict['subject'] = tools.decode_message_header(message, 'Subject')
+
+        email_from = tools.decode_message_header(message, 'From')
+        email_cc = tools.decode_message_header(message, 'cc')
+        email_from_list = tools.email_split_and_format(email_from)
+        email_cc_list = tools.email_split_and_format(email_cc)
+        msg_dict['email_from'] = email_from_list[0] if email_from_list else email_from
+        msg_dict['from'] = msg_dict['email_from']  # compatibility for message_new
+        msg_dict['cc'] = ','.join(email_cc_list) if email_cc_list else email_cc
+        # Delivered-To is a safe bet in most modern MTAs, but we have to fallback on To + Cc values
+        # for all the odd MTAs out there, as there is no standard header for the envelope's `rcpt_to` value.
+        msg_dict['recipients'] = ','.join(formatted_email
+            for address in [
+                tools.decode_message_header(message, 'Delivered-To'),
+                tools.decode_message_header(message, 'To'),
+                tools.decode_message_header(message, 'Cc'),
+                tools.decode_message_header(message, 'Resent-To'),
+                tools.decode_message_header(message, 'Resent-Cc')
+            ] if address
+            for formatted_email in tools.email_split_and_format(address)
+        )
+        msg_dict['to'] = ','.join(formatted_email
+            for address in [
+                tools.decode_message_header(message, 'Delivered-To'),
+                tools.decode_message_header(message, 'To')
+            ] if address
+            for formatted_email in tools.email_split_and_format(address))
+        partner_ids = [x.id for x in self._mail_find_partner_from_emails(tools.email_split(msg_dict['recipients']), records=self) if x]
         msg_dict['partner_ids'] = partner_ids
+        # compute references to find if email_message is a reply to an existing thread
+        msg_dict['references'] = tools.decode_message_header(message, 'References')
+        msg_dict['in_reply_to'] = tools.decode_message_header(message, 'In-Reply-To').strip()
+
         if message.get('Date'):
             try:
-                date_hdr = tools.decode_smtp_header(message.get('Date'))
+                date_hdr = tools.decode_message_header(message, 'Date')
                 parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True)
                 if parsed_date.utcoffset() is None:
                     # naive datetime, so we arbitrarily decide to make it
@@ -1326,25 +1335,25 @@ class MailThread(models.AbstractModel):
                     stored_date = parsed_date.astimezone(tz=pytz.utc)
             except Exception:
                 _logger.info('Failed to parse Date header %r in incoming mail '
-                                'with message-id %r, assuming current date/time.',
-                                message.get('Date'), message_id)
+                             'with message-id %r, assuming current date/time.',
+                             message.get('Date'), message_id)
                 stored_date = datetime.datetime.now()
             msg_dict['date'] = stored_date.strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT)
 
-        if message.get('In-Reply-To'):
-            parent_ids = self.env['mail.message'].search([('message_id', '=', tools.decode_smtp_header(message['In-Reply-To'].strip()))], limit=1)
+        if msg_dict['in_reply_to']:
+            parent_ids = self.env['mail.message'].search([('message_id', '=', msg_dict['in_reply_to'])], limit=1)
             if parent_ids:
                 msg_dict['parent_id'] = parent_ids.id
                 msg_dict['internal'] = parent_ids.subtype_id and parent_ids.subtype_id.internal or False
 
-        if message.get('References') and 'parent_id' not in msg_dict:
-            msg_list = tools.mail_header_msgid_re.findall(tools.decode_smtp_header(message['References']))
-            parent_ids = self.env['mail.message'].search([('message_id', 'in', [x.strip() for x in msg_list])], limit=1)
+        if msg_dict['references'] and 'parent_id' not in msg_dict:
+            references_msg_id_list = tools.mail_header_msgid_re.findall(msg_dict['references'])
+            parent_ids = self.env['mail.message'].search([('message_id', 'in', [x.strip() for x in references_msg_id_list])], limit=1)
             if parent_ids:
                 msg_dict['parent_id'] = parent_ids.id
                 msg_dict['internal'] = parent_ids.subtype_id and parent_ids.subtype_id.internal or False
 
-        msg_dict['body'], msg_dict['attachments'] = self._message_extract_payload(message, save_original=save_original)
+        msg_dict.update(self._message_parse_extract_payload(message, save_original=save_original))
         return msg_dict
 
     # ------------------------------------------------------
@@ -1887,9 +1896,6 @@ class MailThread(models.AbstractModel):
 
     def _message_create(self, values):
         create_values = dict(values)
-        # Avoid warnings about non-existing fields
-        for x in ('from', 'to', 'cc', 'canned_response_ids'):
-            create_values.pop(x, None)
         create_values['partner_ids'] = [(4, pid) for pid in create_values.get('partner_ids', [])]
         create_values['channel_ids'] = [(4, cid) for cid in create_values.get('channel_ids', [])]
         return self.env['mail.message'].create(create_values)
diff --git a/addons/test_mail/tests/test_mail_gateway.py b/addons/test_mail/tests/test_mail_gateway.py
index 482e3688f3b..45470ac6dff 100644
--- a/addons/test_mail/tests/test_mail_gateway.py
+++ b/addons/test_mail/tests/test_mail_gateway.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import email
 import socket
 
 from email.utils import formataddr
@@ -9,7 +10,7 @@ from odoo.addons.test_mail.data import test_mail_data
 from odoo.addons.test_mail.data.test_mail_data import MAIL_TEMPLATE
 from odoo.addons.test_mail.tests.common import BaseFunctionalTest, MockEmails
 from odoo.addons.test_mail.tests.common import mail_new_test_user
-from odoo.tools import mute_logger
+from odoo.tools import mute_logger, pycompat
 
 
 from odoo.tests import tagged
@@ -21,16 +22,16 @@ class TestEmailParsing(BaseFunctionalTest, MockEmails):
     def test_message_parse_body(self):
         # test pure plaintext
         plaintext = self.format(test_mail_data.MAIL_TEMPLATE_PLAINTEXT, email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>')
-        res = self.env['mail.thread'].message_parse(plaintext)
+        res = self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(plaintext)))
         self.assertIn('Please call me as soon as possible this afternoon!', res['body'])
 
         # test multipart / text and html -> html has priority
         multipart = self.format(MAIL_TEMPLATE, email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>')
-        res = self.env['mail.thread'].message_parse(multipart)
+        res = self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(multipart)))
         self.assertIn('<p>Please call me as soon as possible this afternoon!</p>', res['body'])
 
         # test multipart / mixed
-        res = self.env['mail.thread'].message_parse(test_mail_data.MAIL_MULTIPART_MIXED)
+        res = self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(test_mail_data.MAIL_MULTIPART_MIXED)))
         self.assertNotIn(
             'Should create a multipart/mixed: from gmail, *bold*, with attachment', res['body'],
             'message_parse: text version should not be in body after parsing multipart/mixed')
@@ -38,7 +39,7 @@ class TestEmailParsing(BaseFunctionalTest, MockEmails):
             '<div dir="ltr">Should create a multipart/mixed: from gmail, <b>bold</b>, with attachment.<br clear="all"><div><br></div>', res['body'],
             'message_parse: html version should be in body after parsing multipart/mixed')
 
-        res = self.env['mail.thread'].message_parse(test_mail_data.MAIL_MULTIPART_MIXED_TWO)
+        res = self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(test_mail_data.MAIL_MULTIPART_MIXED_TWO)))
         self.assertNotIn('First and second part', res['body'],
                          'message_parse: text version should not be in body after parsing multipart/mixed')
         self.assertIn('First part', res['body'],
@@ -46,22 +47,24 @@ class TestEmailParsing(BaseFunctionalTest, MockEmails):
         self.assertIn('Second part', res['body'],
                       'message_parse: second part of the html version should be in body after parsing multipart/mixed')
 
-        res = self.env['mail.thread'].message_parse(test_mail_data.MAIL_SINGLE_BINARY)
+        res = self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(test_mail_data.MAIL_SINGLE_BINARY)))
         self.assertEqual(res['body'], '')
         self.assertEqual(res['attachments'][0][0], 'thetruth.pdf')
 
     def test_message_parse_eml(self):
         # Test that the parsing of mail with embedded emails as eml(msg) which generates empty attachments, can be processed.
-        self.env['mail.thread'].message_parse(self.format(test_mail_data.MAIL_EML_ATTACHMENT, email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>', to='generic@test.com'))
+        mail = self.format(test_mail_data.MAIL_EML_ATTACHMENT, email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>', to='generic@test.com')
+        self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(mail)))
 
     def test_message_parse_plaintext(self):
         """ Incoming email in plaintext should be stored as html """
-        res = self.env['mail.thread'].message_parse(self.format(test_mail_data.MAIL_TEMPLATE_PLAINTEXT, email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>', to='generic@test.com'))
+        mail = self.format(test_mail_data.MAIL_TEMPLATE_PLAINTEXT, email_from='Sylvie Lelitre <test.sylvie.lelitre@agrolait.com>', to='generic@test.com')
+        res = self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(mail)))
         self.assertIn('<pre>\nPlease call me as soon as possible this afternoon!\n\n--\nSylvie\n</pre>', res['body'])
 
     def test_message_parse_xhtml(self):
         # Test that the parsing of XHTML mails does not fail
-        self.env['mail.thread'].message_parse(test_mail_data.MAIL_XHTML)
+        self.env['mail.thread'].message_parse(email.message_from_string(pycompat.to_text(test_mail_data.MAIL_XHTML)))
 
 
 @tagged('mail_gateway')
@@ -175,7 +178,7 @@ class TestMailgateway(BaseFunctionalTest, MockEmails):
         self.assertEqual(len(self._mails), 0, 'No notification / bounce should be sent')
 
         # Email recognized if partner has a formatted email
-        self.partner_1.write({'email': {'Valid Lelitre <%s>' % self.partner_1.email}})
+        self.partner_1.write({'email': 'Valid Lelitre <%s>' % self.partner_1.email})
         record = self.format_and_process(MAIL_TEMPLATE, self.partner_1.email, 'groups@test.com', subject='Test2')
 
         self.assertEqual(record.message_ids[0].author_id, self.partner_1,
diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py
index c7784a11f03..b32934101e8 100644
--- a/odoo/tools/mail.py
+++ b/odoo/tools/mail.py
@@ -436,12 +436,6 @@ email_re = re.compile(r"""([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63})""",
 # matches a string containing only one email
 single_email_re = re.compile(r"""^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63}$""", re.VERBOSE)
 
-# Updated in 7.0 to match the model name as well
-# Typical form of references is <timestamp-openerp-record_id-model_name@domain>
-# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain
-reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?[^>]*@([^>]*)>", re.UNICODE)
-discussion_re = re.compile("<.*-open(?:object|erp)-private[^>]*@([^>]*)>", re.UNICODE)
-
 mail_header_msgid_re = re.compile('<[^<>]+>')
 
 
@@ -551,20 +545,6 @@ def email_escape_char(email_address):
     """ Escape problematic characters in the given email address string"""
     return email_address.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_')
 
-def email_references(references):
-    ref_match, model, thread_id, hostname, is_private = False, False, False, False, False
-    if references:
-        ref_match = reference_re.search(references)
-    if ref_match:
-        model = ref_match.group(2)
-        thread_id = int(ref_match.group(1))
-        hostname = ref_match.group(3)
-    else:
-        ref_match = discussion_re.search(references)
-        if ref_match:
-            is_private = True
-    return (ref_match, model, thread_id, hostname, is_private)
-
 # was mail_message.decode()
 def decode_smtp_header(smtp_header):
     """Returns unicode() string conversion of the given encoded smtp header