[REF] base,mail,tools: use new python email API

[PEP 594] is going to depreciate the legacy `email.message.Message` API
and its related modules: `email.(charset|header|mime|utils)`.

The new `email.message.EmailMessage` API exposes a much easier interface
to create multiparted emails [1], is capable of doing all the necessary
headers value conversion (RFCs [2045], [2047], [2049]) [2] and get/set
different flavor of payload (text/bytes) in a straightforward way [3].

All headers are structured in a way to support python native types, i.e.
the `Date` header supports `datetime.datetime` objects and automatically
performs the required formatting. The same goes for multi-valued headers
like the `To` header, one can directly set a python list of values, it
will be automatically be formatted according to the RFCs.

The dedicated encoding and decoding functions are no more needed thus
has been removed has part of the refactor. FTR, [RFC2231] is an update
of [RFC2047] and based on tests we've just conducted, both GMail and
Thunderbird now support RFC2231 encoding just fine in all headers:
attachments names, From, etc.

[1]: http://docs.python.org/3/library/email.message.html
[2]: http://docs.python.org/3/library/email.headerregistry.html
[3]: http://docs.python.org/3/library/email.contentmanager.html
[2045]: https://tools.ietf.org/html/rfc2045
[2047]: https://tools.ietf.org/html/rfc2047
[2049]: https://tools.ietf.org/html/rfc2049
[2231]: https://tools.ietf.org/html/rfc2231

closes odoo/odoo#35929

Signed-off-by: Raphael Collet (rco) <rco@openerp.com>

7 files changed, 50 insertions, 29 deletions

diff --git a/addons/l10n_it_edi/models/ir_mail_server.py b/addons/l10n_it_edi/models/ir_mail_server.py
index d4575c90fe1..e465a6b7846 100644
--- a/addons/l10n_it_edi/models/ir_mail_server.py
+++ b/addons/l10n_it_edi/models/ir_mail_server.py
@@ -99,7 +99,7 @@ class FetchmailServer(models.Model):
     def _attachment_invoice(self, msg_txt):
         parsed_values = self.env['mail.thread']._message_parse_extract_payload(msg_txt)
         body, attachments = parsed_values['body'], parsed_values['attachments']
-        from_address = tools.decode_smtp_header(msg_txt.get('from'))
+        from_address = msg_txt.get('from')
         for attachment in attachments:
             split_attachment = attachment.fname.rpartition('.')
             if len(split_attachment) < 3:
diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py
index af4001fb4e3..c37073245f3 100644
--- a/addons/mail/models/mail_thread.py
+++ b/addons/mail/models/mail_thread.py
@@ -1231,7 +1231,7 @@ class MailThread(models.AbstractModel):
         #   type="text/html"
         if message.get_content_maintype() == 'text':
             encoding = message.get_content_charset()
-            body = message.get_payload(decode=True)
+            body = message.get_content()
             body = tools.ustr(body, encoding, errors='replace')
             if message.get_content_type() == 'text/plain':
                 # text/plain -> <pre/>
@@ -1247,40 +1247,29 @@ class MailThread(models.AbstractModel):
                     mixed = True
                 if part.get_content_maintype() == 'multipart':
                     continue  # skip container
-                # part.get_filename returns decoded value if able to decode, coded otherwise.
-                # original get_filename is not able to decode iso-8859-1 (for instance).
-                # therefore, iso encoded attachements are not able to be decoded properly with get_filename
-                # code here partially copy the original get_filename method, but handle more encoding
-                filename = part.get_param('filename', None, 'content-disposition')
-                if not filename:
-                    filename = part.get_param('name', None)
-                if filename:
-                    if isinstance(filename, tuple):
-                        # RFC2231
-                        filename = email.utils.collapse_rfc2231_value(filename).strip()
-                    else:
-                        filename = tools.decode_smtp_header(filename)
+
+                filename = part.get_filename()  # I may not properly handle all charsets
                 encoding = part.get_content_charset()  # None if attachment
 
                 # 0) Inline Attachments -> attachments, with a third part in the tuple to match cid / attachment
                 if filename and part.get('content-id'):
                     inner_cid = part.get('content-id').strip('><')
-                    attachments.append(self._Attachment(filename, part.get_payload(decode=True), {'cid': inner_cid}))
+                    attachments.append(self._Attachment(filename, part.get_content(), {'cid': inner_cid}))
                     continue
                 # 1) Explicit Attachments -> attachments
                 if filename or part.get('content-disposition', '').strip().startswith('attachment'):
-                    attachments.append(self._Attachment(filename or 'attachment', part.get_payload(decode=True), {}))
+                    attachments.append(self._Attachment(filename or 'attachment', part.get_content(), {}))
                     continue
                 # 2) text/plain -> <pre/>
                 if part.get_content_type() == 'text/plain' and (not alternative or not body):
-                    body = tools.append_content_to_html(body, tools.ustr(part.get_payload(decode=True),
+                    body = tools.append_content_to_html(body, tools.ustr(part.get_content(),
                                                                          encoding, errors='replace'), preserve=True)
                 # 3) text/html -> raw
                 elif part.get_content_type() == 'text/html':
                     # mutlipart/alternative have one text and a html part, keep only the second
                     # mixed allows several html parts, append html content
                     append_content = not alternative or (html and mixed)
-                    html = tools.ustr(part.get_payload(decode=True), encoding, errors='replace')
+                    html = tools.ustr(part.get_content(), encoding, errors='replace')
                     if not append_content:
                         body = html
                     else:
@@ -1289,7 +1278,7 @@ class MailThread(models.AbstractModel):
                     body = tools.html_sanitize(body, sanitize_tags=False, strip_classes=True)
                 # 4) Anything else -> attachment
                 else:
-                    attachments.append(self._Attachment(filename or 'attachment', part.get_payload(decode=True), {}))
+                    attachments.append(self._Attachment(filename or 'attachment', part.get_content(), {}))
 
         return self._message_parse_extract_payload_postprocess(message, {'body': body, 'attachments': attachments})
 
diff --git a/addons/mass_mailing/models/mail_thread.py b/addons/mass_mailing/models/mail_thread.py
index 77ff580d594..86683a7e540 100644
--- a/addons/mass_mailing/models/mail_thread.py
+++ b/addons/mass_mailing/models/mail_thread.py
@@ -4,7 +4,6 @@
 import datetime
 
 from odoo import api, models, fields
-from odoo.tools import decode_smtp_header
 
 BLACKLIST_MAX_BOUNCED_LIMIT = 5
 
@@ -19,7 +18,7 @@ class MailThread(models.AbstractModel):
         by using the References header of the incoming message and looking for
         matching message_id in mailing.trace. """
         if message.get('References') and routes:
-            message_ids = [x.strip() for x in decode_smtp_header(message['References']).split()]
+            message_ids = [x.strip() for x in message['References'].split()]
             self.env['mailing.trace'].set_opened(mail_message_ids=message_ids)
             self.env['mailing.trace'].set_replied(mail_message_ids=message_ids)
         return super(MailThread, self)._message_route_process(message, message_dict, routes)
diff --git a/addons/test_mail/data/test_mail_data.py b/addons/test_mail/data/test_mail_data.py
index cfce6317440..fa64fd56e26 100644
--- a/addons/test_mail/data/test_mail_data.py
+++ b/addons/test_mail/data/test_mail_data.py
@@ -184,6 +184,40 @@ Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\))
 SSBhbSB0aGUgQmF0TWFuCg=="""
 
 
+MAIL_MULTIPART_WEIRD_FILENAME = """X-Original-To: john@doe.com
+Delivered-To: johndoe@example.com
+Received: by mail.example.com (Postfix, from userid 10002)
+    id E8166BFACB; Fri, 23 Aug 2013 13:18:02 +0200 (CEST)
+From: "Bruce Wayne" <bruce@wayneenterprises.com>
+Subject: test
+Message-ID: <c0c20fdd-a38e-b296-865b-d9232bf30ce5@odoo.com>
+Date: Mon, 26 Aug 2019 16:55:09 +0200
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+ boundary="------------FACA7766210AAA981EAE01F3"
+Content-Language: en-US
+
+This is a multi-part message in MIME format.
+--------------FACA7766210AAA981EAE01F3
+Content-Type: text/plain; charset=utf-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+plop
+
+
+--------------FACA7766210AAA981EAE01F3
+Content-Type: text/plain; charset=UTF-8;
+ name="=?UTF-8?B?NjJfQDssXVspPS4ow4fDgMOJLnR4dA==?="
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment;
+ filename*0*=utf-8'en-us'%36%32%5F%40%3B%2C%5D%5B%29%3D%2E%28%C3%87%C3%80%C3%89;
+ filename*1*=%2E%74%78%74
+
+SSBhbSBhIGZpbGUgd2l0aCBhIHZhbGlkIHdpbmRvd3MgZmlsZW5hbWUK
+--------------FACA7766210AAA981EAE01F3--
+"""
+
+
 MAIL_MULTIPART_IMAGE = """X-Original-To: raoul@example.com
 Delivered-To: micheline@example.com
 Received: by mail1.example.com (Postfix, from userid 99999)
diff --git a/addons/test_mail/tests/test_discuss.py b/addons/test_mail/tests/test_discuss.py
index be255274ea3..f4fb89a209e 100644
--- a/addons/test_mail/tests/test_discuss.py
+++ b/addons/test_mail/tests/test_discuss.py
@@ -126,7 +126,7 @@ class TestDiscuss(BaseFunctionalTest, TestRecipients, MockEmails):
         record = self.env['mail.test.cc'].create({'email_cc': 'cc1@example.com, cc2@example.com, cc3 <cc3@example.com>'})
         suggestions = record._message_get_suggested_recipients()[record.id]
         self.assertEqual(sorted(suggestions), [
+            (False, '"cc3" <cc3@example.com>', 'CC Email'),
             (False, 'cc1@example.com', 'CC Email'),
             (False, 'cc2@example.com', 'CC Email'),
-            (False, 'cc3 <cc3@example.com>', 'CC Email')
         ], 'cc should be in suggestions')
diff --git a/addons/test_mail/tests/test_mail_gateway.py b/addons/test_mail/tests/test_mail_gateway.py
index 79fb1cf5c3c..440085af206 100644
--- a/addons/test_mail/tests/test_mail_gateway.py
+++ b/addons/test_mail/tests/test_mail_gateway.py
@@ -15,11 +15,7 @@ from odoo.addons.test_mail.models.test_mail_models import MailTestGateway
 from odoo.addons.test_mail.tests.common import BaseFunctionalTest, MockEmails
 from odoo.addons.test_mail.tests.common import mail_new_test_user
 from odoo.tests import tagged
-<<<<<<< HEAD
-from odoo.tools import email_split_and_format, mute_logger, pycompat
-=======
-from odoo.tools import mute_logger, pycompat, formataddr
->>>>>>> [REF] mail: remove deprecated email formataddr
+from odoo.tools import email_split_and_format, mute_logger, pycompat, formataddr
 
 def from_string(text):
     return email.message_from_string(pycompat.to_text(text), policy=email.policy.SMTP)
@@ -60,6 +56,9 @@ class TestEmailParsing(BaseFunctionalTest, MockEmails):
         self.assertEqual(res['body'], '')
         self.assertEqual(res['attachments'][0][0], 'thetruth.pdf')
 
+        res = self.env['mail.thread'].message_parse(from_string(test_mail_data.MAIL_MULTIPART_WEIRD_FILENAME))
+        self.assertEqual(res['attachments'][0][0], '62_@;,][)=.(ÇÀÉ.txt')
+
     def test_message_parse_eml(self):
         # Test that the parsing of mail with embedded emails as eml(msg) which generates empty attachments, can be processed.
         mail = self.format(test_mail_data.MAIL_EML_ATTACHMENT, email_from='"Sylvie Lelitre" <test.sylvie.lelitre@agrolait.com>', to='generic@test.com')
diff --git a/addons/test_mail/tests/test_mail_message.py b/addons/test_mail/tests/test_mail_message.py
index be4b5e7887d..13a41ab84b1 100644
--- a/addons/test_mail/tests/test_mail_message.py
+++ b/addons/test_mail/tests/test_mail_message.py
@@ -69,7 +69,7 @@ class TestMessageValues(common.BaseFunctionalTest, common.MockEmails):
         reply_to_name = '%s %s' % (self.env.user.company_id.name, self.alias_record.name)
         reply_to_email = '%s@%s' % (self.alias_record.alias_name, self.alias_domain)
         self.assertEqual(msg.reply_to, formataddr((reply_to_name, reply_to_email)))
-        self.assertEqual(msg.email_from, '%s <%s>' % (self.user_employee.name, self.user_employee.email))
+        self.assertEqual(msg.email_from, formataddr((self.user_employee.name, self.user_employee.email)))
 
         # no alias domain -> author
         self.env['ir.config_parameter'].search([('key', '=', 'mail.catchall.domain')]).unlink()