[FIX] mail: fixed bounce email recognition + invite email headers + mass mailing statistics not lost anymore

- [FIX] bounce regex: too many emails were considered as bounce and therefore
not displayed in the chatter and lost for the communication history. The regex
was not correctly looking for the bounce alias in the email_to.
- [FIX] invite email: replying to the invitation email (invitation as new
follower) now replies to the user sending the invitation.
- [FIX] mass_mailing: added a column to store the id of the original email
in addition to the many2one column. The many2one is set to null when deleting
the original email. As the information is necessary, it is saved on another
field. The many2one is necessary for indexes purpose as the inverse of
a one2many.

9 files changed, 116 insertions, 87 deletions

diff --git a/addons/mail/mail_message.py b/addons/mail/mail_message.py
index 8109e3422aa..ab568d77004 100644
--- a/addons/mail/mail_message.py
+++ b/addons/mail/mail_message.py
@@ -790,7 +790,7 @@ class mail_message(osv.Model):
 
         if 'email_from' not in values:  # needed to compute reply_to
             values['email_from'] = self._get_default_from(cr, uid, context=context)
-        if 'message_id' not in values:
+        if not values.get('message_id'):
             values['message_id'] = self._get_message_id(cr, uid, values, context=context)
         if 'reply_to' not in values:
             values['reply_to'] = self._get_reply_to(cr, uid, values, context=context)
diff --git a/addons/mail/tests/test_mail_gateway.py b/addons/mail/tests/test_mail_gateway.py
index 1b16f151cdf..9f9c463625a 100644
--- a/addons/mail/tests/test_mail_gateway.py
+++ b/addons/mail/tests/test_mail_gateway.py
@@ -404,7 +404,8 @@ class TestMailgateway(TestMail):
 
         # When 6.1 messages are present, compat mode is available
         # Create a fake 6.1 message
-        tmp_msg_id = self.mail_message.create(cr, uid, {'message_id': False, 'model': 'mail.group', 'res_id': frog_group.id})
+        tmp_msg_id = self.mail_message.create(cr, uid, {'model': 'mail.group', 'res_id': frog_group.id})
+        self.mail_message.write(cr, uid, [tmp_msg_id], {'message_id': False})
         # Do: compat mode accepts partial-matching emails
         frog_groups = format_and_process(MAIL_TEMPLATE, email_from='other5@gmail.com',
                                          msg_id='<1.2.JavaMail.new@agrolait.com>',
@@ -422,7 +423,8 @@ class TestMailgateway(TestMail):
         self.assertEqual(len(frog_group.message_ids), 4, 'message_process: group should contain 4 messages after reply')
 
         # 6.1 compat mode should not work if hostname does not match!
-        tmp_msg_id = self.mail_message.create(cr, uid, {'message_id': False, 'model': 'mail.group', 'res_id': frog_group.id})
+        tmp_msg_id = self.mail_message.create(cr, uid, {'model': 'mail.group', 'res_id': frog_group.id})
+        self.mail_message.write(cr, uid, [tmp_msg_id], {'message_id': False})
         self.assertRaises(ValueError,
                           format_and_process,
                           MAIL_TEMPLATE, email_from='other5@gmail.com',
diff --git a/addons/mail/wizard/invite.py b/addons/mail/wizard/invite.py
index 457eabb319c..9399e4f943f 100644
--- a/addons/mail/wizard/invite.py
+++ b/addons/mail/wizard/invite.py
@@ -60,9 +60,9 @@ class invite_wizard(osv.osv_memory):
             help="If checked, the partners will receive an email warning they have been "
                     "added in the document's followers."),
     }
-    
+
     _defaults = {
-        'send_mail' : True,
+        'send_mail': True,
     }
 
     def add_followers(self, cr, uid, ids, context=None):
@@ -91,10 +91,13 @@ class invite_wizard(osv.osv_memory):
                 mail_id = mail_mail.create(cr, uid, {
                     'model': wizard.res_model,
                     'res_id': wizard.res_id,
+                    'email_from': self.pool['mail.message']._get_default_from(cr, uid, context=context),
+                    'reply_to': self.pool['mail.message']._get_default_from(cr, uid, context=context),
                     'subject': _('Invitation to follow %s: %s') % (model_name, document.name_get()[0][1]),
                     'body_html': '%s' % wizard.message,
                     'auto_delete': True,
+                    'message_id': self.pool['mail.message']._get_message_id(cr, uid, {'no_auto_thread': True}, context=context),
                     'recipient_ids': [(4, id) for id in new_follower_ids]
-                    }, context=context)
+                }, context=context)
                 mail_mail.send(cr, uid, [mail_id], context=context)
         return {'type': 'ir.actions.act_window_close'}
diff --git a/addons/mail/wizard/mail_compose_message.py b/addons/mail/wizard/mail_compose_message.py
index a33982b3860..a0577b6b86a 100644
--- a/addons/mail/wizard/mail_compose_message.py
+++ b/addons/mail/wizard/mail_compose_message.py
@@ -265,6 +265,7 @@ class mail_compose_message(osv.TransientModel):
                 'author_id': wizard.author_id.id,
                 'email_from': wizard.email_from,
                 'record_name': wizard.record_name,
+                'no_auto_thread': wizard.no_auto_thread,
             }
             # mass mailing: rendering override wizard static values
             if mass_mail_mode and wizard.model:
diff --git a/addons/mass_mailing/models/mail_mail.py b/addons/mass_mailing/models/mail_mail.py
index 41be9c21cbf..03c7b155ad6 100644
--- a/addons/mass_mailing/models/mail_mail.py
+++ b/addons/mass_mailing/models/mail_mail.py
@@ -45,7 +45,7 @@ class MailMail(osv.Model):
         # TDE note: should be after 'all values computed', to have values (FIXME after merging other branch holding create refactoring)
         mail_id = super(MailMail, self).create(cr, uid, values, context=context)
         if values.get('statistics_ids'):
-            mail = self.browse(cr, SUPERUSER_ID, mail_id)
+            mail = self.browse(cr, SUPERUSER_ID, mail_id, context=context)
             for stat in mail.statistics_ids:
                 self.pool['mail.mail.statistics'].write(cr, uid, [stat.id], {'message_id': mail.message_id}, context=context)
         return mail_id
diff --git a/addons/mass_mailing/models/mail_thread.py b/addons/mass_mailing/models/mail_thread.py
index 524341f88d7..7bfce705043 100644
--- a/addons/mass_mailing/models/mail_thread.py
+++ b/addons/mass_mailing/models/mail_thread.py
@@ -20,8 +20,8 @@
 ##############################################################################
 
 import logging
+import re
 
-from openerp import tools
 from openerp.addons.mail.mail_message import decode
 from openerp.addons.mail.mail_thread import decode_header
 from openerp.osv import osv
@@ -46,7 +46,11 @@ class MailThread(osv.AbstractModel):
 
         # 0. Verify whether this is a bounced email (wrong destination,...) -> use it to collect data, such as dead leads
         if bounce_alias in email_to:
-            bounce_match = tools.bounce_re.search(email_to)
+            # Bounce regex
+            # Typical form of bounce is bounce_alias-128-crm.lead-34@domain
+            # group(1) = the mail ID; group(2) = the model (if any); group(3) = the record ID
+            bounce_re = re.compile("%s-(\d+)-?([\w.]+)?-?(\d+)?" % re.escape(bounce_alias), re.UNICODE)
+            bounce_match = bounce_re.search(email_to)
             if bounce_match:
                 bounced_model, bounced_thread_id = None, False
                 bounced_mail_id = bounce_match.group(1)
diff --git a/addons/mass_mailing/models/mass_mailing_stats.py b/addons/mass_mailing/models/mass_mailing_stats.py
index 7e8c194f42e..c3cd4314fa6 100644
--- a/addons/mass_mailing/models/mass_mailing_stats.py
+++ b/addons/mass_mailing/models/mass_mailing_stats.py
@@ -21,6 +21,7 @@
 
 from openerp.osv import osv, fields
 
+
 class MailMailStats(osv.Model):
     """ MailMailStats models the statistics collected about emails. Those statistics
     are stored in a separated model and table to avoid bloating the mail_mail table
@@ -33,7 +34,13 @@ class MailMailStats(osv.Model):
     _order = 'message_id'
 
     _columns = {
-        'mail_mail_id': fields.many2one('mail.mail', 'Mail ID', ondelete='set null'),
+        'mail_mail_id': fields.many2one('mail.mail', 'Mail', ondelete='set null'),
+        'mail_mail_id_int': fields.integer(
+            'Mail ID (tech)',
+            help='ID of the related mail_mail. This field is an integer field because'
+                 'the related mail_mail can be deleted separately from its statistics.'
+                 'However the ID is needed for several action and controllers.'
+        ),
         'message_id': fields.char('Message-ID'),
         'model': fields.char('Document model'),
         'res_id': fields.integer('Document ID'),
@@ -62,9 +69,15 @@ class MailMailStats(osv.Model):
         'scheduled': fields.datetime.now,
     }
 
+    def create(self, cr, uid, values, context=None):
+        if 'mail_mail_id' in values:
+            values['mail_mail_id_int'] = values['mail_mail_id']
+        res = super(MailMailStats, self).create(cr, uid, values, context=context)
+        return res
+
     def _get_ids(self, cr, uid, ids=None, mail_mail_ids=None, mail_message_ids=None, domain=None, context=None):
         if not ids and mail_mail_ids:
-            base_domain = [('mail_mail_id', 'in', mail_mail_ids)]
+            base_domain = [('mail_mail_id_int', 'in', mail_mail_ids)]
         elif not ids and mail_message_ids:
             base_domain = [('message_id', 'in', mail_message_ids)]
         else:
diff --git a/addons/mass_mailing/views/mass_mailing.xml b/addons/mass_mailing/views/mass_mailing.xml
index 2939ebc3711..c37a2b045dd 100644
--- a/addons/mass_mailing/views/mass_mailing.xml
+++ b/addons/mass_mailing/views/mass_mailing.xml
@@ -19,6 +19,82 @@
         <menuitem name="Configuration" id="marketing_configuration"
             parent="base.marketing_menu" sequence="99"/>
 
+        <!--  MAIL MAIL STATISTICS !-->
+        <record model="ir.ui.view" id="view_mail_mail_statistics_search">
+            <field name="name">mail.mail.statistics.search</field>
+            <field name="model">mail.mail.statistics</field>
+            <field name="arch" type="xml">
+               <search string="Mail Statistics">
+                    <field name="mail_mail_id_int"/>
+                    <field name="message_id"/>
+                    <field name="mass_mailing_id"/>
+                </search>
+            </field>
+        </record>
+
+        <record model="ir.ui.view" id="view_mail_mail_statistics_tree">
+            <field name="name">mail.mail.statistics.tree</field>
+            <field name="model">mail.mail.statistics</field>
+            <field name="arch" type="xml">
+                <tree string="Mail Statistics">
+                    <field name="mail_mail_id_int"/>
+                    <field name="message_id"/>
+                    <field name="sent"/>
+                    <field name="exception"/>
+                    <field name="opened"/>
+                    <field name="replied"/>
+                    <field name="bounced"/>
+                </tree>
+            </field>
+        </record>
+
+        <record model="ir.ui.view" id="view_mail_mail_statistics_form">
+            <field name="name">mail.mail.statistics.form</field>
+            <field name="model">mail.mail.statistics</field>
+            <field name="arch" type="xml">
+                <form string="Mail Statistics">
+                    <group>
+                        <group>
+                            <field name="mail_mail_id"/>
+                            <field name="mail_mail_id_int"/>
+                            <field name="message_id"/>
+                            <field name="exception"/>
+                            <field name="sent"/>
+                            <field name="opened"/>
+                            <field name="replied"/>
+                            <field name="bounced"/>
+                        </group>
+                        <group>
+                            <field name="mass_mailing_id"/>
+                            <field name="mass_mailing_campaign_id"/>
+                            <field name="model"/>
+                            <field name="res_id"/>
+                        </group>
+                    </group>
+                </form>
+            </field>
+        </record>
+
+        <record id="action_view_mail_mail_statistics" model="ir.actions.act_window">
+            <field name="name">Mail Statistics</field>
+            <field name="res_model">mail.mail.statistics</field>
+            <field name="view_type">form</field>
+            <field name="view_mode">tree,form</field>
+        </record>
+
+        <record id="action_view_mail_mail_statistics_mailing" model="ir.actions.act_window">
+            <field name="name">Mail Statistics</field>
+            <field name="res_model">mail.mail.statistics</field>
+            <field name="view_type">form</field>
+            <field name="view_mode">tree,form</field>
+            <field name="context">{'search_default_mass_mailing_id': active_id}</field>
+        </record>
+
+        <!-- Add in Technical/Email -->
+        <menuitem name="Mail Statistics" id="menu_email_statistics"
+            parent="base.menu_email" sequence="50"
+            action="action_view_mail_mail_statistics"/>
+
         <!--  MASS MAILING CONTACT -->
         <record model="ir.ui.view" id="view_mail_mass_mailing_contact_search">
             <field name="name">mail.mass_mailing.contact.search</field>
@@ -207,28 +283,28 @@
                     </div>
                     <sheet>
                         <div class="oe_button_box pull-right" attrs="{'invisible': [('state', 'in', ('draft','test'))]}">
-                            <button name="%(action_view_mass_mailing_contacts)d"
+                            <button name="%(action_view_mail_mail_statistics_mailing)d"
                                 type="action" class="oe_stat_button">
                                 <field name="received_ratio" string="Received" widget="percentpie"/>
                             </button>
-                            <button name="%(action_view_mass_mailing_contacts)d"
+                            <button name="%(action_view_mail_mail_statistics_mailing)d"
                                 type="action" class="oe_stat_button">
                                 <field name="opened_ratio" string="Opened" widget="percentpie"/>
                             </button>
-                            <button name="%(action_view_mass_mailing_contacts)d"
+                            <button name="%(action_view_mail_mail_statistics_mailing)d"
                                 type="action" class="oe_stat_button">
                                 <field name="replied_ratio" string="Replied" widget="percentpie"/>
                             </button>
-                            <button name="%(action_view_mass_mailing_contacts)d"
+                            <button name="%(action_view_mail_mail_statistics_mailing)d"
                                 type="action" class="oe_stat_button oe_inline">
                                 <field name="opened_daily" string="Opened Daily" widget="barchart"/>
                             </button>
-                            <button name="%(action_view_mass_mailing_contacts)d"
+                            <button name="%(action_view_mail_mail_statistics_mailing)d"
                                 type="action" class="oe_stat_button oe_inline">
                                 <field name="replied_daily" string="Replied Daily" widget="barchart"/>
                             </button>
                         </div>
-                        <button name="%(action_view_mass_mailing_contacts)d" type="action"
+                        <button name="%(action_view_mail_mail_statistics_mailing)d" type="action"
                             icon="fa-envelope-o" class="oe_stat_button"
                             attrs="{'invisible': [('total', '=', 0)]}" >
                             <field name="total" string="Emails" widget="statinfo"/>
@@ -571,71 +647,5 @@
             action="action_view_mass_mailing_campaigns"
             groups="mass_mailing.group_mass_mailing_campaign"/>
 
-        <!--  MAIL MAIL STATISTICS !-->
-        <record model="ir.ui.view" id="view_mail_mail_statistics_search">
-            <field name="name">mail.mail.statistics.search</field>
-            <field name="model">mail.mail.statistics</field>
-            <field name="arch" type="xml">
-               <search string="Mail Statistics">
-                    <field name="mail_mail_id"/>
-                    <field name="message_id"/>
-                </search>
-            </field>
-        </record>
-
-        <record model="ir.ui.view" id="view_mail_mail_statistics_tree">
-            <field name="name">mail.mail.statistics.tree</field>
-            <field name="model">mail.mail.statistics</field>
-            <field name="arch" type="xml">
-                <tree string="Mail Statistics">
-                    <field name="mail_mail_id"/>
-                    <field name="message_id"/>
-                    <field name="sent"/>
-                    <field name="exception"/>
-                    <field name="opened"/>
-                    <field name="replied"/>
-                    <field name="bounced"/>
-                </tree>
-            </field>
-        </record>
-
-        <record model="ir.ui.view" id="view_mail_mail_statistics_form">
-            <field name="name">mail.mail.statistics.form</field>
-            <field name="model">mail.mail.statistics</field>
-            <field name="arch" type="xml">
-                <form string="Mail Statistics">
-                    <group>
-                        <group>
-                            <field name="mail_mail_id"/>
-                            <field name="message_id"/>
-                            <field name="exception"/>
-                            <field name="sent"/>
-                            <field name="opened"/>
-                            <field name="replied"/>
-                            <field name="bounced"/>
-                        </group>
-                        <group>
-                            <field name="mass_mailing_id"/>
-                            <field name="mass_mailing_campaign_id"/>
-                            <field name="model"/>
-                            <field name="res_id"/>
-                        </group>
-                    </group>
-                </form>
-            </field>
-        </record>
-
-        <record id="action_view_mail_mail_statistics" model="ir.actions.act_window">
-            <field name="name">Mail Statistics</field>
-            <field name="res_model">mail.mail.statistics</field>
-            <field name="view_type">form</field>
-            <field name="view_mode">tree,form</field>
-        </record>
-
-        <!-- Add in Technical/Email -->
-        <menuitem name="Mail Statistics" id="menu_email_statistics"
-            parent="base.menu_email" sequence="50"
-            action="action_view_mail_mail_statistics"/>
-
     </data>
 </openerp>
diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index f6406ebb67a..d6ac1c1cb43 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -603,10 +603,6 @@ command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE)
 # group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain
 reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE)
 
-# Bounce regex
-# Typical form of bounce is bounce-128-crm.lead-34@domain
-# group(1) = the mail ID; group(2) = the model (if any); group(3) = the record ID
-bounce_re = re.compile("[\w]+-(\d+)-?([\w.]+)?-?(\d+)?", re.UNICODE)
 
 def generate_tracking_message_id(res_id):
     """Returns a string that can be used in the Message-ID RFC822 header field