[MOV] link_tracker, various: more link shortening tools to mail.render.mixin

PURPOSE

Move links shortening tools on mail.render.mixin to have rendering and
post procesing tools available on that mixin.

SPECIFICATIONS

Move html and text links shortening methods on mail.render.mixin class.
That way rendering and post processing is done in that mixin instead of
being specific to link.tracker model. Indeed those methods are tools and
not really business related methods.

Future commits will improve the way mail and sms composer use the render
mixin. This commit is therefore both a cleaning and preparing commit.

LINKS

Task ID 1963529
Community PR odoo/odoo#32397

12 files changed, 94 insertions, 71 deletions

diff --git a/addons/link_tracker/__manifest__.py b/addons/link_tracker/__manifest__.py
index 3f622632cc4..038fe807173 100644
--- a/addons/link_tracker/__manifest__.py
+++ b/addons/link_tracker/__manifest__.py
@@ -8,7 +8,7 @@
 Shorten URLs and use them to track clicks and UTMs
 """,
     'version': '1.0',
-    'depends': ['utm'],
+    'depends': ['utm', 'mail'],
     'data': [
         'views/link_tracker_views.xml',
         'views/utm_campaign_views.xml',
diff --git a/addons/link_tracker/models/__init__.py b/addons/link_tracker/models/__init__.py
index 847f935569e..6f884bd7c2e 100644
--- a/addons/link_tracker/models/__init__.py
+++ b/addons/link_tracker/models/__init__.py
@@ -2,4 +2,5 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from . import link_tracker
+from . import mail_render_mixin
 from . import utm
diff --git a/addons/link_tracker/models/link_tracker.py b/addons/link_tracker/models/link_tracker.py
index 3af4accb9ba..ec73046e1be 100644
--- a/addons/link_tracker/models/link_tracker.py
+++ b/addons/link_tracker/models/link_tracker.py
@@ -1,28 +1,14 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import base64
-import datetime
 import random
-import re
 import requests
 import string
 
 from lxml import html
-from werkzeug import urls, utils
+from werkzeug import urls
 
-from odoo import models, fields, api, _
-
-
-URL_REGEX = r'(\bhref=[\'"](?!mailto:|tel:|sms:)([^\'"]+)[\'"])'
-TEXT_URL_REGEX = r'https?://[a-zA-Z0-9@:%._\+~#=/-]+(?:\?\S+)?'
-
-
-def VALIDATE_URL(url):
-    if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
-        return 'http://' + url
-
-    return url
+from odoo import tools, models, fields, api, _
 
 
 class LinkTracker(models.Model):
@@ -101,7 +87,7 @@ class LinkTracker(models.Model):
         if 'url' not in create_vals:
             raise ValueError('URL field required')
         else:
-            create_vals['url'] = VALIDATE_URL(vals['url'])
+            create_vals['url'] = tools.validate_url(vals['url'])
 
         search_domain = []
         for fname, value in create_vals.items():
@@ -129,44 +115,10 @@ class LinkTracker(models.Model):
 
     @api.model
     def convert_links(self, html, vals, blacklist=None):
-        for match in re.findall(URL_REGEX, html):
-
-            short_schema = self.env['ir.config_parameter'].sudo().get_param('web.base.url') + '/r/'
-
-            href = match[0]
-            long_url = match[1]
-
-            vals['url'] = utils.unescape(long_url)
-
-            if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema):
-                link = self.create(vals)
-                shorten_url = self.browse(link.id)[0].short_url
-
-                if shorten_url:
-                    new_href = href.replace(long_url, shorten_url)
-                    html = html.replace(href, new_href)
-
-        return html
+        raise NotImplementedError('Moved on mail.render.mixin')
 
     def _convert_links_text(self, body, vals, blacklist=None):
-        shortened_schema = self.env['ir.config_parameter'].sudo().get_param('web.base.url') + '/r/'
-        unsubscribe_schema = self.env['ir.config_parameter'].sudo().get_param('web.base.url') + '/sms/'
-        for original_url in re.findall(TEXT_URL_REGEX, body):
-            # don't shorten already-shortened links or links towards unsubscribe page
-            if original_url.startswith(shortened_schema) or original_url.startswith(unsubscribe_schema):
-                continue
-            # support blacklist items in path, like /u/
-            parsed = urls.url_parse(original_url, scheme='http')
-            if blacklist and any(item in parsed.path for item in blacklist):
-                continue
-
-            vals['url'] = utils.unescape(original_url)
-            link = self.create(vals)
-            shortened_url = link.short_url
-            if shortened_url:
-                body = body.replace(original_url, shortened_url, 1)
-
-        return body
+        raise NotImplementedError('Moved on mail.render.mixin')
 
     def action_view_statistics(self):
         action = self.env['ir.actions.act_window'].for_xml_id('link_tracker', 'link_tracker_click_action_statistics')
diff --git a/addons/link_tracker/models/mail_render_mixin.py b/addons/link_tracker/models/mail_render_mixin.py
new file mode 100644
index 00000000000..0bc30de960b
--- /dev/null
+++ b/addons/link_tracker/models/mail_render_mixin.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+import re
+
+from werkzeug import urls, utils
+
+from odoo import api, models, tools
+
+
+class MailRenderMixin(models.AbstractModel):
+    _inherit = "mail.render.mixin"
+
+    # ------------------------------------------------------------
+    # TOOLS
+    # ------------------------------------------------------------
+
+    @api.model
+    def _shorten_links(self, html, link_tracker_vals, blacklist=None, base_url=None):
+        """
+
+        :param link_tracker_vals: values given to the created link.tracker, containing
+          for example: campaign_id, medium_id, source_id, and any other relevant fields
+          like mass_mailing_id in mass_mailing;
+
+        :return: update
+        """
+        base_url = base_url or self.env['ir.config_parameter'].sudo().get_param('web.base.url')
+        short_schema = base_url + '/r/'
+        for match in re.findall(tools.URL_REGEX, html):
+            href = match[0]
+            long_url = match[1]
+
+            if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema):
+                create_vals = dict(link_tracker_vals, url=utils.unescape(long_url))
+                link = self.env['link.tracker'].create(create_vals)
+                if link.short_url:
+                    new_href = href.replace(long_url, link.short_url)
+                    html = html.replace(href, new_href)
+
+        return html
+
+    @api.model
+    def _shorten_links_text(self, html, link_tracker_vals, blacklist=None, base_url=None):
+        base_url = base_url or self.env['ir.config_parameter'].sudo().get_param('web.base.url')
+        shortened_schema = base_url + '/r/'
+        unsubscribe_schema = base_url + '/sms/'
+        for original_url in re.findall(tools.TEXT_URL_REGEX, html):
+            # don't shorten already-shortened links or links towards unsubscribe page
+            if original_url.startswith(shortened_schema) or original_url.startswith(unsubscribe_schema):
+                continue
+            # support blacklist items in path, like /u/
+            parsed = urls.url_parse(original_url, scheme='http')
+            if blacklist and any(item in parsed.path for item in blacklist):
+                continue
+
+            create_vals = dict(link_tracker_vals, url= utils.unescape(original_url))
+            link = self.env['link.tracker'].create(create_vals)
+            if link.short_url:
+                html = html.replace(original_url, link.short_url, 1)
+
+        return html
diff --git a/addons/link_tracker/models/utm.py b/addons/link_tracker/models/utm.py
index ebb343472e7..75eff850150 100644
--- a/addons/link_tracker/models/utm.py
+++ b/addons/link_tracker/models/utm.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-from odoo import fields, models, api, SUPERUSER_ID
+from odoo import fields, models
 
 
 class UtmCampaign(models.Model):
diff --git a/addons/mass_mailing/models/mail_mail.py b/addons/mass_mailing/models/mail_mail.py
index 68a653a38db..d35ae72f4be 100644
--- a/addons/mass_mailing/models/mail_mail.py
+++ b/addons/mass_mailing/models/mail_mail.py
@@ -6,8 +6,6 @@ import werkzeug.urls
 
 from odoo import api, fields, models, tools
 
-from odoo.addons.link_tracker.models.link_tracker import URL_REGEX
-
 
 class MailMail(models.Model):
     """Add the mass mailing campaign data to mail"""
@@ -54,7 +52,7 @@ class MailMail(models.Model):
         body = super(MailMail, self)._send_prepare_body()
 
         if self.mailing_id and body and self.mailing_trace_ids:
-            for match in re.findall(URL_REGEX, self.body_html):
+            for match in re.findall(tools.URL_REGEX, self.body_html):
                 href = match[0]
                 url = match[1]
 
diff --git a/addons/mass_mailing/models/mail_render_mixin.py b/addons/mass_mailing/models/mail_render_mixin.py
index d38667366ed..4447f3931c8 100644
--- a/addons/mass_mailing/models/mail_render_mixin.py
+++ b/addons/mass_mailing/models/mail_render_mixin.py
@@ -15,7 +15,7 @@ class MailRenderMixin(models.AbstractModel):
         # apply shortener after
         if self.env.context.get('post_convert_links'):
             for res_id, html in rendered.items():
-                rendered[res_id] = self.env['link.tracker'].convert_links(
+                rendered[res_id] = self._shorten_links(
                     html,
                     self.env.context['post_convert_links'],
                     blacklist=['/unsubscribe_from_list']
diff --git a/addons/mass_mailing/models/mailing.py b/addons/mass_mailing/models/mailing.py
index 88c5a9c4574..1ce47e6cfa4 100644
--- a/addons/mass_mailing/models/mailing.py
+++ b/addons/mass_mailing/models/mailing.py
@@ -561,7 +561,7 @@ class MassMailing(models.Model):
             if mass_mailing.medium_id:
                 vals['medium_id'] = mass_mailing.medium_id.id
 
-            res[mass_mailing.id] = self.env['link.tracker'].convert_links(html, vals, blacklist=['/unsubscribe_from_list'])
+            res[mass_mailing.id] = self._shorten_links(html, vals, blacklist=['/unsubscribe_from_list'])
 
         return res
 
diff --git a/addons/mass_mailing_sms/models/sms_sms.py b/addons/mass_mailing_sms/models/sms_sms.py
index d83359aa47a..c9c14e144cd 100644
--- a/addons/mass_mailing_sms/models/sms_sms.py
+++ b/addons/mass_mailing_sms/models/sms_sms.py
@@ -3,9 +3,7 @@
 
 import re
 
-from odoo import fields, models
-
-TEXT_URL_REGEX = r'https?://[a-zA-Z0-9@:%._+~#=/-]+'
+from odoo import fields, models, tools
 
 
 class SmsSms(models.Model):
@@ -25,7 +23,7 @@ class SmsSms(models.Model):
                 continue
 
             body = sms.body
-            for url in re.findall(TEXT_URL_REGEX, body):
+            for url in re.findall(tools.TEXT_URL_REGEX, body):
                 if url.startswith(shortened_schema):
                     body = body.replace(url, url + '/s/%s' % sms.id)
             res[sms.id] = body
diff --git a/addons/mass_mailing_sms/wizard/sms_composer.py b/addons/mass_mailing_sms/wizard/sms_composer.py
index fefbeeb0137..b3e50234036 100644
--- a/addons/mass_mailing_sms/wizard/sms_composer.py
+++ b/addons/mass_mailing_sms/wizard/sms_composer.py
@@ -70,7 +70,7 @@ class SMSComposer(models.TransientModel):
         if self.mailing_id:
             tracker_values = self.mailing_id._get_link_tracker_values()
             for sms_id, body in all_bodies.items():
-                body = self.env['link.tracker'].sudo()._convert_links_text(body, tracker_values)
+                body = self.env['mail.render.mixin'].sudo()._shorten_links_text(body, tracker_values)
                 all_bodies[sms_id] = body
         return all_bodies
 
diff --git a/addons/test_mail_full/tests/test_sms_sms.py b/addons/test_mail_full/tests/test_sms_sms.py
index de7e37ac641..e404eaec572 100644
--- a/addons/test_mail_full/tests/test_sms_sms.py
+++ b/addons/test_mail_full/tests/test_sms_sms.py
@@ -64,7 +64,7 @@ class TestSMSPost(test_mail_full_common.TestSMSCommon, LinkTrackerMock):
     def test_body_link_shorten(self):
         link = 'http://www.example.com'
         self.env['link.tracker'].search([('url', '=', link)]).unlink()
-        new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values)
+        new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values)
         self.assertNotIn(link, new_body)
         self.assertLinkTracker(link, {'utm_campaign': self.utm_c.name, 'utm_medium': self.utm_m.name})
         link = self.env['link.tracker'].search([('url', '=', link)])
@@ -72,7 +72,7 @@ class TestSMSPost(test_mail_full_common.TestSMSCommon, LinkTrackerMock):
 
         link = 'https://test.odoo.com/my/super_page?test[0]=42&toto=áâà#title3'
         self.env['link.tracker'].search([('url', '=', link)]).unlink()
-        new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values)
+        new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values)
         self.assertNotIn(link, new_body)
         self.assertLinkTracker(link, {
             'utm_campaign': self.utm_c.name,
@@ -86,14 +86,14 @@ class TestSMSPost(test_mail_full_common.TestSMSCommon, LinkTrackerMock):
     def test_body_link_shorten_wshort(self):
         link = 'https://test.odoo.com/r/RAOUL'
         self.env['link.tracker'].search([('url', '=', link)]).unlink()
-        new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values)
+        new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values)
         self.assertIn(link, new_body)
         self.assertFalse(self.env['link.tracker'].search([('url', '=', link)]))
 
     def test_body_link_shorten_wunsubscribe(self):
         link = 'https://test.odoo.com/sms/3/'
         self.env['link.tracker'].search([('url', '=', link)]).unlink()
-        new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values)
+        new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values)
         self.assertIn(link, new_body)
         self.assertFalse(self.env['link.tracker'].search([('url', '=', link)]))
 
diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py
index 21756fe0640..d1bc961e8ff 100644
--- a/odoo/tools/mail.py
+++ b/odoo/tools/mail.py
@@ -13,6 +13,7 @@ import time
 
 from email.utils import getaddresses
 from lxml import etree
+from werkzeug import urls
 
 import odoo
 from odoo.loglevels import ustr
@@ -250,9 +251,19 @@ def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=Fals
 
     return cleaned
 
-#----------------------------------------------------------
+# ----------------------------------------------------------
 # HTML/Text management
-#----------------------------------------------------------
+# ----------------------------------------------------------
+
+URL_REGEX = r'(\bhref=[\'"](?!mailto:|tel:|sms:)([^\'"]+)[\'"])'
+TEXT_URL_REGEX = r'https?://[a-zA-Z0-9@:%._\+~#=/-]+(?:\?\S+)?'
+
+
+def validate_url(url):
+    if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
+        return 'http://' + url
+
+    return url
 
 
 def is_html_empty(html_content):
@@ -281,6 +292,7 @@ def html_keep_url(text):
     final += text[idx:]
     return final
 
+
 def html2plaintext(html, body_id=None, encoding='utf-8'):
     """ From an HTML text, convert the HTML to plain text.
     If @param body_id is provided then this is the tag where the