initial commit 2

9 files changed, 950 insertions, 0 deletions

diff --git a/addons/test_mass_mailing/tests/__init__.py b/addons/test_mass_mailing/tests/__init__.py
new file mode 100644
index 00000000..978ceb47
--- /dev/null
+++ b/addons/test_mass_mailing/tests/__init__.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from . import test_blacklist
+from . import test_blacklist_behavior
+from . import test_blacklist_mixin
+from . import test_link_tracker
+from . import test_mailing_test
+from . import test_mailing
+from . import test_performance
diff --git a/addons/test_mass_mailing/tests/common.py b/addons/test_mass_mailing/tests/common.py
new file mode 100644
index 00000000..ff368bf5
--- /dev/null
+++ b/addons/test_mass_mailing/tests/common.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.addons.mass_mailing.tests.common import MassMailCommon
+from odoo.addons.test_mail.tests.common import TestMailCommon
+
+
+class TestMassMailCommon(MassMailCommon, TestMailCommon):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestMassMailCommon, cls).setUpClass()
+
+        cls.test_alias = cls.env['mail.alias'].create({
+            'alias_name': 'test.alias',
+            'alias_user_id': False,
+            'alias_model_id': cls.env['ir.model']._get('mailing.test.simple').id,
+            'alias_contact': 'everyone'
+        })
+
+        # enforce last update by user_marketing to match _process_mass_mailing_queue
+        # taking last writer as user running a batch
+        cls.mailing_bl = cls.env['mailing.mailing'].with_user(cls.user_marketing).create({
+            'name': 'SourceName',
+            'subject': 'MailingSubject',
+            'preview': 'Hi ${object.name} :)',
+            'body_html': """<div><p>Hello ${object.name}</p>,
+% set url = "www.odoo.com"
+% set httpurl = "https://www.odoo.eu"
+<span>Website0: <a id="url0" href="https://www.odoo.tz/my/${object.name}">https://www.odoo.tz/my/${object.name}</a></span>
+<span>Website1: <a id="url1" href="https://www.odoo.be">https://www.odoo.be</a></span>
+<span>Website2: <a id="url2" href="https://${url}">https://${url}</a></span>
+<span>Website3: <a id="url3" href="${httpurl}">${httpurl}</a></span>
+<span>External1: <a id="url4" href="https://www.example.com/foo/bar?baz=qux">Youpie</a></span>
+<span>Internal1: <a id="url5" href="/event/dummy-event-0">Internal link</a></span>
+<span>Internal2: <a id="url6" href="/view"/>View link</a></span>
+<span>Email: <a id="url7" href="mailto:test@odoo.com">test@odoo.com</a></span>
+<p>Stop spam ? <a id="url8" role="button" href="/unsubscribe_from_list">Ok</a></p>
+</div>""",
+            'mailing_type': 'mail',
+            'mailing_model_id': cls.env['ir.model']._get('mailing.test.blacklist').id,
+            'reply_to_mode': 'thread',
+        })
+
+    @classmethod
+    def _create_test_blacklist_records(cls, model='mailing.test.blacklist', count=1):
+        """ Deprecated, remove in 14.4 """
+        return cls.__create_mailing_test_records(model=model, count=count)
+
+    @classmethod
+    def _create_mailing_test_records(cls, model='mailing.test.blacklist', partners=None, count=1):
+        """ Helper to create data. Currently simple, to be improved. """
+        Model = cls.env[model]
+        email_field = 'email' if 'email' in Model else 'email_from'
+        partner_field = 'customer_id' if 'customer_id' in Model else 'partner_id'
+
+        vals_list = []
+        for x in range(0, count):
+            vals = {
+                'name': 'TestRecord_%02d' % x,
+                email_field: '"TestCustomer %02d" <test.record.%02d@test.example.com>' % (x, x),
+            }
+            if partners:
+                vals[partner_field] = partners[x % len(partners)]
+
+            vals_list.append(vals)
+
+        return cls.env[model].create(vals_list)
diff --git a/addons/test_mass_mailing/tests/test_blacklist.py b/addons/test_mass_mailing/tests/test_blacklist.py
new file mode 100644
index 00000000..c00a752c
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_blacklist.py
@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.tests.common import users
+from odoo.addons.test_mass_mailing.tests import common
+from odoo.exceptions import AccessError
+
+
+class TestBLAccessRights(common.TestMassMailCommon):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestBLAccessRights, cls).setUpClass()
+        cls._create_portal_user()
+
+        cls.bl_rec = cls.env['mail.blacklist'].create([
+            {'email': 'Not A Stark <john.snow@example.com>'},
+        ])
+        cls.bl_previous = cls.env['mail.blacklist'].search([])
+
+    @users('employee')
+    def test_bl_crud_employee(self):
+        with self.assertRaises(AccessError):
+            self.env['mail.blacklist'].create([{'email': 'Arya.Stark@example.com'}])
+
+        with self.assertRaises(AccessError):
+            self.bl_rec.with_user(self.env.user).read([])
+
+        with self.assertRaises(AccessError):
+            self.bl_rec.with_user(self.env.user).write({'email': 'jaimie.lannister@example.com'})
+
+        with self.assertRaises(AccessError):
+            self.bl_rec.with_user(self.env.user).unlink()
+
+    @users('portal_test')
+    def test_bl_crud_portal(self):
+        with self.assertRaises(AccessError):
+            self.env['mail.blacklist'].create([{'email': 'Arya.Stark@example.com'}])
+
+        with self.assertRaises(AccessError):
+            self.bl_rec.with_user(self.env.user).read([])
+
+        with self.assertRaises(AccessError):
+            self.bl_rec.with_user(self.env.user).write({'email': 'jaimie.lannister@example.com'})
+
+        with self.assertRaises(AccessError):
+            self.bl_rec.with_user(self.env.user).unlink()
+
+    @users('user_marketing')
+    def test_bl_crud_marketing(self):
+        self.env['mail.blacklist'].create([{'email': 'Arya.Stark@example.com'}])
+
+        read_res = self.bl_rec.with_user(self.env.user).read([])
+        self.assertEqual(read_res[0]['id'], self.bl_rec.id)
+
+        self.bl_rec.with_user(self.env.user).write({'email': 'jaimie.lannister@example.com'})
+        self.assertEqual(self.bl_rec.email, 'jaimie.lannister@example.com')
+
+        self.bl_rec.with_user(self.env.user).unlink()
+
+
+class TestBLConsistency(common.TestMassMailCommon):
+    _base_list = ['Arya.Stark@example.com', 'ned.stark@example.com']
+
+    def setUp(self):
+        super(TestBLConsistency, self).setUp()
+        self.bl_rec = self.env['mail.blacklist'].create([
+            {'email': 'Not A Stark <john.snow@example.com>'},
+        ])
+
+        self.bl_previous = self.env['mail.blacklist'].search([])
+
+    @users('user_marketing')
+    def test_bl_check_case_add(self):
+        """ Test emails case when adding through _add """
+        bl_sudo = self.env['mail.blacklist'].sudo()
+        existing = bl_sudo.create({
+            'email': 'arya.stark@example.com',
+            'active': False,
+        })
+
+        added = self.env['mail.blacklist']._add('Arya.Stark@EXAMPLE.com')
+        self.assertEqual(existing, added)
+        self.assertTrue(existing.active)
+
+    @users('user_marketing')
+    def test_bl_check_case_remove(self):
+        """ Test emails case when deactivating through _remove """
+        bl_sudo = self.env['mail.blacklist'].sudo()
+        existing = bl_sudo.create({
+            'email': 'arya.stark@example.com',
+            'active': True,
+        })
+
+        added = self.env['mail.blacklist']._remove('Arya.Stark@EXAMPLE.com')
+        self.assertEqual(existing, added)
+        self.assertFalse(existing.active)
+
+    @users('user_marketing')
+    def test_bl_create_duplicate(self):
+        """ Test emails are inserted only once if duplicated """
+        bl_sudo = self.env['mail.blacklist'].sudo()
+        self.env['mail.blacklist'].create([
+            {'email': self._base_list[0]},
+            {'email': self._base_list[1]},
+            {'email': 'Another Ned Stark <%s>' % self._base_list[1]},
+        ])
+
+        new_bl = bl_sudo.search([('id', 'not in', self.bl_previous.ids)])
+
+        self.assertEqual(len(new_bl), 2)
+        self.assertEqual(
+            set(v.lower() for v in self._base_list),
+            set(v.lower() for v in new_bl.mapped('email'))
+        )
+
+    @users('user_marketing')
+    def test_bl_create_parsing(self):
+        """ Test email is correctly extracted from given entries """
+        bl_sudo = self.env['mail.blacklist'].sudo()
+        self.env['mail.blacklist'].create([
+            {'email': self._base_list[0]},
+            {'email': self._base_list[1]},
+            {'email': 'Not Ned Stark <jaimie.lannister@example.com>'},
+        ])
+
+        new_bl = bl_sudo.search([('id', 'not in', self.bl_previous.ids)])
+
+        self.assertEqual(len(new_bl), 3)
+        self.assertEqual(
+            set(v.lower() for v in self._base_list + ['jaimie.lannister@example.com']),
+            set(v.lower() for v in new_bl.mapped('email'))
+        )
+
+    @users('user_marketing')
+    def test_bl_search_exact(self):
+        search_res = self.env['mail.blacklist'].search([('email', '=', 'john.snow@example.com')])
+        self.assertEqual(search_res, self.bl_rec)
+
+    @users('user_marketing')
+    def test_bl_search_parsing(self):
+        search_res = self.env['mail.blacklist'].search([('email', '=', 'Not A Stark <john.snow@example.com>')])
+
+        self.assertEqual(search_res, self.bl_rec)
+
+        search_res = self.env['mail.blacklist'].search([('email', '=', '"John J. Snow" <john.snow@example.com>')])
+        self.assertEqual(search_res, self.bl_rec)
+
+        search_res = self.env['mail.blacklist'].search([('email', '=', 'Aegon? <john.snow@example.com>')])
+        self.assertEqual(search_res, self.bl_rec)
+
+        search_res = self.env['mail.blacklist'].search([('email', '=', '"John; \"You know Nothing\" Snow" <john.snow@example.com>')])
+        self.assertEqual(search_res, self.bl_rec)
+
+    @users('user_marketing')
+    def test_bl_search_case(self):
+        search_res = self.env['mail.blacklist'].search([('email', '=', 'john.SNOW@example.COM>')])
+        self.assertEqual(search_res, self.bl_rec)
+
+    @users('user_marketing')
+    def test_bl_search_partial(self):
+        search_res = self.env['mail.blacklist'].search([('email', 'ilike', 'John')])
+        self.assertEqual(search_res, self.bl_rec)
+        search_res = self.env['mail.blacklist'].search([('email', 'ilike', 'n.SNOW@example.cO>')])
+        self.assertEqual(search_res, self.bl_rec)
diff --git a/addons/test_mass_mailing/tests/test_blacklist_behavior.py b/addons/test_mass_mailing/tests/test_blacklist_behavior.py
new file mode 100644
index 00000000..a5a77c7d
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_blacklist_behavior.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+import datetime
+
+from odoo.addons.test_mass_mailing.tests import common
+from odoo.tests.common import users
+from odoo.addons.mass_mailing.models.mail_thread import BLACKLIST_MAX_BOUNCED_LIMIT
+
+
+class TestAutoBlacklist(common.TestMassMailCommon):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestAutoBlacklist, cls).setUpClass()
+        cls.target_rec = cls._create_mailing_test_records()[0]
+        cls.mailing_bl.write({'mailing_domain': [('id', 'in', cls.target_rec.ids)]})
+
+    @users('user_marketing')
+    def test_mailing_bounce_w_auto_bl(self):
+        self._test_mailing_bounce_w_auto_bl(None)
+
+    @users('user_marketing')
+    def test_mailing_bounce_w_auto_bl_partner(self):
+        bounced_partner = self.env['res.partner'].sudo().create({
+            'name': 'Bounced Partner',
+            'email': self.target_rec.email_from,
+            'message_bounce': BLACKLIST_MAX_BOUNCED_LIMIT,
+        })
+        self._test_mailing_bounce_w_auto_bl({'bounced_partner': bounced_partner})
+
+    @users('user_marketing')
+    def test_mailing_bounce_w_auto_bl_partner_duplicates(self):
+        bounced_partners = self.env['res.partner'].sudo().create({
+            'name': 'Bounced Partner1',
+            'email': self.target_rec.email_from,
+            'message_bounce': BLACKLIST_MAX_BOUNCED_LIMIT,
+        }) | self.env['res.partner'].sudo().create({
+            'name': 'Bounced Partner2',
+            'email': self.target_rec.email_from,
+            'message_bounce': BLACKLIST_MAX_BOUNCED_LIMIT,
+        })
+        self._test_mailing_bounce_w_auto_bl({'bounced_partner': bounced_partners})
+
+    def _test_mailing_bounce_w_auto_bl(self, bounce_base_values):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        target = self.env['mailing.test.blacklist'].browse(self.target_rec.ids)
+
+        # create bounced history of 4 statistics
+        for idx in range(4):
+            new_mailing = mailing.copy()
+            self._create_bounce_trace(new_mailing, target, dt=datetime.datetime.now() - datetime.timedelta(weeks=idx+2))
+            self.gateway_mail_bounce(new_mailing, target, bounce_base_values)
+
+        # mass mail record: ok, not blacklisted yet
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        self.assertMailTraces(
+            [{'email': 'test.record.00@test.example.com'}],
+            mailing, target,
+            check_mail=True
+        )
+
+        # call bounced
+        self.gateway_mail_bounce(mailing, target, bounce_base_values)
+
+        # check blacklist
+        blacklist_record = self.env['mail.blacklist'].sudo().search([('email', '=', target.email_normalized)])
+        self.assertEqual(len(blacklist_record), 1)
+        self.assertTrue(target.is_blacklisted)
+
+        # mass mail record: ko, blacklisted
+        new_mailing = mailing.copy()
+        new_mailing.write({'mailing_domain': [('id', 'in', target.ids)]})
+        new_mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            new_mailing._process_mass_mailing_queue()
+        self.assertMailTraces(
+            [{'email': 'test.record.00@test.example.com', 'state': 'ignored'}],
+            new_mailing, target, check_mail=True
+        )
diff --git a/addons/test_mass_mailing/tests/test_blacklist_mixin.py b/addons/test_mass_mailing/tests/test_blacklist_mixin.py
new file mode 100644
index 00000000..57504fb5
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_blacklist_mixin.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.addons.test_mass_mailing.models.mailing_models import MailingBLacklist
+from odoo.addons.test_mass_mailing.tests import common
+from odoo.exceptions import UserError
+from odoo.tests.common import users
+
+
+class TestBLMixin(common.TestMassMailCommon):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestBLMixin, cls).setUpClass()
+
+        cls.env['mail.blacklist'].create([{
+            'email': 'Arya.Stark@example.com',
+            'active': True,
+        }, {
+            'email': 'Sansa.Stark@example.com',
+            'active': False,
+        }])
+
+    @users('employee')
+    def test_bl_mixin_primary_field_consistency(self):
+        MailingBLacklist._primary_email = 'not_a_field'
+        with self.assertRaises(UserError):
+            self.env['mailing.test.blacklist'].search([('is_blacklisted', '=', False)])
+
+        MailingBLacklist._primary_email = ['not_a_str']
+        with self.assertRaises(UserError):
+            self.env['mailing.test.blacklist'].search([('is_blacklisted', '=', False)])
+
+        MailingBLacklist._primary_email = 'email_from'
+        self.env['mailing.test.blacklist'].search([('is_blacklisted', '=', False)])
+
+    @users('employee')
+    def test_bl_mixin_is_blacklisted(self):
+        """ Test is_blacklisted field computation """
+        record = self.env['mailing.test.blacklist'].create({'email_from': 'arya.stark@example.com'})
+        self.assertTrue(record.is_blacklisted)
+
+        record = self.env['mailing.test.blacklist'].create({'email_from': 'not.arya.stark@example.com'})
+        self.assertFalse(record.is_blacklisted)
+
+    @users('employee')
+    def test_bl_mixin_search_blacklisted(self):
+        """ Test is_blacklisted field search implementation """
+        record1 = self.env['mailing.test.blacklist'].create({'email_from': 'arya.stark@example.com'})
+        record2 = self.env['mailing.test.blacklist'].create({'email_from': 'not.arya.stark@example.com'})
+
+        search_res = self.env['mailing.test.blacklist'].search([('is_blacklisted', '=', False)])
+        self.assertEqual(search_res, record2)
+
+        search_res = self.env['mailing.test.blacklist'].search([('is_blacklisted', '!=', True)])
+        self.assertEqual(search_res, record2)
+
+        search_res = self.env['mailing.test.blacklist'].search([('is_blacklisted', '=', True)])
+        self.assertEqual(search_res, record1)
+
+        search_res = self.env['mailing.test.blacklist'].search([('is_blacklisted', '!=', False)])
+        self.assertEqual(search_res, record1)
+
+    @users('employee')
+    def test_bl_mixin_search_blacklisted_format(self):
+        """ Test is_blacklisted field search using email parsing """
+        record1 = self.env['mailing.test.blacklist'].create({'email_from': 'Arya Stark <arya.stark@example.com>'})
+        self.assertTrue(record1.is_blacklisted)
+
+        search_res = self.env['mailing.test.blacklist'].search([('is_blacklisted', '=', True)])
+        self.assertEqual(search_res, record1)
diff --git a/addons/test_mass_mailing/tests/test_link_tracker.py b/addons/test_mass_mailing/tests/test_link_tracker.py
new file mode 100644
index 00000000..df2a23ae
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_link_tracker.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.tests.common import users
+from odoo.addons.test_mass_mailing.tests import common
+
+
+class TestLinkTracker(common.TestMassMailCommon):
+
+    def setUp(self):
+        super(TestLinkTracker, self).setUp()
+
+        self.link = self.env['link.tracker'].create({
+            'url': 'https://www.example.com'
+        })
+
+        self.click = self.env['link.tracker.click'].create({
+            'link_id': self.link.id,
+            'ip': '100.00.00.00',
+            'country_id': self.env.ref('base.fr').id,
+        })
+
+    def test_add_link(self):
+        code = self.link.code
+        self.assertEqual(self.link.count, 1)
+
+        # click from a new IP should create a new entry
+        click = self.env['link.tracker.click'].sudo().add_click(
+            code,
+            ip='100.00.00.01',
+            country_code='BEL'
+        )
+        self.assertEqual(click.ip, '100.00.00.01')
+        self.assertEqual(click.country_id, self.env.ref('base.be'))
+        self.assertEqual(self.link.count, 2)
+
+        # click from same IP (even another country) does not create a new entry
+        click = self.env['link.tracker.click'].sudo().add_click(
+            code,
+            ip='100.00.00.01',
+            country_code='FRA'
+        )
+        self.assertEqual(click, None)
+        self.assertEqual(self.link.count, 2)
+
+    @users('user_marketing')
+    def test_add_link_mail_stat(self):
+        mailing = self.env['mailing.mailing'].create({'name': 'Test Mailing', "subject": "Hi!"})
+        code = self.link.code
+        self.assertEqual(self.link.count, 1)
+        stat = self.env['mailing.trace'].create({'mass_mailing_id': mailing.id})
+        self.assertFalse(stat.opened)
+        self.assertFalse(stat.clicked)
+
+        # click from a new IP should create a new entry and update stat when provided
+        click = self.env['link.tracker.click'].sudo().add_click(
+            code,
+            ip='100.00.00.01',
+            country_code='BEL',
+            mailing_trace_id=stat.id
+        )
+        self.assertEqual(self.link.count, 2)
+        self.assertEqual(click.mass_mailing_id, mailing)
+        self.assertTrue(stat.opened)
+        self.assertTrue(stat.clicked)
diff --git a/addons/test_mass_mailing/tests/test_mailing.py b/addons/test_mass_mailing/tests/test_mailing.py
new file mode 100644
index 00000000..4f6fa34f
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_mailing.py
@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.addons.test_mass_mailing.data.mail_test_data import MAIL_TEMPLATE
+from odoo.addons.test_mass_mailing.tests.common import TestMassMailCommon
+from odoo.tests import tagged
+from odoo.tests.common import users
+from odoo.tools import mute_logger
+
+
+@tagged('mass_mailing')
+class TestMassMailing(TestMassMailCommon):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestMassMailing, cls).setUpClass()
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_thread')
+    def test_mailing_gateway_reply(self):
+        customers = self.env['res.partner']
+        for x in range(0, 3):
+            customers |= self.env['res.partner'].create({
+                'name': 'Customer_%02d' % x,
+                'email': '"Customer_%02d" <customer_%02d@test.example.com' % (x, x),
+            })
+
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'TestName',
+            'subject': 'TestSubject',
+            'body_html': 'Hello ${object.name}',
+            'reply_to_mode': 'email',
+            'reply_to': '%s@%s' % (self.test_alias.alias_name, self.test_alias.alias_domain),
+            'keep_archives': True,
+            'mailing_model_id': self.env['ir.model']._get('res.partner').id,
+            'mailing_domain': '%s' % [('id', 'in', customers.ids)],
+        })
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing.action_send_mail()
+
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, customers[0], use_in_reply_to=True)
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, customers[1], use_in_reply_to=False)
+
+        # customer2 looses headers
+        mail_mail = self._find_mail_mail_wrecord(customers[2])
+        self.format_and_process(
+            MAIL_TEMPLATE,
+            mail_mail.email_to,
+            mail_mail.reply_to,
+            subject='Re: %s' % mail_mail.subject,
+            extra='',
+            msg_id='<123456.%s.%d@test.example.com>' % (customers[2]._name, customers[2].id),
+            target_model=customers[2]._name, target_field=customers[2]._rec_name,
+        )
+        mailing.flush()
+
+        # check traces status
+        traces = self.env['mailing.trace'].search([('model', '=', customers._name), ('res_id', 'in', customers.ids)])
+        self.assertEqual(len(traces), 3)
+        customer0_trace = traces.filtered(lambda t: t.res_id == customers[0].id)
+        self.assertEqual(customer0_trace.state, 'replied')
+        customer1_trace = traces.filtered(lambda t: t.res_id == customers[1].id)
+        self.assertEqual(customer1_trace.state, 'replied')
+        customer2_trace = traces.filtered(lambda t: t.res_id == customers[2].id)
+        self.assertEqual(customer2_trace.state, 'sent')
+
+        # check mailing statistics
+        self.assertEqual(mailing.sent, 3)
+        self.assertEqual(mailing.delivered, 3)
+        self.assertEqual(mailing.opened, 2)
+        self.assertEqual(mailing.replied, 2)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_gateway_update(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(model='mailing.test.optout', count=5)
+        self.assertEqual(len(recipients), 5)
+
+        mailing.write({
+            'mailing_model_id': self.env['ir.model']._get('mailing.test.optout'),
+            'mailing_domain': [('id', 'in', recipients.ids)]
+        })
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        self.assertMailTraces(
+            [{'email': record.email_normalized}
+             for record in recipients],
+            mailing, recipients,
+            mail_links_info=[[
+                ('url0', 'https://www.odoo.tz/my/%s' % record.name, True, {}),
+                ('url1', 'https://www.odoo.be', True, {}),
+                ('url2', 'https://www.odoo.com', True, {}),
+                ('url3', 'https://www.odoo.eu', True, {}),
+                ('url4', 'https://www.example.com/foo/bar?baz=qux', True, {'baz': 'qux'}),
+                ('url5', '%s/event/dummy-event-0' % mailing.get_base_url(), True, {}),
+                # view is not shortened and parsed at sending
+                ('url6', '%s/view' % mailing.get_base_url(), False, {}),
+                ('url7', 'mailto:test@odoo.com', False, {}),
+                # unsubscribe is not shortened and parsed at sending
+                ('url8', '%s/unsubscribe_from_list' % mailing.get_base_url(), False, {}),
+            ] for record in recipients],
+            check_mail=True
+        )
+        self.assertMailingStatistics(mailing, expected=5, delivered=5, sent=5)
+
+        # simulate a click
+        self.gateway_mail_click(mailing, recipients[0], 'https://www.odoo.be')
+        mailing.invalidate_cache()
+        self.assertMailingStatistics(mailing, expected=5, delivered=5, sent=5, opened=1, clicked=1)
+
+        # simulate a bounce
+        self.assertEqual(recipients[1].message_bounce, 0)
+        self.gateway_mail_bounce(mailing, recipients[1])
+        mailing.invalidate_cache()
+        self.assertMailingStatistics(mailing, expected=5, delivered=4, sent=5, opened=1, clicked=1, bounced=1)
+        self.assertEqual(recipients[1].message_bounce, 1)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_reply_to_mode_new(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(model='mailing.test.blacklist', count=5)
+        self.assertEqual(len(recipients), 5)
+        initial_messages = recipients.message_ids
+        mailing.write({
+            'mailing_domain': [('id', 'in', recipients.ids)],
+            'keep_archives': False,
+            'reply_to_mode': 'email',
+            'reply_to': self.test_alias.display_name,
+        })
+
+        with self.mock_mail_gateway(mail_unlink_sent=True):
+            mailing.action_send_mail()
+
+        answer_rec = self.gateway_mail_reply_wemail(MAIL_TEMPLATE, recipients[0].email_normalized, target_model=self.test_alias.alias_model_id.model)
+        self.assertTrue(bool(answer_rec))
+        self.assertEqual(answer_rec.name, 'Re: %s' % mailing.subject)
+        self.assertEqual(
+            answer_rec.message_ids.subject, 'Re: %s' % mailing.subject,
+            'Answer should be logged')
+        self.assertEqual(recipients.message_ids, initial_messages)
+
+        self.assertMailingStatistics(mailing, expected=5, delivered=5, sent=5, opened=1, replied=1)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_reply_to_mode_update(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(model='mailing.test.blacklist', count=5)
+        self.assertEqual(len(recipients), 5)
+        mailing.write({
+            'mailing_domain': [('id', 'in', recipients.ids)],
+            'keep_archives': False,
+            'reply_to_mode': 'thread',
+            'reply_to': self.test_alias.display_name,
+        })
+
+        with self.mock_mail_gateway(mail_unlink_sent=True):
+            mailing.action_send_mail()
+
+        answer_rec = self.gateway_mail_reply_wemail(MAIL_TEMPLATE, recipients[0].email_normalized, target_model=self.test_alias.alias_model_id.model)
+        self.assertFalse(bool(answer_rec))
+        self.assertEqual(
+            recipients[0].message_ids[1].subject, mailing.subject,
+            'Should have keep a log (to enable thread-based answer)')
+        self.assertEqual(
+            recipients[0].message_ids[0].subject, 'Re: %s' % mailing.subject,
+            'Answer should be logged')
+
+        self.assertMailingStatistics(mailing, expected=5, delivered=5, sent=5, opened=1, replied=1)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_thread')
+    def test_mailing_trace_utm(self):
+        """ Test mailing UTMs are caught on reply"""
+        self._create_mailing_list()
+        self.test_alias.write({
+            'alias_model_id': self.env['ir.model']._get('mailing.test.utm').id
+        })
+
+        source = self.env['utm.source'].create({'name': 'Source test'})
+        medium = self.env['utm.medium'].create({'name': 'Medium test'})
+        campaign = self.env['utm.campaign'].create({'name': 'Campaign test'})
+        subject = 'MassMailingTestUTM'
+
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'UTMTest',
+            'subject': subject,
+            'body_html': '<p>Hello ${object.name}</p>',
+            'reply_to_mode': 'email',
+            'reply_to': '%s@%s' % (self.test_alias.alias_name, self.test_alias.alias_domain),
+            'keep_archives': True,
+            'mailing_model_id': self.env['ir.model']._get('mailing.list').id,
+            'contact_list_ids': [(4, self.mailing_list_1.id)],
+            'source_id': source.id,
+            'medium_id': medium.id,
+            'campaign_id': campaign.id
+        })
+
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        traces = self.env['mailing.trace'].search([('model', '=', self.mailing_list_1.contact_ids._name), ('res_id', 'in', self.mailing_list_1.contact_ids.ids)])
+        self.assertEqual(len(traces), 3)
+
+        # simulate response to mailing
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, self.mailing_list_1.contact_ids[0], use_in_reply_to=True)
+        self.gateway_mail_reply_wrecord(MAIL_TEMPLATE, self.mailing_list_1.contact_ids[1], use_in_reply_to=False)
+
+        mailing_test_utms = self.env['mailing.test.utm'].search([('name', '=', 'Re: %s' % subject)])
+        self.assertEqual(len(mailing_test_utms), 2)
+        for test_utm in mailing_test_utms:
+            self.assertEqual(test_utm.campaign_id, campaign)
+            self.assertEqual(test_utm.source_id, source)
+            self.assertEqual(test_utm.medium_id, medium)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_w_blacklist(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(count=5)
+
+        # blacklist records 2, 3, 4
+        self.env['mail.blacklist'].create({'email': recipients[2].email_normalized})
+        self.env['mail.blacklist'].create({'email': recipients[3].email_normalized})
+        self.env['mail.blacklist'].create({'email': recipients[4].email_normalized})
+
+        # unblacklist record 2
+        self.env['mail.blacklist'].action_remove_with_reason(
+            recipients[2].email_normalized, "human error"
+        )
+        self.env['mail.blacklist'].flush(['active'])
+
+        mailing.write({'mailing_domain': [('id', 'in', recipients.ids)]})
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        self.assertMailTraces(
+            [{'email': 'test.record.00@test.example.com'},
+             {'email': 'test.record.01@test.example.com'},
+             {'email': 'test.record.02@test.example.com'},
+             {'email': 'test.record.03@test.example.com', 'state': 'ignored'},
+             {'email': 'test.record.04@test.example.com', 'state': 'ignored'}],
+            mailing, recipients, check_mail=True
+        )
+        self.assertEqual(mailing.ignored, 2)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_w_opt_out(self):
+        mailing = self.env['mailing.mailing'].browse(self.mailing_bl.ids)
+        recipients = self._create_mailing_test_records(model='mailing.test.optout', count=5)
+
+        # optout records 0 and 1
+        (recipients[0] | recipients[1]).write({'opt_out': True})
+        # blacklist records 4
+        self.env['mail.blacklist'].create({'email': recipients[4].email_normalized})
+
+        mailing.write({
+            'mailing_model_id': self.env['ir.model']._get('mailing.test.optout'),
+            'mailing_domain': [('id', 'in', recipients.ids)]
+        })
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        self.assertMailTraces(
+            [{'email': 'test.record.00@test.example.com', 'state': 'ignored'},
+             {'email': 'test.record.01@test.example.com', 'state': 'ignored'},
+             {'email': 'test.record.02@test.example.com'},
+             {'email': 'test.record.03@test.example.com'},
+             {'email': 'test.record.04@test.example.com', 'state': 'ignored'}],
+            mailing, recipients, check_mail=True
+        )
+        self.assertEqual(mailing.ignored, 3)
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_mail')
+    def test_mailing_mailing_list_optout(self):
+        """ Test mailing list model specific optout behavior """
+        mailing_contact_1 = self.env['mailing.contact'].create({'name': 'test 1A', 'email': 'test@test.example.com'})
+        mailing_contact_2 = self.env['mailing.contact'].create({'name': 'test 1B', 'email': 'test@test.example.com'})
+        mailing_contact_3 = self.env['mailing.contact'].create({'name': 'test 3', 'email': 'test3@test.example.com'})
+        mailing_contact_4 = self.env['mailing.contact'].create({'name': 'test 4', 'email': 'test4@test.example.com'})
+        mailing_contact_5 = self.env['mailing.contact'].create({'name': 'test 5', 'email': 'test5@test.example.com'})
+
+        # create mailing list record
+        mailing_list_1 = self.env['mailing.list'].create({
+            'name': 'A',
+            'contact_ids': [
+                (4, mailing_contact_1.id),
+                (4, mailing_contact_2.id),
+                (4, mailing_contact_3.id),
+                (4, mailing_contact_5.id),
+            ]
+        })
+        mailing_list_2 = self.env['mailing.list'].create({
+            'name': 'B',
+            'contact_ids': [
+                (4, mailing_contact_3.id),
+                (4, mailing_contact_4.id),
+            ]
+        })
+        # contact_1 is optout but same email is not optout from the same list
+        # contact 3 is optout in list 1 but not in list 2
+        # contact 5 is optout
+        Sub = self.env['mailing.contact.subscription']
+        Sub.search([('contact_id', '=', mailing_contact_1.id), ('list_id', '=', mailing_list_1.id)]).write({'opt_out': True})
+        Sub.search([('contact_id', '=', mailing_contact_3.id), ('list_id', '=', mailing_list_1.id)]).write({'opt_out': True})
+        Sub.search([('contact_id', '=', mailing_contact_5.id), ('list_id', '=', mailing_list_1.id)]).write({'opt_out': True})
+
+        # create mass mailing record
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'SourceName',
+            'subject': 'MailingSubject',
+            'body_html': '<p>Hello ${object.name}</p>',
+            'mailing_model_id': self.env['ir.model']._get('mailing.list').id,
+            'contact_list_ids': [(4, ml.id) for ml in mailing_list_1 | mailing_list_2],
+        })
+        mailing.action_put_in_queue()
+        with self.mock_mail_gateway(mail_unlink_sent=False):
+            mailing._process_mass_mailing_queue()
+
+        self.assertMailTraces(
+            [{'email': 'test@test.example.com', 'state': 'sent'},
+             {'email': 'test@test.example.com', 'state': 'ignored'},
+             {'email': 'test3@test.example.com'},
+             {'email': 'test4@test.example.com'},
+             {'email': 'test5@test.example.com', 'state': 'ignored'}],
+            mailing,
+            mailing_contact_1 + mailing_contact_2 + mailing_contact_3 + mailing_contact_4 + mailing_contact_5,
+            check_mail=True
+        )
+        self.assertEqual(mailing.ignored, 2)
diff --git a/addons/test_mass_mailing/tests/test_mailing_test.py b/addons/test_mass_mailing/tests/test_mailing_test.py
new file mode 100644
index 00000000..33c0419a
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_mailing_test.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.addons.test_mass_mailing.tests.common import TestMassMailCommon
+from odoo.tests.common import users
+from odoo.tools import mute_logger
+
+
+class TestMailingTest(TestMassMailCommon):
+
+    @users('user_marketing')
+    @mute_logger('odoo.addons.mail.models.mail_render_mixin')
+    def test_mailing_test_button(self):
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'TestButton',
+            'subject': 'Subject ${object.name}',
+            'preview': 'Preview ${object.name}',
+            'state': 'draft',
+            'mailing_type': 'mail',
+            'body_html': '<p>Hello ${object.name}</p>',
+            'mailing_model_id': self.env['ir.model']._get('res.partner').id,
+        })
+        mailing_test = self.env['mailing.mailing.test'].create({
+            'email_to': 'test@test.com',
+            'mass_mailing_id': mailing.id,
+        })
+
+        with self.mock_mail_gateway():
+            mailing_test.send_mail_test()
+
+        # Test if bad jinja in the subject raises an error
+        mailing.write({'subject': 'Subject ${object.name_id.id}'})
+        with self.mock_mail_gateway(), self.assertRaises(Exception):
+            mailing_test.send_mail_test()
+
+        # Test if bad jinja in the body raises an error
+        mailing.write({
+            'subject': 'Subject ${object.name}',
+            'body_html': '<p>Hello ${object.name_id.id}</p>',
+        })
+        with self.mock_mail_gateway(), self.assertRaises(Exception):
+            mailing_test.send_mail_test()
+
+        # Test if bad jinja in the preview raises an error
+        mailing.write({
+            'body_html': '<p>Hello ${object.name}</p>',
+            'preview': 'Preview ${object.name_id.id}',
+        })
+        with self.mock_mail_gateway(), self.assertRaises(Exception):
+            mailing_test.send_mail_test()
diff --git a/addons/test_mass_mailing/tests/test_performance.py b/addons/test_mass_mailing/tests/test_performance.py
new file mode 100644
index 00000000..680eacf5
--- /dev/null
+++ b/addons/test_mass_mailing/tests/test_performance.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.addons.mail.tests.common import mail_new_test_user
+from odoo.tests.common import TransactionCase, users, warmup
+from odoo.tests import tagged
+from odoo.tools import mute_logger
+
+
+class TestMassMailPerformanceBase(TransactionCase):
+
+    def setUp(self):
+        super(TestMassMailPerformanceBase, self).setUp()
+
+        self.user_employee = mail_new_test_user(
+            self.env, login='emp',
+            groups='base.group_user',
+            name='Ernest Employee', notification_type='inbox')
+
+        self.user_marketing = mail_new_test_user(
+            self.env, login='marketing',
+            groups='base.group_user,mass_mailing.group_mass_mailing_user',
+            name='Martial Marketing', signature='--\nMartial')
+
+        # patch registry to simulate a ready environment
+        self.patch(self.env.registry, 'ready', True)
+
+
+@tagged('mail_performance')
+class TestMassMailPerformance(TestMassMailPerformanceBase):
+
+    def setUp(self):
+        super(TestMassMailPerformance, self).setUp()
+        values = [{
+            'name': 'Recipient %s' % x,
+            'email_from': 'Recipient <rec.%s@example.com>' % x,
+        } for x in range(0, 50)]
+        self.mm_recs = self.env['mailing.performance'].create(values)
+
+    @users('__system__', 'marketing')
+    @warmup
+    @mute_logger('odoo.addons.mail.models.mail_mail', 'odoo.models.unlink', 'odoo.tests')
+    def test_send_mailing(self):
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'Test',
+            'subject': 'Test',
+            'body_html': '<p>Hello <a role="button" href="https://www.example.com/foo/bar?baz=qux">quux</a><a role="button" href="/unsubscribe_from_list">Unsubscribe</a></p>',
+            'reply_to_mode': 'email',
+            'mailing_model_id': self.ref('test_mass_mailing.model_mailing_performance'),
+            'mailing_domain': [('id', 'in', self.mm_recs.ids)],
+        })
+
+        # runbot needs +51 compared to local
+        with self.assertQueryCount(__system__=1715, marketing=1716):  # test_mass_mailing_only: 1664 - 1665
+            mailing.action_send_mail()
+
+        self.assertEqual(mailing.sent, 50)
+        self.assertEqual(mailing.delivered, 50)
+
+
+@tagged('mail_performance')
+class TestMassMailBlPerformance(TestMassMailPerformanceBase):
+
+    def setUp(self):
+        """ In this setup we prepare 20 blacklist entries. We therefore add
+        20 recipients compared to first test in order to have comparable results. """
+        super(TestMassMailBlPerformance, self).setUp()
+        values = [{
+            'name': 'Recipient %s' % x,
+            'email_from': 'Recipient <rec.%s@example.com>' % x,
+        } for x in range(0, 62)]
+        self.mm_recs = self.env['mailing.performance.blacklist'].create(values)
+
+        for x in range(1, 13):
+            self.env['mail.blacklist'].create({
+                'email': 'rec.%s@example.com' % (x * 5)
+            })
+        self.env['mailing.performance.blacklist'].flush()
+
+    @users('__system__', 'marketing')
+    @warmup
+    @mute_logger('odoo.addons.mail.models.mail_mail', 'odoo.models.unlink', 'odoo.tests')
+    def test_send_mailing_w_bl(self):
+        mailing = self.env['mailing.mailing'].create({
+            'name': 'Test',
+            'subject': 'Test',
+            'body_html': '<p>Hello <a role="button" href="https://www.example.com/foo/bar?baz=qux">quux</a><a role="button" href="/unsubscribe_from_list">Unsubscribe</a></p>',
+            'reply_to_mode': 'email',
+            'mailing_model_id': self.ref('test_mass_mailing.model_mailing_performance_blacklist'),
+            'mailing_domain': [('id', 'in', self.mm_recs.ids)],
+        })
+
+        # runbot needs +63 compared to local
+        with self.assertQueryCount(__system__=1992, marketing=1993):  # test_mass_mailing only: 1929 - 1930
+            mailing.action_send_mail()
+
+        self.assertEqual(mailing.sent, 50)
+        self.assertEqual(mailing.delivered, 50)