From 404d2e8f329dc8a386b7e476b4502e1998b18028 Mon Sep 17 00:00:00 2001 From: Parthiv Patel Date: Sun, 25 Jun 2023 08:34:26 +0000 Subject: [PATCH] [PATCH] Upstream patch - 25062023 --- addons/base_import_module/models/ir_module.py | 95 ++++++++----------- .../tests/test_import_module.py | 20 ++++ addons/mail/models/mail_thread.py | 4 + addons/maintenance/security/maintenance.xml | 2 +- addons/test_mail/data/test_mail_data.py | 44 +++++++++ addons/test_mail/tests/test_mail_gateway.py | 7 ++ .../static/src/js/website_links_charts.js | 27 ++++-- 7 files changed, 137 insertions(+), 62 deletions(-) diff --git a/addons/base_import_module/models/ir_module.py b/addons/base_import_module/models/ir_module.py index 7f3165f0a2..1f1d325979 100644 --- a/addons/base_import_module/models/ir_module.py +++ b/addons/base_import_module/models/ir_module.py @@ -10,25 +10,16 @@ import zipfile from collections import defaultdict from os.path import join as opj -import flectra from flectra import api, fields, models, _ from flectra.exceptions import UserError from flectra.modules.module import MANIFEST_NAMES from flectra.tools import convert_csv_import, convert_sql_import, convert_xml_import, exception_to_unicode +from flectra.tools import file_open, file_open_temporary_directory _logger = logging.getLogger(__name__) MAX_FILE_SIZE = 100 * 1024 * 1024 # in megabytes -__import_paths__ = {} - -def _file_open(env, path): - path = os.path.normcase(os.path.normpath(path)) - import_path = __import_paths__.get(env) - if import_path and path.startswith(import_path): - return open(path, 'rb') - return flectra.tools.file_open(path, 'rb') - class IrModule(models.Model): _inherit = "ir.module.module" @@ -54,7 +45,7 @@ class IrModule(models.Model): terp = {} manifest_path = next((opj(path, name) for name in MANIFEST_NAMES if os.path.exists(opj(path, name))), None) if manifest_path: - with _file_open(self.env, manifest_path) as f: + with file_open(manifest_path, 'rb', env=self.env) as f: terp.update(ast.literal_eval(f.read().decode())) if not terp: return False @@ -101,7 +92,7 @@ class IrModule(models.Model): noupdate = True pathname = opj(path, filename) idref = {} - with _file_open(self.env, pathname) as fp: + with file_open(pathname, 'rb', env=self.env) as fp: if ext == '.csv': convert_csv_import(self.env.cr, module, pathname, fp.read(), idref, mode, noupdate) elif ext == '.sql': @@ -115,7 +106,7 @@ class IrModule(models.Model): for root, dirs, files in os.walk(path_static): for static_file in files: full_path = opj(root, static_file) - with _file_open(self.env, full_path) as fp: + with file_open(full_path, 'rb', env=self.env) as fp: data = base64.b64encode(fp.read()) url_path = '/{}{}'.format(module, full_path.split(path)[1].replace(os.path.sep, '/')) if not isinstance(url_path, str): @@ -151,49 +142,45 @@ class IrModule(models.Model): if zf.file_size > MAX_FILE_SIZE: raise UserError(_("File '%s' exceed maximum allowed file size", zf.filename)) - with tempfile.TemporaryDirectory() as module_dir: - try: - __import_paths__[self.env] = module_dir - manifest_files = [ - file - for file in z.filelist - if file.filename.count('/') == 1 - and file.filename.split('/')[1] in MANIFEST_NAMES - ] - module_data_files = defaultdict(list) - for manifest in manifest_files: - manifest_path = z.extract(manifest, module_dir) - mod_name = manifest.filename.split('/')[0] - try: - with _file_open(self.env, manifest_path) as f: - terp = ast.literal_eval(f.read().decode()) - except Exception: + with file_open_temporary_directory(self.env) as module_dir: + manifest_files = [ + file + for file in z.filelist + if file.filename.count('/') == 1 + and file.filename.split('/')[1] in MANIFEST_NAMES + ] + module_data_files = defaultdict(list) + for manifest in manifest_files: + manifest_path = z.extract(manifest, module_dir) + mod_name = manifest.filename.split('/')[0] + try: + with file_open(manifest_path, 'rb', env=self.env) as f: + terp = ast.literal_eval(f.read().decode()) + except Exception: + continue + for filename in terp.get('data', []) + terp.get('init_xml', []) + terp.get('update_xml', []): + if os.path.splitext(filename)[1].lower() not in ('.xml', '.csv', '.sql'): continue - for filename in terp.get('data', []) + terp.get('init_xml', []) + terp.get('update_xml', []): - if os.path.splitext(filename)[1].lower() not in ('.xml', '.csv', '.sql'): - continue - module_data_files[mod_name].append('%s/%s' % (mod_name, filename)) - for file in z.filelist: - filename = file.filename - mod_name = filename.split('/')[0] - is_data_file = filename in module_data_files[mod_name] - is_static = filename.startswith('%s/static' % mod_name) - if is_data_file or is_static: - z.extract(file, module_dir) + module_data_files[mod_name].append('%s/%s' % (mod_name, filename)) + for file in z.filelist: + filename = file.filename + mod_name = filename.split('/')[0] + is_data_file = filename in module_data_files[mod_name] + is_static = filename.startswith('%s/static' % mod_name) + if is_data_file or is_static: + z.extract(file, module_dir) - dirs = [d for d in os.listdir(module_dir) if os.path.isdir(opj(module_dir, d))] - for mod_name in dirs: - module_names.append(mod_name) - try: - # assert mod_name.startswith('theme_') - path = opj(module_dir, mod_name) - if self._import_module(mod_name, path, force=force): - success.append(mod_name) - except Exception as e: - _logger.exception('Error while importing module') - errors[mod_name] = exception_to_unicode(e) - finally: - __import_paths__.pop(self.env) + dirs = [d for d in os.listdir(module_dir) if os.path.isdir(opj(module_dir, d))] + for mod_name in dirs: + module_names.append(mod_name) + try: + # assert mod_name.startswith('theme_') + path = opj(module_dir, mod_name) + if self._import_module(mod_name, path, force=force): + success.append(mod_name) + except Exception as e: + _logger.exception('Error while importing module') + errors[mod_name] = exception_to_unicode(e) r = ["Successfully imported module '%s'" % mod for mod in success] for mod, error in errors.items(): r.append("Error while importing module '%s'.\n\n %s \n Make sure those modules are installed and try again." % (mod, error)) diff --git a/addons/base_import_module/tests/test_import_module.py b/addons/base_import_module/tests/test_import_module.py index 944e865497..89edc92228 100644 --- a/addons/base_import_module/tests/test_import_module.py +++ b/addons/base_import_module/tests/test_import_module.py @@ -206,3 +206,23 @@ class TestImportModuleHttp(TestImportModule, HttpCase): self.assertEqual(self.url_open('/' + foo_icon_path).content, foo_icon_data) # Assert icon of module bar, which must be the icon of the base module as none was provided self.assertEqual(self.env.ref('base.module_bar').icon_image, self.env.ref('base.module_base').icon_image) + + def test_import_module_field_file(self): + files = [ + ('foo/__manifest__.py', b"{'data': ['data.xml']}"), + ('foo/data.xml', b""" + + + + """), + ('foo/static/src/img/content/logo.png', b"foo_logo"), + ] + self.import_zipfile(files) + logo_path, logo_data = files[2] + self.assertEqual(base64.b64decode(self.env.ref('foo.logo').datas), logo_data) + self.assertEqual(self.url_open('/' + logo_path).content, logo_data) diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 4f72020222..3d6f44c94b 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -1286,6 +1286,10 @@ class MailThread(models.AbstractModel): continue # skip container filename = part.get_filename() # I may not properly handle all charsets + if part.get_content_type() == 'text/xml' and not part.get_param('charset'): + # for text/xml with omitted charset, the charset is assumed to be ASCII by the `email` module + # although the payload might be in UTF8 + part.set_charset('utf-8') encoding = part.get_content_charset() # None if attachment content = part.get_content() diff --git a/addons/maintenance/security/maintenance.xml b/addons/maintenance/security/maintenance.xml index 27da28248b..a12d67128e 100644 --- a/addons/maintenance/security/maintenance.xml +++ b/addons/maintenance/security/maintenance.xml @@ -14,7 +14,7 @@ Users are allowed to access their own maintenance requests - ['|', ('message_partner_ids', 'in', [user.partner_id.id]), ('user_id.id', '=', user.id)] + ['|', '|', ('owner_user_id', '=', user.id), ('message_partner_ids', 'in', [user.partner_id.id]), ('user_id.id', '=', user.id)] diff --git a/addons/test_mail/data/test_mail_data.py b/addons/test_mail/data/test_mail_data.py index 2cabf0e037..c50391ffe7 100644 --- a/addons/test_mail/data/test_mail_data.py +++ b/addons/test_mail/data/test_mail_data.py @@ -326,6 +326,50 @@ Q2hhdXNz77+977+9ZSBkZSBCcnV4ZWxsZXM8L2NiYzpTdHJlZXROYW1lPg0KPC9JbnZvaWNlPg0K --00000000000005d9da05fa394cc0-- """ +MAIL_MULTIPART_OMITTED_CHARSET = """Return-Path: +To: {to} +cc: {cc} +Received: by mail1.openerp.com (Postfix, from userid 10002) + id 5DF9ABFB2A; Fri, 10 Aug 2012 16:16:39 +0200 (CEST) +From: {email_from} +Subject: {subject} +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="00000000000005d9da05fa394cc0" +Date: Fri, 10 Aug 2012 14:16:26 +0000 +Message-ID: {msg_id} +{extra} + +--00000000000005d9da05fa394cc0 +Content-Type: multipart/alternative; boundary="00000000000005d9d905fa394cbe" + +--00000000000005d9d905fa394cbe +Content-Type: text/plain; charset="UTF-8" + +Dear customer, + +Please find attached the UBL attachment of your invoice + +Cheers, + +--00000000000005d9d905fa394cbe +Content-Type: text/html; charset="UTF-8" + +
Dear customer,

Please find attached the UBL attachment of your invoice

Cheers,
+ +--00000000000005d9d905fa394cbe-- + +--00000000000005d9da05fa394cc0 +Content-Disposition: attachment; filename="bis3.xml" +Content-Transfer-Encoding: base64 +Content-Type: text/xml; name="bis3.xml" +Content-ID: +X-Attachment-Id: f_lgxgdqx40 + +PEludm9pY2U+Q2hhdXNzw6llIGRlIEJydXhlbGxlczwvSW52b2ljZT4= +--00000000000005d9da05fa394cc0-- +""" + MAIL_SINGLE_BINARY = """X-Original-To: raoul@grosbedon.fr Delivered-To: raoul@grosbedon.fr diff --git a/addons/test_mail/tests/test_mail_gateway.py b/addons/test_mail/tests/test_mail_gateway.py index 025d09533b..2cfdd6d5fa 100644 --- a/addons/test_mail/tests/test_mail_gateway.py +++ b/addons/test_mail/tests/test_mail_gateway.py @@ -1507,6 +1507,13 @@ class TestMailgateway(TestMailCommon): # This explains the multiple "�" in the attachment. self.assertIn("Chauss������e de Bruxelles", record.message_main_attachment_id.raw.decode()) + def test_message_process_file_omitted_charset(self): + """ For incoming email containing an xml attachment with omitted charset and containing an UTF8 payload we + should parse the attachment using UTF-8. + """ + record = self.format_and_process(test_mail_data.MAIL_MULTIPART_OMITTED_CHARSET, self.email_from, 'groups@test.com') + self.assertEqual(record.message_main_attachment_id.name, 'bis3.xml') + self.assertEqual("Chaussée de Bruxelles", record.message_main_attachment_id.raw.decode()) class TestMailThreadCC(TestMailCommon): diff --git a/addons/website_links/static/src/js/website_links_charts.js b/addons/website_links/static/src/js/website_links_charts.js index 30d6b8a1a5..008eff0b08 100644 --- a/addons/website_links/static/src/js/website_links_charts.js +++ b/addons/website_links/static/src/js/website_links_charts.js @@ -19,7 +19,7 @@ var BarChart = publicWidget.Widget.extend({ */ init: function (parent, beginDate, endDate, dates) { this._super.apply(this, arguments); - this.beginDate = beginDate; + this.beginDate = beginDate.locale("en"); this.endDate = endDate; this.number_of_days = this.endDate.diff(this.beginDate, 'days') + 2; this.dates = dates; @@ -162,11 +162,18 @@ publicWidget.registry.websiteLinksCharts = publicWidget.Widget.extend({ var formattedClicksByDay = {}; var beginDate; for (var i = 0; i < _clicksByDay.length; i++) { - var date = moment(_clicksByDay[i]['create_date:day'], 'DD MMMM YYYY'); + // This is a trick to get the date without the local formatting. + // We can't simply do .locale("en") because some Flectra languages + // are not supported by moment.js (eg: Arabic Syria). + const date = moment( + _clicksByDay[i]["__domain"].find((el) => el.length && el.includes(">="))[2] + .split(" ")[0], "YYYY MM DD" + ); if (i === 0) { beginDate = date; } - formattedClicksByDay[date.format('YYYY-MM-DD')] = _clicksByDay[i]['create_date_count']; + formattedClicksByDay[date.locale("en").format("YYYY-MM-DD")] = + _clicksByDay[i]["create_date_count"]; } // Process all time line chart data @@ -241,11 +248,14 @@ publicWidget.registry.websiteLinksCharts = publicWidget.Widget.extend({ * @private */ _lastWeekClicksByCountry: function () { - var interval = moment().subtract(7, 'days').format('YYYY-MM-DD'); + // 7 days * 24 hours * 60 minutes * 60 seconds * 1000 milliseconds. + const aWeekAgoDate = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + // get the date in the format YYYY-MM-DD. + const aWeekAgoString = aWeekAgoDate.toISOString().split("T")[0]; return this._rpc({ model: 'link.tracker.click', method: 'read_group', - args: [[this.links_domain, ['create_date', '>', interval]], ['country_id']], + args: [[this.links_domain, ["create_date", ">", aWeekAgoString]], ["country_id"]], kwargs: {groupby: 'country_id'}, }); }, @@ -253,11 +263,14 @@ publicWidget.registry.websiteLinksCharts = publicWidget.Widget.extend({ * @private */ _lastMonthClicksByCountry: function () { - var interval = moment().subtract(30, 'days').format('YYYY-MM-DD'); + // 30 days * 24 hours * 60 minutes * 60 seconds * 1000 milliseconds. + const aMonthAgoDate = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + // get the date in the format YYYY-MM-DD. + const aMonthAgoString = aMonthAgoDate.toISOString().split("T")[0]; return this._rpc({ model: 'link.tracker.click', method: 'read_group', - args: [[this.links_domain, ['create_date', '>', interval]], ['country_id']], + args: [[this.links_domain, ["create_date", ">", aMonthAgoString]], ["country_id"]], kwargs: {groupby: 'country_id'}, }); },