[FIX] upstream pathc

flectra/tools/cloc.py

@@ -17,6 +17,7 @@ DEFAULT_EXCLUDE = [
     "static/lib/**/*",
     "static/tests/**/*",
     "migrations/**/*",
+    "upgrades/**/*",
 ]
 
 STANDARD_MODULES = ['web', 'web_enterprise', 'website_animate', 'base']

flectra/tools/image.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Part of Odoo, Flectra. See LICENSE file for full copyright and licensing details.
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
 import base64
 import binascii
 import io
@@ -88,6 +88,68 @@ class ImageProcess():
             if verify_resolution and w * h > IMAGE_MAX_RESOLUTION:
                 raise ValueError(_("Image size excessive, uploaded images must be smaller than %s million pixels.", str(IMAGE_MAX_RESOLUTION / 10e6)))
 
+    def image_quality(self, quality=0, output_format=''):
+        """Return the image resulting of all the image processing
+        operations that have been applied previously.
+
+        Return False if the initialized `image` was falsy, and return
+        the initialized `image` without change if it was SVG.
+
+        Also return the initialized `image` if no operations have been applied
+        and the `output_format` is the same as the original format and the
+        quality is not specified.
+
+        :param quality: quality setting to apply. Default to 0.
+            - for JPEG: 1 is worse, 95 is best. Values above 95 should be
+                avoided. Falsy values will fallback to 95, but only if the image
+                was changed, otherwise the original image is returned.
+            - for PNG: set falsy to prevent conversion to a WEB palette.
+            - for other formats: no effect.
+        :type quality: int
+
+        :param output_format: the output format. Can be PNG, JPEG, GIF, or ICO.
+            Default to the format of the original image. BMP is converted to
+            PNG, other formats than those mentioned above are converted to JPEG.
+        :type output_format: string
+
+        :return: image
+        :rtype: bytes or False
+        """
+        if not self.image:
+            return self.image
+
+        output_image = self.image
+
+        output_format = output_format.upper() or self.original_format
+        if output_format == 'BMP':
+            output_format = 'PNG'
+        elif output_format not in ['PNG', 'JPEG', 'GIF', 'ICO']:
+            output_format = 'JPEG'
+
+        if not self.operationsCount and output_format == self.original_format and not quality:
+            return self.image
+
+        opt = {'format': output_format}
+
+        if output_format == 'PNG':
+            opt['optimize'] = True
+            if quality:
+                if output_image.mode != 'P':
+                    # Floyd Steinberg dithering by default
+                    output_image = output_image.convert('RGBA').convert('P', palette=Image.WEB, colors=256)
+        if output_format == 'JPEG':
+            opt['optimize'] = True
+            opt['quality'] = quality or 95
+        if output_format == 'GIF':
+            opt['optimize'] = True
+            opt['save_all'] = True
+
+        if output_image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (output_format == 'JPEG' and output_image.mode == 'RGBA'):
+            output_image = output_image.convert("RGB")
+
+        return image_apply_opt(output_image, **opt)
+
+    # TODO: rename to image_quality_base64 in master~saas-15.1
     def image_base64(self, quality=0, output_format=''):
         """Return the base64 encoded image resulting of all the image processing
         operations that have been applied previously.
@@ -115,39 +177,15 @@ class ImageProcess():
         :return: image base64 encoded or False
         :rtype: bytes or False
         """
-        output_image = self.image
 
-        if not output_image:
+        if not self.image:
             return self.base64_source
 
-        output_format = output_format.upper() or self.original_format
-        if output_format == 'BMP':
-            output_format = 'PNG'
-        elif output_format not in ['PNG', 'JPEG', 'GIF', 'ICO']:
-            output_format = 'JPEG'
+        stream = self.image_quality(quality=quality, output_format=output_format)
 
-        if not self.operationsCount and output_format == self.original_format and not quality:
-            return self.base64_source
-
-        opt = {'format': output_format}
-
-        if output_format == 'PNG':
-            opt['optimize'] = True
-            if quality:
-                if output_image.mode != 'P':
-                    # Floyd Steinberg dithering by default
-                    output_image = output_image.convert('RGBA').convert('P', palette=Image.WEB, colors=256)
-        if output_format == 'JPEG':
-            opt['optimize'] = True
-            opt['quality'] = quality or 95
-        if output_format == 'GIF':
-            opt['optimize'] = True
-            opt['save_all'] = True
-
-        if output_image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (output_format == 'JPEG' and output_image.mode == 'RGBA'):
-            output_image = output_image.convert("RGB")
-
-        return image_to_base64(output_image, **opt)
+        if stream != self.image:
+            return base64.b64encode(stream)
+        return self.base64_source
 
     def resize(self, max_width=0, max_height=0):
         """Resize the image.
@@ -406,6 +444,22 @@ def base64_to_image(base64_source):
         raise UserError(_("This file could not be decoded as an image file. Please try with a different file."))
 
 
+def image_apply_opt(image, format, **params):
+    """Return the given PIL `image` using `params`.
+
+    :param image: the PIL image
+    :type image: PIL.Image
+
+    :param params: params to expand when calling PIL.Image.save()
+    :type params: dict
+
+    :return: the image formatted
+    :rtype: bytes
+    """
+    stream = io.BytesIO()
+    image.save(stream, format=format, **params)
+    return stream.getvalue()
+
 def image_to_base64(image, format, **params):
     """Return a base64_image from the given PIL `image` using `params`.
 
@@ -418,10 +472,8 @@ def image_to_base64(image, format, **params):
     :return: the image base64 encoded
     :rtype: bytes
     """
-    stream = io.BytesIO()
-    image.save(stream, format=format, **params)
-    return base64.b64encode(stream.getvalue())
-
+    stream = image_apply_opt(image, format, **params)
+    return base64.b64encode(stream)
 
 def is_image_size_above(base64_source_1, base64_source_2):
     """Return whether or not the size of the given image `base64_source_1` is

flectra/tools/mimetypes.py

@@ -5,6 +5,7 @@ Mimetypes-related utilities
 # TODO: reexport stdlib mimetypes?
 """
 import collections
+import functools
 import io
 import logging
 import re
@@ -168,15 +169,22 @@ except ImportError:
     magic = None
 else:
     # There are 2 python libs named 'magic' with incompatible api.
-
     # magic from pypi https://pypi.python.org/pypi/python-magic/
-    if hasattr(magic,'from_buffer'):
-        guess_mimetype = lambda bin_data, default=None: magic.from_buffer(bin_data, mime=True)
+    if hasattr(magic, 'from_buffer'):
+        _guesser = functools.partial(magic.from_buffer, mime=True)
     # magic from file(1) https://packages.debian.org/squeeze/python-magic
-    elif hasattr(magic,'open'):
+    elif hasattr(magic, 'open'):
         ms = magic.open(magic.MAGIC_MIME_TYPE)
         ms.load()
-        guess_mimetype = lambda bin_data, default=None: ms.buffer(bin_data)
+        _guesser = ms.buffer
+
+    def guess_mimetype(bin_data, default=None):
+        mimetype = _guesser(bin_data)
+        # upgrade incorrect mimetype to official one, fixed upstream
+        # https://github.com/file/file/commit/1a08bb5c235700ba623ffa6f3c95938fe295b262
+        if mimetype == 'image/svg':
+            return 'image/svg+xml'
+        return mimetype
 
 def neuter_mimetype(mimetype, user):
     wrong_type = 'ht' in mimetype or 'xml' in mimetype or 'svg' in mimetype

flectra/tools/pdf.py

@@ -1,14 +1,23 @@
 # -*- coding: utf-8 -*-
-# Part of Odoo, Flectra. See LICENSE file for full copyright and licensing details.
-from PyPDF2 import PdfFileWriter, PdfFileReader
-from PyPDF2.generic import DictionaryObject, DecodedStreamObject, NameObject, createStringObject, ArrayObject
-from PyPDF2.utils import b_
-from datetime import datetime
-
+# Part of Flectra. See LICENSE file for full copyright and licensing details.
+import base64
 import io
-import hashlib
 
+from datetime import datetime
+from hashlib import md5
+from logging import getLogger
+from PyPDF2 import PdfFileWriter, PdfFileReader
+from PyPDF2.generic import DictionaryObject, NameObject, ArrayObject, DecodedStreamObject, NumberObject, createStringObject, ByteStringObject
+from zlib import compress, decompress
 
+try:
+    from fontTools.ttLib import TTFont
+except ImportError:
+    TTFont = None
+
+from flectra.tools.misc import file_open
+
+_logger = getLogger(__name__)
 DEFAULT_PDF_DATETIME_FORMAT = "D:%Y%m%d%H%M%S+00'00'"
 
 
@@ -99,14 +108,202 @@ class FlectraPdfFileReader(PdfFileReader):
 
 
 class FlectraPdfFileWriter(PdfFileWriter):
-    # OVERRIDE of PdfFileWriter to add the management of multiple embedded files.
+
+    def __init__(self, *args, **kwargs):
+        """
+        Override of the init to initialise additional variables.
+        :param pdf_content: if given, will initialise the reader with the pdf content.
+        """
+        super().__init__(*args, **kwargs)
+        self._reader = None
+        self.is_pdfa = False
+
+    def addAttachment(self, name, data, subtype=""):
+        """
+        Add an attachment to the pdf. Supports adding multiple attachment, while respecting PDF/A rules.
+        :param name: The name of the attachement
+        :param data: The data of the attachement
+        :param subtype: The mime-type of the attachement. This is required by PDF/A, but not essential otherwise.
+        It should take the form of "/xxx%2Fxxx". E.g. for "text/xml": "/text%2Fxml"
+        """
+        attachment = self._create_attachment_object({
+            'filename': name,
+            'content': data,
+            'subtype': subtype,
+        })
+        if self._root_object.get('/Names') and self._root_object['/Names'].get('/EmbeddedFiles'):
+            names_array = self._root_object["/Names"]["/EmbeddedFiles"]["/Names"]
+            names_array.extend([attachment.getObject()['/F'], attachment])
+        else:
+            names_array = ArrayObject()
+            names_array.extend([attachment.getObject()['/F'], attachment])
+
+            embedded_files_names_dictionary = DictionaryObject()
+            embedded_files_names_dictionary.update({
+                NameObject("/Names"): names_array
+            })
+            embedded_files_dictionary = DictionaryObject()
+            embedded_files_dictionary.update({
+                NameObject("/EmbeddedFiles"): embedded_files_names_dictionary
+            })
+            self._root_object.update({
+                NameObject("/Names"): embedded_files_dictionary
+            })
+
+        if self._root_object.get('/AF'):
+            attachment_array = self._root_object['/AF']
+            attachment_array.extend([attachment])
+        else:
+            # Create a new object containing an array referencing embedded file
+            # And reference this array in the root catalogue
+            attachment_array = self._addObject(ArrayObject([attachment]))
+            self._root_object.update({
+                NameObject("/AF"): attachment_array
+            })
+
+    def embed_flectra_attachment(self, attachment):
+        assert attachment, "embed_flectra_attachment cannot be called without attachment."
+        self.addAttachment(attachment.name, attachment.raw, attachment.mimetype)
+
+    def cloneReaderDocumentRoot(self, reader):
+        super().cloneReaderDocumentRoot(reader)
+        self._reader = reader
+        # Try to read the header coming in, and reuse it in our new PDF
+        # This is done in order to allows modifying PDF/A files after creating them (as PyPDF does not read it)
+        stream = reader.stream
+        stream.seek(0)
+        header = stream.readlines(9)
+        # Should always be true, the first line of a pdf should have 9 bytes (%PDF-1.x plus a newline)
+        if len(header) == 1:
+            # If we found a header, set it back to the new pdf
+            self._header = header[0]
+            # Also check the second line. If it is PDF/A, it should be a line starting by % following by four bytes + \n
+            second_line = stream.readlines(1)[0]
+            if second_line.decode('latin-1')[0] == '%' and len(second_line) == 6:
+                self._header += second_line
+                self.is_pdfa = True
+        # Look if we have an ID in the incoming stream and use it.
+        pdf_id = reader.trailer.get('/ID', None)
+        if pdf_id:
+            self._ID = pdf_id
+
+    def convert_to_pdfa(self):
+        """
+        Transform the opened PDF file into a PDF/A compliant file
+        """
+        # Set the PDF version to 1.7 (as PDF/A-3 is based on version 1.7) and make it PDF/A compliant.
+        # See https://github.com/veraPDF/veraPDF-validation-profiles/wiki/PDFA-Parts-2-and-3-rules#rule-612-1
+
+        # " The file header shall begin at byte zero and shall consist of "%PDF-1.n" followed by a single EOL marker,
+        # where 'n' is a single digit number between 0 (30h) and 7 (37h) "
+        # " The aforementioned EOL marker shall be immediately followed by a % (25h) character followed by at least four
+        # bytes, each of whose encoded byte values shall have a decimal value greater than 127 "
+        self._header = b"%PDF-1.7\n%\xFF\xFF\xFF\xFF"
+
+        # Add a document ID to the trailer. This is only needed when using encryption with regular PDF, but is required
+        # when using PDF/A
+        pdf_id = ByteStringObject(md5(self._reader.stream.getvalue()).digest())
+        # The first string is based on the content at the time of creating the file, while the second is based on the
+        # content of the file when it was last updated. When creating a PDF, both are set to the same value.
+        self._ID = ArrayObject((pdf_id, pdf_id))
+
+        with file_open('data/files/sRGB2014.icc', subdir='tools', mode='rb') as icc_profile:
+            icc_profile_file_data = compress(icc_profile.read())
+
+        icc_profile_stream_obj = DecodedStreamObject()
+        icc_profile_stream_obj.setData(icc_profile_file_data)
+        icc_profile_stream_obj.update({
+            NameObject("/Filter"): NameObject("/FlateDecode"),
+            NameObject("/N"): NumberObject(3),
+            NameObject("/Length"): NameObject(str(len(icc_profile_file_data))),
+        })
+
+        icc_profile_obj = self._addObject(icc_profile_stream_obj)
+
+        output_intent_dict_obj = DictionaryObject()
+        output_intent_dict_obj.update({
+            NameObject("/S"): NameObject("/GTS_PDFA1"),
+            NameObject("/OutputConditionIdentifier"): createStringObject("sRGB"),
+            NameObject("/DestOutputProfile"): icc_profile_obj,
+            NameObject("/Type"): NameObject("/OutputIntent"),
+        })
+
+        output_intent_obj = self._addObject(output_intent_dict_obj)
+        self._root_object.update({
+            NameObject("/OutputIntents"): ArrayObject([output_intent_obj]),
+        })
+
+        pages = self._root_object['/Pages']['/Kids']
+
+        # PDF/A needs the glyphs width array embedded in the pdf to be consistent with the ones from the font file.
+        # But it seems like it is not the case when exporting from wkhtmltopdf.
+        if TTFont:
+            fonts = {}
+            # First browse through all the pages of the pdf file, to get a reference to all the fonts used in the PDF.
+            for page in pages:
+                for font in page.getObject()['/Resources']['/Font'].values():
+                    for descendant in font.getObject()['/DescendantFonts']:
+                        fonts[descendant.idnum] = descendant.getObject()
+
+            # Then for each font, rewrite the width array with the information taken directly from the font file.
+            # The new width are calculated such as width = round(1000 * font_glyph_width / font_units_per_em)
+            # See: http://martin.hoppenheit.info/blog/2018/pdfa-validation-and-inconsistent-glyph-width-information/
+            for font in fonts.values():
+                font_file = font['/FontDescriptor']['/FontFile2']
+                stream = io.BytesIO(decompress(font_file._data))
+                ttfont = TTFont(stream)
+                font_upm = ttfont['head'].unitsPerEm
+                glyphs = ttfont.getGlyphSet()._hmtx.metrics
+                glyph_widths = []
+                for key, values in glyphs.items():
+                    if key[:5] == 'glyph':
+                        glyph_widths.append(NumberObject(round(1000.0 * values[0] / font_upm)))
+
+                font[NameObject('/W')] = ArrayObject([NumberObject(1), ArrayObject(glyph_widths)])
+                stream.close()
+        else:
+            _logger.warning('The fonttools package is not installed. Generated PDF may not be PDF/A compliant.')
+
+        outlines = self._root_object['/Outlines'].getObject()
+        outlines[NameObject('/Count')] = NumberObject(1)
+
+        # Set flectra as producer
+        self.addMetadata({
+            '/Creator': "Flectra",
+            '/Producer': "Flectra",
+        })
+        self.is_pdfa = True
+
+    def add_file_metadata(self, metadata_content):
+        """
+        Set the XMP metadata of the pdf, wrapping it with the necessary XMP header/footer.
+        These are required for a PDF/A file to be completely compliant. Ommiting them would result in validation errors.
+        :param metadata_content: bytes of the metadata to add to the pdf.
+        """
+        # See https://wwwimages2.adobe.com/content/dam/acom/en/devnet/xmp/pdfs/XMP%20SDK%20Release%20cc-2016-08/XMPSpecificationPart1.pdf
+        # Page 10/11
+        header = b'<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>'
+        footer = b'<?xpacket end="w"?>'
+        metadata = b'%s%s%s' % (header, metadata_content, footer)
+        file_entry = DecodedStreamObject()
+        file_entry.setData(metadata)
+        file_entry.update({
+            NameObject("/Type"): NameObject("/Metadata"),
+            NameObject("/Subtype"): NameObject("/XML"),
+            NameObject("/Length"): NameObject(str(len(metadata))),
+        })
+
+        # Add the new metadata to the pdf, then redirect the reference to refer to this new object.
+        metadata_object = self._addObject(file_entry)
+        self._root_object.update({NameObject("/Metadata"): metadata_object})
 
     def _create_attachment_object(self, attachment):
         ''' Create a PyPdf2.generic object representing an embedded file.
 
         :param attachment: A dictionary containing:
-            * filename: The name of the file to embed (require).
-            * content:  The content of the file encoded in base64 (require).
+            * filename: The name of the file to embed (required)
+            * content:  The bytes of the file to embed (required)
+            * subtype: The mime-type of the file to embed (optional)
         :return:
         '''
         file_entry = DecodedStreamObject()
@@ -115,7 +312,7 @@ class FlectraPdfFileWriter(PdfFileWriter):
             NameObject("/Type"): NameObject("/EmbeddedFile"),
             NameObject("/Params"):
                 DictionaryObject({
-                    NameObject('/CheckSum'): createStringObject(hashlib.md5(attachment['content']).hexdigest()),
+                    NameObject('/CheckSum'): createStringObject(md5(attachment['content']).hexdigest()),
                     NameObject('/ModDate'): createStringObject(datetime.now().strftime(DEFAULT_PDF_DATETIME_FORMAT)),
                     NameObject('/Size'): NameObject(str(len(attachment['content']))),
                 }),
@@ -140,12 +337,3 @@ class FlectraPdfFileWriter(PdfFileWriter):
         if attachment.get('description'):
             filespec_object.update({NameObject("/Desc"): createStringObject(attachment['description'])})
         return self._addObject(filespec_object)
-
-    def addAttachment(self, fname, fdata):
-        # OVERRIDE of the AddAttachment method to allow appending attachemnts when some already exist
-        if self._root_object.get('/Names') and self._root_object['/Names'].get('/EmbeddedFiles'):
-            attachments = self._root_object["/Names"]["/EmbeddedFiles"]["/Names"]
-            new_attachment = self._create_attachment_object({'filename': fname, 'content': fdata})
-            attachments.extend([new_attachment.getObject()['/F'], new_attachment])
-        else:
-            super().addAttachment(fname, fdata)