Compression plugin

conan/api/subapi/cache.py

@@ -1,12 +1,10 @@
 import json
 import os
 import shutil
-import tarfile
 import tempfile
 
 from conan.api.model import PackagesList
 from conan.api.output import ConanOutput
-from conan.internal.api.uploader import compress_files
 from conan.internal.cache.cache import PkgCache
 from conan.internal.cache.conan_reference_layout import EXPORT_SRC_FOLDER, EXPORT_FOLDER, SRC_FOLDER, \
     METADATA, DOWNLOAD_EXPORT_FOLDER
@@ -18,6 +16,7 @@
 from conan.api.model import RecipeReference
 from conan.internal.util.dates import revision_timestamp_now
 from conan.internal.util.files import rmdir, mkdir, remove, save
+from conan.internal.util.compression import tar_compress, tar_extract
 
 
 class CacheAPI:
@@ -172,7 +171,8 @@ def save(self, package_list, tgz_path, no_source=False):
         pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json")
         save(pkglist_path, serialized)
         tar_files["pkglist.json"] = pkglist_path
-        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, recursive=True)
+        tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel,
+                    recursive=True, ref=None, compress_plugin=self.conan_api.config.compress_plugin)
         remove(pkglist_path)
 
     def restore(self, path):
@@ -181,14 +181,10 @@ def restore(self, path):
 
         cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf)
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
-
-        with open(path, mode='rb') as file_handler:
-            the_tar = tarfile.open(fileobj=file_handler)
-            fileobj = the_tar.extractfile("pkglist.json")
-            pkglist = fileobj.read()
-            the_tar.extraction_filter = (lambda member, _: member)  # fully_trusted (Py 3.14)
-            the_tar.extractall(path=cache_folder)
-            the_tar.close()
+        tar_extract(path, cache_folder, compress_plugin=self.conan_api.config.compress_plugin)
+        # Retrieve the package list from the already extracted archive
+        with open(os.path.join(cache_folder, "pkglist.json")) as file_handler:
+            pkglist = file_handler.read()
 
         # After unzipping the files, we need to update the DB that references these files
         out = ConanOutput()

conan/api/subapi/config.py

@@ -3,6 +3,7 @@
 import platform
 import textwrap
 import yaml
+from conan.internal.util.compression import load_compression_plugin
 from jinja2 import Environment, FileSystemLoader
 
 from conan import conan_version
@@ -31,6 +32,7 @@ def __init__(self, conan_api):
         self._new_config = None
         self._cli_core_confs = None
         self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path)
+        self._compress_plugin = None
 
     def home(self):
         return self.conan_api.cache_folder
@@ -236,4 +238,11 @@ def reinit(self):
         if self._new_config is not None:
             self._new_config.clear()
             self._populate_global_conf()
+            self._compress_plugin = None
         self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path)
+
+    @property
+    def compression_plugin(self):
+        if not self._compress_plugin:
+            self._compress_plugin = load_compression_plugin(self.conan_api.cache_folder)
+        return self._compress_plugin

conan/internal/api/uploader.py

@@ -12,6 +12,7 @@
 from conan.errors import ConanException
 from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
                                   EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
+from conan.internal.util.compression import load_compression_plugin
 from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
                                        set_dirty_context_manager, mkdir, human_size)
 
@@ -157,7 +158,8 @@ def add_tgz(tgz_name, tgz_files):
             elif tgz_files:
                 compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
                 tgz = compress_files(tgz_files, tgz_name, download_export_folder,
-                                     compresslevel=compresslevel, ref=ref)
+                                     compresslevel=compresslevel, ref=ref,
+                                     compress_plugin=self._app.conan_api.config.compress_plugin)
                 result[tgz_name] = tgz
 
         add_tgz(EXPORT_TGZ_NAME, files)
@@ -204,7 +206,8 @@ def _compress_package_files(self, layout, pref):
             tgz_files = {f: path for f, path in files.items()}
             compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
             tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel, ref=pref)
+                                      compresslevel=compresslevel, ref=pref,
+                                      compress_plugin=self._app.conan_api.config.compress_plugin)
             assert tgz_path == package_tgz
             assert os.path.exists(package_tgz)
 
@@ -271,12 +274,15 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False,
+                   compress_plugin=None):
+    if compress_plugin:
+        return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref)
+
     t1 = time.time()
     # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
-    if ref:
-        ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}")
+    ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}")
     with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
         tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
         for filename, abs_path in sorted(files.items()):

conan/internal/cache/home_paths.py

@@ -90,3 +90,7 @@ def settings_path_user(self):
     @property
     def config_version_path(self):
         return os.path.join(self._home, "config_version.json")
+
+    @property
+    def compression_plugin_path(self):
+        return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py")

conan/internal/rest/remote_manager.py

@@ -1,3 +1,4 @@
+from gzip import compress
 import os
 import shutil
 from typing import List
@@ -15,6 +16,7 @@
 from conan.internal.model.info import load_binary_info
 from conan.api.model import PkgReference
 from conan.api.model import RecipeReference
+from conan.internal.util.compression import load_compression_plugin
 from conan.internal.util.files import rmdir, human_size
 from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
 from conan.internal.util.files import mkdir, tar_extract
@@ -27,6 +29,7 @@ def __init__(self, cache, auth_manager, home_folder):
         self._auth_manager = auth_manager
         self._signer = PkgSignaturesPlugin(cache, home_folder)
         self._home_folder = home_folder
+        self._compression_plugin = load_compression_plugin(home_folder) # TODO: should use the instantiated one in ConfigAPI
 
     def _local_folder_remote(self, remote):
         if remote.remote_type == LOCAL_RECIPES_INDEX:
@@ -81,7 +84,7 @@ def get_recipe(self, ref, remote, metadata=None):
         tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
 
         if tgz_file:
-            uncompress_file(tgz_file, export_folder, scope=str(ref))
+            uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._compression_plugin)
         mkdir(export_folder)
         for file_name, file_path in zipped_files.items():  # copy CONANFILE
             shutil.move(file_path, os.path.join(export_folder, file_name))
@@ -123,7 +126,7 @@ def get_recipe_sources(self, ref, layout, remote):
 
         self._signer.verify(ref, download_folder, files=zipped_files)
         tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
-        uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
+        uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._compression_plugin)
 
     def get_package(self, pref, remote, metadata=None):
         output = ConanOutput(scope=str(pref.ref))
@@ -171,7 +174,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
 
             tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
             package_folder = layout.package()
-            uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
+            uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._compression_plugin)
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
             for file_name, file_path in zipped_files.items():  # copy CONANINFO and CONANMANIFEST
                 shutil.move(file_path, os.path.join(package_folder, file_name))
@@ -281,15 +284,18 @@ def _call_remote(self, remote, method, *args, **kwargs):
             raise ConanException(exc, remote=remote)
 
 
-def uncompress_file(src_path, dest_folder, scope=None):
+def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None):
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB
         if big_file:
             hs = human_size(filesize)
             ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}")
-        with open(src_path, mode='rb') as file_handler:
-            tar_extract(file_handler, dest_folder)
+
+        if compression_plugin:
+            compression_plugin.tar_extract(src_path, dest_folder)
+        else:
+            tar_extract(src_path, dest_folder)
     except Exception as e:
         error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\
                     % (src_path, dest_folder, str(e))

conan/internal/util/compression.py

@@ -0,0 +1,79 @@
+from functools import lru_cache
+from conan.internal.cache.home_paths import HomePaths
+from conan.internal.loader import load_python_file
+from conan.internal.errors import ConanException
+
+import os
+import gzip
+import time
+import tarfile
+from conan.api.output import ConanOutput
+from conan.internal.util.files import set_dirty_context_manager
+
+def tar_extract(src_path, destination_dir, compress_plugin=None):
+    if compress_plugin:
+       return compress_plugin.tar_extract(src_path, destination_dir)
+
+    with open(src_path, mode='rb') as file_handler:
+        the_tar = tarfile.open(fileobj=file_handler)
+        # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to
+        # "could not change modification time", with time=0
+        # the_tar.errorlevel = 2  # raise exception if any error
+        the_tar.extraction_filter = (lambda member, path: member)  # fully_trusted, avoid Py3.14 break
+        the_tar.extractall(path=destination_dir)
+        the_tar.close()
+
+
+def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compress_plugin=None):
+    if compress_plugin:
+        return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref)
+
+    t1 = time.time()
+    # FIXME, better write to disk sequentially and not keep tgz contents in memory
+    tgz_path = os.path.join(dest_dir, name)
+    ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}")
+    with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
+        tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
+        for filename, abs_path in sorted(files.items()):
+            # recursive is False in case it is a symlink to a folder
+            tgz.add(abs_path, filename, recursive=recursive)
+        tgz.close()
+
+    duration = time.time() - t1
+    ConanOutput().debug(f"{name} compressed in {duration} time")
+    return tgz_path
+
+def gzopen_without_timestamps(name, fileobj, compresslevel=None):
+    """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was
+        setted in Gzip file causing md5 to change. Not possible using the
+        previous tarfile open because arguments are not passed to GzipFile constructor
+    """
+    compresslevel = compresslevel if compresslevel is not None else 9  # default Gzip = 9
+    fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0)
+    # Format is forced because in Python3.8, it changed and it generates different tarfiles
+    # with different checksums, which break hashes of tgzs
+    # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions
+    t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT)
+    t._extfileobj = False
+    return t
+
+
+def load_compression_plugin(cache_folder):
+    if not cache_folder:
+        return None
+    compression_plugin_path = HomePaths(cache_folder).compression_plugin_path
+    if not os.path.exists(compression_plugin_path):
+        return None
+
+    mod, _ = load_python_file(compression_plugin_path)
+    if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"):
+        raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions")
+    return mod
+
+
+"""
+Plugin `compression.py` interface:
+
+    def tar_extract(src_path: str, destination_dir: str) -> None
+    def tar_compress(files: List[str], name: str, dest_dir: str, compresslevel=None, ref: str=None, cache_folder:str, recursive: bool = False) -> str
+"""