From 434ed3cd807faeba5e9e02896b13d3d16e3cf70d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n?= Date: Wed, 6 Dec 2023 02:23:02 +0100 Subject: [PATCH 1/5] First sketch for populating download cache on conan upload --- conan/cli/commands/upload.py | 3 +++ conan/internal/conan_app.py | 1 + conans/client/cmd/uploader.py | 13 +++++++++++++ .../test/integration/command/upload/upload_test.py | 3 +++ 4 files changed, 20 insertions(+) diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 9ab19ade83b..13a6f391061 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -1,9 +1,12 @@ +import os + from conan.api.conan_api import ConanAPI from conan.api.model import ListPattern, MultiPackagesList from conan.api.output import ConanOutput from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument from conan.cli.commands.list import print_list_json, print_serial +from conans.client.downloaders.download_cache import DownloadCache from conans.client.userio import UserInput from conan.errors import ConanException diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index b42e16a61fc..36a23bddea4 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -38,6 +38,7 @@ def __init__(self, requester, cmd_wrapper, global_conf, cache): class ConanApp(object): def __init__(self, cache_folder, global_conf): + self.global_conf = global_conf self.cache_folder = cache_folder self.cache = ClientCache(self.cache_folder, global_conf) diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index f16ee7a5eae..f0a4a4be771 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -4,6 +4,7 @@ from conan.internal.conan_app import ConanApp from conan.api.output import ConanOutput +from conans.client.downloaders.download_cache import DownloadCache from conans.client.source import retrieve_exports_sources from conans.errors import ConanException, NotFoundException from conans.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, @@ -244,6 +245,18 @@ def upload_package(self, pref, prev_bundle, remote): self._app.remote_manager.upload_package(pref, cache_files, remote) duration = time.time() - t1 self._output.debug(f"Upload {pref} in {duration} time") + for cache_file in cache_files: + self._local_cache_file(pref, cache_file, remote) + + def _local_cache_file(self, pref, cache_file, remote): + approved_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt"] + if cache_file not in approved_files: + return + _download_cache = self._app.global_conf.get("core.download:download_cache") + if True or _download_cache and os.path.isabs(_download_cache): + url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.package_file(pref, cache_file) + cached_path = DownloadCache(_download_cache).cached_path(url) + self._output.info(f"Caching future {url} to local cache '{cached_path}'") def compress_files(files, name, dest_dir, compresslevel=None, ref=None): diff --git a/conans/test/integration/command/upload/upload_test.py b/conans/test/integration/command/upload/upload_test.py index f35a9b14426..c8b77f678fd 100644 --- a/conans/test/integration/command/upload/upload_test.py +++ b/conans/test/integration/command/upload/upload_test.py @@ -15,6 +15,7 @@ from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ TurboTestClient, GenConanfile, TestRequester, TestingResponse from conans.util.files import gzopen_without_timestamps, is_dirty, save, set_dirty +from utils.test_files import temp_folder conanfile = """from conan import ConanFile from conan.tools.files import copy @@ -231,6 +232,8 @@ def test_upload_unmodified_package(self): client.save({"conanfile.py": conanfile, "hello.cpp": ""}) + download_cache = temp_folder() + client.save_home({"global.conf": f"core.download:download_cache={download_cache}"}) ref = RecipeReference.loads("hello0/1.2.1@frodo/stable") client.run("create . --user=frodo --channel=stable") client.run("upload hello0/1.2.1@frodo/stable -r default") From 6f9bc96fbe5218ce1b6bcc5f95fb8f9b7e17ebc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n?= Date: Wed, 6 Dec 2023 02:33:41 +0100 Subject: [PATCH 2/5] Minor cleanup to simplify diff --- conan/cli/commands/upload.py | 1 - conans/client/cmd/uploader.py | 4 ++-- conans/test/integration/command/upload/upload_test.py | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 13a6f391061..5c236176eb4 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -6,7 +6,6 @@ from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument from conan.cli.commands.list import print_list_json, print_serial -from conans.client.downloaders.download_cache import DownloadCache from conans.client.userio import UserInput from conan.errors import ConanException diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index f0a4a4be771..ebe3d55953a 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -253,10 +253,10 @@ def _local_cache_file(self, pref, cache_file, remote): if cache_file not in approved_files: return _download_cache = self._app.global_conf.get("core.download:download_cache") - if True or _download_cache and os.path.isabs(_download_cache): + if _download_cache and os.path.isabs(_download_cache): url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.package_file(pref, cache_file) cached_path = DownloadCache(_download_cache).cached_path(url) - self._output.info(f"Caching future {url} to local cache '{cached_path}'") + self._output.info(f"Caching {cache_file} to local cache '{cached_path}' - was uploaded to {url}") def compress_files(files, name, dest_dir, compresslevel=None, ref=None): diff --git a/conans/test/integration/command/upload/upload_test.py b/conans/test/integration/command/upload/upload_test.py index c8b77f678fd..c3bcf5aaa68 100644 --- a/conans/test/integration/command/upload/upload_test.py +++ b/conans/test/integration/command/upload/upload_test.py @@ -238,6 +238,8 @@ def test_upload_unmodified_package(self): client.run("create . --user=frodo --channel=stable") client.run("upload hello0/1.2.1@frodo/stable -r default") + assert "Caching conan_package.tgz to local cache" in client.out + client2 = TestClient(servers=client.servers, inputs=["admin", "password"]) client2.save({"conanfile.py": conanfile, "hello.cpp": ""}) From 0bb98b47a3c63de18219f144104b457cf53a3158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n?= Date: Mon, 11 Dec 2023 18:39:07 +0100 Subject: [PATCH 3/5] Wip --- conans/client/cmd/uploader.py | 36 ++++++++++++++++--- .../integration/command/upload/upload_test.py | 3 +- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index ebe3d55953a..082a8d40a1a 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -10,7 +10,8 @@ from conans.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) from conans.util.files import (clean_dirty, is_dirty, gather_files, - gzopen_without_timestamps, set_dirty_context_manager, mkdir) + gzopen_without_timestamps, set_dirty_context_manager, mkdir, + remove_if_dirty) UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -233,6 +234,9 @@ def upload_recipe(self, ref, bundle, remote): duration = time.time() - t1 self._output.debug(f"Upload {ref} in {duration} time") + for cache_file in cache_files: + self._local_cache_recipe_file(ref, cache_file, remote) + return ref def upload_package(self, pref, prev_bundle, remote): @@ -246,17 +250,39 @@ def upload_package(self, pref, prev_bundle, remote): duration = time.time() - t1 self._output.debug(f"Upload {pref} in {duration} time") for cache_file in cache_files: - self._local_cache_file(pref, cache_file, remote) + self._local_cache_package_file(pref, cache_file, remote) - def _local_cache_file(self, pref, cache_file, remote): + def _local_cache_recipe_file(self, ref, cache_file, remote): + approved_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt"] + if cache_file not in approved_files: + return + _download_cache = self._app.global_conf.get("core.download:download_cache") + if _download_cache and os.path.isabs(_download_cache): + url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.recipe_file(ref, cache_file) + download_cache = DownloadCache(_download_cache) + cached_path, h = download_cache.cached_path(url) + with download_cache.lock(h): + remove_if_dirty(cached_path) + with set_dirty_context_manager(cached_path): + # shutil.copy2(cache_file, cached_path) + self._output.info( + f"Caching {cache_file} to local cache '{cached_path}' - was uploaded to {url}") + + def _local_cache_package_file(self, pref, cache_file, remote): approved_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt"] if cache_file not in approved_files: return _download_cache = self._app.global_conf.get("core.download:download_cache") if _download_cache and os.path.isabs(_download_cache): url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.package_file(pref, cache_file) - cached_path = DownloadCache(_download_cache).cached_path(url) - self._output.info(f"Caching {cache_file} to local cache '{cached_path}' - was uploaded to {url}") + download_cache = DownloadCache(_download_cache) + cached_path, h = download_cache.cached_path(url) + with download_cache.lock(h): + remove_if_dirty(cached_path) + with set_dirty_context_manager(cached_path): + # shutil.copy2(cache_file, cached_path) + self._output.info( + f"Caching {cache_file} to local cache '{cached_path}' - was uploaded to {url}") def compress_files(files, name, dest_dir, compresslevel=None, ref=None): diff --git a/conans/test/integration/command/upload/upload_test.py b/conans/test/integration/command/upload/upload_test.py index c3bcf5aaa68..caed4f411c9 100644 --- a/conans/test/integration/command/upload/upload_test.py +++ b/conans/test/integration/command/upload/upload_test.py @@ -14,8 +14,9 @@ from conans.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ TurboTestClient, GenConanfile, TestRequester, TestingResponse +from conans.test.utils.test_files import temp_folder from conans.util.files import gzopen_without_timestamps, is_dirty, save, set_dirty -from utils.test_files import temp_folder + conanfile = """from conan import ConanFile from conan.tools.files import copy From 07415ce23fb5feb21c7b1b980cbf9d3c6f9bad23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n?= Date: Tue, 12 Dec 2023 10:02:23 +0100 Subject: [PATCH 4/5] Actually cache the files --- conans/client/cmd/uploader.py | 31 +++++++++---------- .../integration/command/upload/upload_test.py | 5 ++- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index 082a8d40a1a..1622101cffb 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -234,7 +234,7 @@ def upload_recipe(self, ref, bundle, remote): duration = time.time() - t1 self._output.debug(f"Upload {ref} in {duration} time") - for cache_file in cache_files: + for cache_file in cache_files.items(): self._local_cache_recipe_file(ref, cache_file, remote) return ref @@ -249,40 +249,37 @@ def upload_package(self, pref, prev_bundle, remote): self._app.remote_manager.upload_package(pref, cache_files, remote) duration = time.time() - t1 self._output.debug(f"Upload {pref} in {duration} time") - for cache_file in cache_files: + for cache_file in cache_files.items(): self._local_cache_package_file(pref, cache_file, remote) def _local_cache_recipe_file(self, ref, cache_file, remote): approved_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt"] - if cache_file not in approved_files: + filename, path = cache_file + if filename not in approved_files: return _download_cache = self._app.global_conf.get("core.download:download_cache") - if _download_cache and os.path.isabs(_download_cache): - url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.recipe_file(ref, cache_file) - download_cache = DownloadCache(_download_cache) - cached_path, h = download_cache.cached_path(url) - with download_cache.lock(h): - remove_if_dirty(cached_path) - with set_dirty_context_manager(cached_path): - # shutil.copy2(cache_file, cached_path) - self._output.info( - f"Caching {cache_file} to local cache '{cached_path}' - was uploaded to {url}") + url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.recipe_file(ref, filename) + self._cache_local_upload_url(url, path) def _local_cache_package_file(self, pref, cache_file, remote): approved_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt"] - if cache_file not in approved_files: + filename, path = cache_file + if filename not in approved_files: return + url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.package_file(pref, filename) + self._cache_local_upload_url(url, path) + + def _cache_local_upload_url(self, url, path): _download_cache = self._app.global_conf.get("core.download:download_cache") if _download_cache and os.path.isabs(_download_cache): - url = self._app.remote_manager._auth_manager._get_rest_client(remote)._get_api().router.package_file(pref, cache_file) download_cache = DownloadCache(_download_cache) cached_path, h = download_cache.cached_path(url) with download_cache.lock(h): remove_if_dirty(cached_path) with set_dirty_context_manager(cached_path): - # shutil.copy2(cache_file, cached_path) + shutil.copy2(path, cached_path) self._output.info( - f"Caching {cache_file} to local cache '{cached_path}' - was uploaded to {url}") + f"Caching {path} to local cache '{cached_path}' - was uploaded to {url}") def compress_files(files, name, dest_dir, compresslevel=None, ref=None): diff --git a/conans/test/integration/command/upload/upload_test.py b/conans/test/integration/command/upload/upload_test.py index caed4f411c9..ba3b9f2795b 100644 --- a/conans/test/integration/command/upload/upload_test.py +++ b/conans/test/integration/command/upload/upload_test.py @@ -15,8 +15,7 @@ from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ TurboTestClient, GenConanfile, TestRequester, TestingResponse from conans.test.utils.test_files import temp_folder -from conans.util.files import gzopen_without_timestamps, is_dirty, save, set_dirty - +from conans.util.files import gzopen_without_timestamps, is_dirty, save, set_dirty, load conanfile = """from conan import ConanFile from conan.tools.files import copy @@ -239,7 +238,7 @@ def test_upload_unmodified_package(self): client.run("create . --user=frodo --channel=stable") client.run("upload hello0/1.2.1@frodo/stable -r default") - assert "Caching conan_package.tgz to local cache" in client.out + assert len(os.listdir(os.path.join(download_cache, "c"))) == 5 client2 = TestClient(servers=client.servers, inputs=["admin", "password"]) client2.save({"conanfile.py": conanfile, From d948be274fafc569920e8ebd795cf785872d2165 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 13 Sep 2024 01:58:32 +0200 Subject: [PATCH 5/5] review minor --- conan/internal/api/uploader.py | 1 - test/integration/command/upload/upload_test.py | 1 - 2 files changed, 2 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 0b988b9b1b1..ceb1e690825 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -256,7 +256,6 @@ def upload_package(self, pref, prev_bundle, remote): t1 = time.time() self._app.remote_manager.upload_package(pref, cache_files, remote) duration = time.time() - t1 - output.debug(f"Upload {pref} in {duration} time") self._cache_local(cache_files, remote, pref) diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 6c25a90bc8d..c6eea7e19aa 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -239,7 +239,6 @@ def test_upload_unmodified_package(self): ref = RecipeReference.loads("hello0/1.2.1@frodo/stable") client.run("create . --user=frodo --channel=stable") client.run("upload hello0/1.2.1@frodo/stable -r default") - print(client.out) assert len(os.listdir(os.path.join(download_cache, "c"))) == 6 client2 = TestClient(servers=client.servers, inputs=["admin", "password"])