Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sketch to show that populating download cache on conan upload is possible #15223

Draft
wants to merge 6 commits into
base: develop2
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion conan/api/subapi/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def prepare(self, package_list, enabled_remotes, metadata=None):
def upload(self, package_list, remote):
app = ConanApp(self.conan_api)
app.remote_manager.check_credentials(remote)
executor = UploadExecutor(app)
executor = UploadExecutor(app, self.conan_api.config.global_conf)
executor.upload(package_list, remote)

def upload_full(self, package_list, remote, enabled_remotes, check_integrity=False, force=False,
Expand Down
27 changes: 25 additions & 2 deletions conan/internal/api/uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,17 @@

from conan.internal.conan_app import ConanApp
from conan.api.output import ConanOutput
from conans.client.downloaders.download_cache import DownloadCache
from conans.client.rest.client_routes import ClientV2Router
from conans.client.source import retrieve_exports_sources
from conans.errors import ConanException, NotFoundException
from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
from conans.model.recipe_ref import RecipeReference
from conans.util.files import (clean_dirty, is_dirty, gather_files,
gzopen_without_timestamps, set_dirty_context_manager, mkdir,
human_size)
remove_if_dirty, human_size)


UPLOAD_POLICY_FORCE = "force-upload"
UPLOAD_POLICY_SKIP = "skip-upload"
Expand Down Expand Up @@ -215,8 +219,9 @@ class UploadExecutor:
been computed and are passed in the ``upload_data`` parameter, so this executor is also
agnostic about which files are transferred
"""
def __init__(self, app: ConanApp):
def __init__(self, app: ConanApp, global_conf):
self._app = app
self._download_cache = global_conf.get("core.download:download_cache")

def upload(self, upload_data, remote):
for ref, bundle in upload_data.refs().items():
Expand All @@ -237,6 +242,7 @@ def upload_recipe(self, ref, bundle, remote):

duration = time.time() - t1
output.debug(f"Upload {ref} in {duration} time")
self._cache_local(cache_files, remote, ref)
return ref

def upload_package(self, pref, prev_bundle, remote):
Expand All @@ -251,6 +257,23 @@ def upload_package(self, pref, prev_bundle, remote):
self._app.remote_manager.upload_package(pref, cache_files, remote)
duration = time.time() - t1
output.debug(f"Upload {pref} in {duration} time")
self._cache_local(cache_files, remote, pref)

def _cache_local(self, cache_files, remote, ref):
if self._download_cache is None:
return
# It will raise in other place if not absolute
download_cache = DownloadCache(self._download_cache)
router = ClientV2Router(remote.url)
for cache_file, path in cache_files.items():
url = router.recipe_file(ref, cache_file) if isinstance(ref, RecipeReference) else \
router.package_file(ref, cache_file)
cached_path, h = download_cache.cached_path(url)
with download_cache.lock(h):
remove_if_dirty(cached_path)
with set_dirty_context_manager(cached_path):
shutil.copy2(path, cached_path)
ConanOutput().verbose(f"Caching uploaded file '{cache_file}' in download cache")


def compress_files(files, name, dest_dir, compresslevel=None, ref=None):
Expand Down
4 changes: 4 additions & 0 deletions test/integration/command/upload/upload_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from mock import patch
from requests import Response

from conan.test.utils.test_files import temp_folder
from conans.errors import ConanException
from conans.model.package_ref import PkgReference
from conans.model.recipe_ref import RecipeReference
Expand Down Expand Up @@ -233,9 +234,12 @@ def test_upload_unmodified_package(self):

client.save({"conanfile.py": conanfile,
"hello.cpp": ""})
download_cache = temp_folder()
client.save_home({"global.conf": f"core.download:download_cache={download_cache}"})
ref = RecipeReference.loads("hello0/1.2.1@frodo/stable")
client.run("create . --user=frodo --channel=stable")
client.run("upload hello0/1.2.1@frodo/stable -r default")
assert len(os.listdir(os.path.join(download_cache, "c"))) == 6

client2 = TestClient(servers=client.servers, inputs=["admin", "password"])
client2.save({"conanfile.py": conanfile,
Expand Down