From d2a93907a941ff1dda86f8dfcbf661782fb9a1bd Mon Sep 17 00:00:00 2001 From: Igor Gatis Date: Mon, 8 Mar 2021 17:11:26 +0000 Subject: [PATCH] Allows container_push to postpone image download to build phase. --- container/go/cmd/puller/puller.go | 15 +++++--- container/go/pkg/compat/write.go | 6 ++-- container/pull.bzl | 60 ++++++++++++++++++++++++++----- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/container/go/cmd/puller/puller.go b/container/go/cmd/puller/puller.go index 630af9e39..4806115b9 100644 --- a/container/go/cmd/puller/puller.go +++ b/container/go/cmd/puller/puller.go @@ -53,6 +53,7 @@ var ( variant = flag.String("variant", "", "Image's CPU variant, if referring to a multi-platform manifest list.") features = flag.String("features", "", "Image's CPU features, if referring to a multi-platform manifest list.") timeout = flag.Int("timeout", 600, "Timeout in seconds for the puller. e.g., --timeout=1000 for a 1000 second timeout.") + metadataOnly = flag.Bool("metadata-only", false, "Fetches only the metadata such as the list of layers for this image.") ) // Tag applied to images that were pulled by digest. This denotes @@ -84,7 +85,7 @@ func getTag(ref name.Reference) name.Reference { // copy of the image will be loaded from the given cache path if available. If // the given image name points to a list of images, the given platform will // be used to select the image to pull. -func pull(imgName, dstPath, cachePath string, platform v1.Platform, transport *http.Transport) error { +func pull(imgName, dstPath, cachePath string, metadataOnly bool, platform v1.Platform, transport *http.Transport) error { // Get a digest/tag based on the name. ref, err := name.ParseReference(imgName) if err != nil { @@ -100,8 +101,14 @@ func pull(imgName, dstPath, cachePath string, platform v1.Platform, transport *h img = cache.Image(img, cache.NewFilesystemCache(cachePath)) } - if err := compat.WriteImage(img, dstPath); err != nil { - return errors.Wrapf(err, "unable to save remote image %v", ref) + if metadataOnly { + if err := compat.WriteImageMetadata(img, dstPath); err != nil { + return errors.Wrapf(err, "unable to save remote image metadata %v", ref) + } + } else { + if err := compat.WriteImage(img, dstPath); err != nil { + return errors.Wrapf(err, "unable to save remote image %v", ref) + } } return nil @@ -154,7 +161,7 @@ func main() { Proxy: http.ProxyFromEnvironment, } - if err := pull(*imgName, *directory, *cachePath, platform, t); err != nil { + if err := pull(*imgName, *directory, *cachePath, *metadataOnly, platform, t); err != nil { log.Fatalf("Image pull was unsuccessful: %v", err) } diff --git a/container/go/pkg/compat/write.go b/container/go/pkg/compat/write.go index fdfa9e460..d83ee064b 100644 --- a/container/go/pkg/compat/write.go +++ b/container/go/pkg/compat/write.go @@ -35,13 +35,13 @@ func isCompressed(m types.MediaType) bool { return m == types.DockerLayer || m == types.OCILayer } -// writeImageMetadata generates the following files in the given directory +// WriteImageMetadata generates the following files in the given directory // for the given image: // directory/ // config.json <-- only *.json, the image's config // digest <-- sha256 digest of the image's manifest // manifest.json <-- the image's manifest -func writeImageMetadata(img v1.Image, outDir string) error { +func WriteImageMetadata(img v1.Image, outDir string) error { c, err := img.RawConfigFile() if err != nil { return errors.Wrap(err, "unable to get raw config from image") @@ -144,7 +144,7 @@ func writeImageLayers(img v1.Image, outDir string) error { // We pad layer indices to only 3 digits because of a known ceiling on the number // of filesystem layers Docker supports. func WriteImage(img v1.Image, outDir string) error { - if err := writeImageMetadata(img, outDir); err != nil { + if err := WriteImageMetadata(img, outDir); err != nil { return errors.Wrap(err, "unable to write image metadata") } if err := writeImageLayers(img, outDir); err != nil { diff --git a/container/pull.bzl b/container/pull.bzl index d8dbb544c..3696e0b9b 100644 --- a/container/pull.bzl +++ b/container/pull.bzl @@ -96,6 +96,10 @@ _container_pull_attrs = { doc = "(optional) The tag of the image, default to 'latest' " + "if this and 'digest' remain unspecified.", ), + "lazy_download": attr.bool( + default = False, + doc = "(optional) Indicates whether donwload is postponed to build phase.", + ), } def _impl(repository_ctx): @@ -120,9 +124,6 @@ def _impl(repository_ctx): puller = repository_ctx.attr.puller_linux_s390x args = [ - repository_ctx.path(puller), - "-directory", - repository_ctx.path("image"), "-os", repository_ctx.attr.os, "-os-version", @@ -184,16 +185,24 @@ def _impl(repository_ctx): else: fail("'%s' is invalid value for PULLER_TIMEOUT. Must be an integer." % (timeout_in_secs)) - result = repository_ctx.execute(args, **kwargs) - if result.return_code: - fail("Pull command failed: %s (%s)" % (result.stderr, " ".join([str(a) for a in args]))) - updated_attrs = { k: getattr(repository_ctx.attr, k) for k in _container_pull_attrs.keys() } updated_attrs["name"] = repository_ctx.name + fetch_args = [ + repository_ctx.path(puller), + "-directory", + repository_ctx.path("image"), + ] + args + + if repository_ctx.attr.lazy_download: + fetch_args += ["-metadata-only"] + + result = repository_ctx.execute(fetch_args, **kwargs) + if result.return_code: + fail("Pull command failed: %s (%s)" % (result.stderr, " ".join([str(a) for a in fetch_args]))) updated_attrs["digest"] = repository_ctx.read(repository_ctx.path("image/digest")) if repository_ctx.attr.digest and repository_ctx.attr.digest != updated_attrs["digest"]: @@ -210,13 +219,44 @@ def _impl(repository_ctx): # foo.digest for an image named foo. repository_ctx.symlink(repository_ctx.path("image/digest"), repository_ctx.path("image/image.digest")) + tar_gz_files = [] + sha256_files = [] + config = json.decode(repository_ctx.read(repository_ctx.path("image/config.json"))) + for i in range(len(config["rootfs"]["diff_ids"])): + layer_id = "{}".format(i) + layer_id = "000"[0:3-len(layer_id)] + layer_id + tar_gz_files.append("{}.tar.gz".format(layer_id)) + sha256_files.append("{}.sha256".format(layer_id)) + + download_target = "" + if repository_ctx.attr.lazy_download: + later_fetch_args = [ + "$(location {})".format(puller), + "-directory", + "$(RULEDIR)", + ] + args + + download_target = """ +genrule( + name = "download", + message = "Fetching", + outs = ["{outs}"], + cmd = "{args}", + tools = ["{tool}"], +) +""".format( + outs = "\", \"".join(tar_gz_files + sha256_files), + args = " ".join(later_fetch_args), + tool = "{}".format(puller), + ) + repository_ctx.file("image/BUILD", """package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_docker//container:import.bzl", "container_import") - +{download_hook} container_import( name = "image", config = "config.json", - layers = glob(["*.tar.gz"]), + layers = ["{layers}"], base_image_registry = "{registry}", base_image_repository = "{repository}", base_image_digest = "{digest}", @@ -225,6 +265,8 @@ container_import( exports_files(["image.digest", "digest"]) """.format( + download_hook = download_target, + layers = "\", \"".join(tar_gz_files), registry = updated_attrs["registry"], repository = updated_attrs["repository"], digest = updated_attrs["digest"],