Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allows container_pull to postpone image download to build phase. #1749

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+65 −16
Open

Allows container_pull to postpone image download to build phase. #1749

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions container/go/cmd/puller/puller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ var (
variant = flag.String("variant", "", "Image's CPU variant, if referring to a multi-platform manifest list.")
features = flag.String("features", "", "Image's CPU features, if referring to a multi-platform manifest list.")
timeout = flag.Int("timeout", 600, "Timeout in seconds for the puller. e.g., --timeout=1000 for a 1000 second timeout.")
metadataOnly = flag.Bool("metadata-only", false, "Fetches only the metadata such as the list of layers for this image.")
)

// Tag applied to images that were pulled by digest. This denotes
Expand Down Expand Up @@ -84,7 +85,7 @@ func getTag(ref name.Reference) name.Reference {
// copy of the image will be loaded from the given cache path if available. If
// the given image name points to a list of images, the given platform will
// be used to select the image to pull.
func pull(imgName, dstPath, cachePath string, platform v1.Platform, transport *http.Transport) error {
func pull(imgName, dstPath, cachePath string, metadataOnly bool, platform v1.Platform, transport *http.Transport) error {
// Get a digest/tag based on the name.
ref, err := name.ParseReference(imgName)
if err != nil {
Expand All @@ -100,8 +101,14 @@ func pull(imgName, dstPath, cachePath string, platform v1.Platform, transport *h
img = cache.Image(img, cache.NewFilesystemCache(cachePath))
}

if err := compat.WriteImage(img, dstPath); err != nil {
return errors.Wrapf(err, "unable to save remote image %v", ref)
if metadataOnly {
if err := compat.WriteImageMetadata(img, dstPath); err != nil {
return errors.Wrapf(err, "unable to save remote image metadata %v", ref)
}
} else {
if err := compat.WriteImage(img, dstPath); err != nil {
return errors.Wrapf(err, "unable to save remote image %v", ref)
}
}

return nil
Expand Down Expand Up @@ -154,7 +161,7 @@ func main() {
Proxy: http.ProxyFromEnvironment,
}

if err := pull(*imgName, *directory, *cachePath, platform, t); err != nil {
if err := pull(*imgName, *directory, *cachePath, *metadataOnly, platform, t); err != nil {
log.Fatalf("Image pull was unsuccessful: %v", err)
}

Expand Down
6 changes: 3 additions & 3 deletions container/go/pkg/compat/write.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,13 @@ func isCompressed(m types.MediaType) bool {
return m == types.DockerLayer || m == types.OCILayer
}

// writeImageMetadata generates the following files in the given directory
// WriteImageMetadata generates the following files in the given directory
// for the given image:
// directory/
// config.json <-- only *.json, the image's config
// digest <-- sha256 digest of the image's manifest
// manifest.json <-- the image's manifest
func writeImageMetadata(img v1.Image, outDir string) error {
func WriteImageMetadata(img v1.Image, outDir string) error {
c, err := img.RawConfigFile()
if err != nil {
return errors.Wrap(err, "unable to get raw config from image")
Expand Down Expand Up @@ -144,7 +144,7 @@ func writeImageLayers(img v1.Image, outDir string) error {
// We pad layer indices to only 3 digits because of a known ceiling on the number
// of filesystem layers Docker supports.
func WriteImage(img v1.Image, outDir string) error {
if err := writeImageMetadata(img, outDir); err != nil {
if err := WriteImageMetadata(img, outDir); err != nil {
return errors.Wrap(err, "unable to write image metadata")
}
if err := writeImageLayers(img, outDir); err != nil {
Expand Down
60 changes: 51 additions & 9 deletions container/pull.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ _container_pull_attrs = {
doc = "(optional) The tag of the image, default to 'latest' " +
"if this and 'digest' remain unspecified.",
),
"lazy_download": attr.bool(
default = False,
doc = "(optional) Indicates whether donwload is postponed to build phase.",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Spelling

),
}

def _impl(repository_ctx):
Expand All @@ -120,9 +124,6 @@ def _impl(repository_ctx):
puller = repository_ctx.attr.puller_linux_s390x

args = [
repository_ctx.path(puller),
"-directory",
repository_ctx.path("image"),
"-os",
repository_ctx.attr.os,
"-os-version",
Expand Down Expand Up @@ -184,16 +185,24 @@ def _impl(repository_ctx):
else:
fail("'%s' is invalid value for PULLER_TIMEOUT. Must be an integer." % (timeout_in_secs))

result = repository_ctx.execute(args, **kwargs)
if result.return_code:
fail("Pull command failed: %s (%s)" % (result.stderr, " ".join([str(a) for a in args])))

updated_attrs = {
k: getattr(repository_ctx.attr, k)
for k in _container_pull_attrs.keys()
}
updated_attrs["name"] = repository_ctx.name

fetch_args = [
repository_ctx.path(puller),
"-directory",
repository_ctx.path("image"),
] + args

if repository_ctx.attr.lazy_download:
fetch_args += ["-metadata-only"]

result = repository_ctx.execute(fetch_args, **kwargs)
if result.return_code:
fail("Pull command failed: %s (%s)" % (result.stderr, " ".join([str(a) for a in fetch_args])))
updated_attrs["digest"] = repository_ctx.read(repository_ctx.path("image/digest"))

if repository_ctx.attr.digest and repository_ctx.attr.digest != updated_attrs["digest"]:
Expand All @@ -210,13 +219,44 @@ def _impl(repository_ctx):
# foo.digest for an image named foo.
repository_ctx.symlink(repository_ctx.path("image/digest"), repository_ctx.path("image/image.digest"))

tar_gz_files = []
sha256_files = []
config = json.decode(repository_ctx.read(repository_ctx.path("image/config.json")))
for i in range(len(config["rootfs"]["diff_ids"])):
layer_id = "{}".format(i)
layer_id = "000"[0:3-len(layer_id)] + layer_id
tar_gz_files.append("{}.tar.gz".format(layer_id))
sha256_files.append("{}.sha256".format(layer_id))

download_target = ""
if repository_ctx.attr.lazy_download:
later_fetch_args = [
"$(location {})".format(puller),
"-directory",
"$(RULEDIR)",
] + args

download_target = """
genrule(
name = "download",
message = "Fetching",
outs = ["{outs}"],
cmd = "{args}",
tools = ["{tool}"],
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
tools = ["{tool}"],
tools = ["{tool}"],
tags = ["requires-network"],

to make sure network is allowed even in build sandbox

)
""".format(
outs = "\", \"".join(tar_gz_files + sha256_files),
args = " ".join(later_fetch_args),
tool = "{}".format(puller),
)

repository_ctx.file("image/BUILD", """package(default_visibility = ["//visibility:public"])
load("@io_bazel_rules_docker//container:import.bzl", "container_import")

{download_hook}
container_import(
name = "image",
config = "config.json",
layers = glob(["*.tar.gz"]),
layers = ["{layers}"],
base_image_registry = "{registry}",
base_image_repository = "{repository}",
base_image_digest = "{digest}",
Expand All @@ -225,6 +265,8 @@ container_import(

exports_files(["image.digest", "digest"])
""".format(
download_hook = download_target,
layers = "\", \"".join(tar_gz_files),
registry = updated_attrs["registry"],
repository = updated_attrs["repository"],
digest = updated_attrs["digest"],
Expand Down