[WIP] Chroot tooling

Dockerfile.chroot

@@ -0,0 +1,11 @@
+FROM golang:1.12.1
+COPY --from=gcr.io/makisu-project/makisu:v0.1.10 /makisu-internal/makisu /usr/local/bin/makisu
+
+ADD . /workspace
+WORKDIR /workspace
+
+ENV SSL_CERT_DIR=/etc/ssl/certs
+RUN ./tools/scripts/makisu_chroot.sh build -t test:121 --modifyfs=true testdata/build-context/symlink
+RUN ./tools/scripts/makisu_chroot.sh build -t test:121 --modifyfs=true testdata/build-context/user-change
+RUN ./tools/scripts/makisu_chroot.sh build -t test:121 --modifyfs=true testdata/build-context/chroot-simple
+RUN ./tools/scripts/makisu_chroot.sh build -t test:121 --modifyfs=true testdata/build-context/go-from-scratch

bin/makisu/cmd/build.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"syscall"
 	"time"
 
 	"github.com/uber/makisu/lib/builder"
@@ -67,6 +68,7 @@ type buildCmd struct {
 	compressionLevel string
 
 	preserveRoot bool
+	chroot       string
 }
 
 func getBuildCmd() *buildCmd {
@@ -88,7 +90,6 @@ func getBuildCmd() *buildCmd {
 			log.Errorf("failed to process flags: %s", err)
 			os.Exit(1)
 		}
-
 		if err := buildCmd.Build(args[0]); err != nil {
 			log.Error(err)
 			os.Exit(1)
@@ -124,6 +125,7 @@ func getBuildCmd() *buildCmd {
 	buildCmd.PersistentFlags().StringVar(&buildCmd.compressionLevel, "compression", "default", "Image compression level, could be 'no', 'speed', 'size', 'default'")
 
 	buildCmd.PersistentFlags().BoolVar(&buildCmd.preserveRoot, "preserve-root", false, "Copy / in the storage dir and copy it back after build.")
+	rootCmd.PersistentFlags().StringVar(&buildCmd.chroot, "chroot", "", "Executes the command in a chrooted environment.")
 
 	buildCmd.MarkFlagRequired("tag")
 	buildCmd.Flags().SortFlags = false
@@ -216,6 +218,12 @@ func (cmd *buildCmd) newBuildPlan(
 func (cmd *buildCmd) Build(contextDir string) error {
 	log.Infof("Starting Makisu build (version=%s)", utils.BuildHash)
 
+	if cmd.chroot != "" {
+		if err := cmd.prepareChroot(contextDir); err != nil {
+			return fmt.Errorf("failed to prepare chroot environment: %s", err)
+		}
+	}
+
 	// Create BuildContext.
 	contextDirAbs, err := filepath.Abs(contextDir)
 	if err != nil {
@@ -298,3 +306,29 @@ func (cmd *buildCmd) Build(contextDir string) error {
 	log.Infof("Finished building %s", imageName.ShortName())
 	return nil
 }
+
+func (cmd *buildCmd) prepareChroot(context string) error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("cannot prepare chroot on %s", runtime.GOOS)
+	}
+
+	dest, err := filepath.Abs(cmd.chroot)
+	if err != nil {
+		return fmt.Errorf("failed to convert chroot path to absolute: %s", err)
+	} else if _, err := os.Lstat(dest); err == nil || !os.IsNotExist(err) {
+		return fmt.Errorf("chroot target must not exist: %s", err)
+	}
+
+	certs := filepath.Join(cmd.chroot, pathutils.DefaultInternalDir, "certs")
+	if err := os.MkdirAll(certs, 0644); err != nil {
+		return fmt.Errorf("failed to create chroot cert dir: %s", err)
+	}
+	// TODO: copy all certs into the chroot environment.
+
+	dev := filepath.Join(cmd.chroot, "dev")
+	if err := os.MkdirAll(dev, 0644); err != nil {
+		return fmt.Errorf("failed to create chroot /dev dir: %s", err)
+	}
+	// TODO: Create all nods
+	return syscall.Chroot(dest)
+}

testdata/build-context/chroot-simple/Dockerfile

@@ -0,0 +1,3 @@
+FROM debian:9
+RUN echo "ASD" > /asd
+RUN cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1

tools/scripts/makisu_chroot.sh

@@ -0,0 +1,62 @@
+#! /bin/bash
+
+set -e
+
+[ -z "$1" ] && echo "Must at least provide context of build" && exit 1
+
+CHROOT=${CHROOT_LOCATION:-$HOME/.makisu-chroot-$RANDOM}
+SSL_CERT_DIR=${SSL_CERTS:-/etc/ssl/certs}
+CONTEXT=${@: -1}
+BUILD_VOLUMES="$CONTEXT:/context,$BUILD_VOLUMES"
+
+function makisu::prepare_internals () {
+    mkdir -p $CHROOT/makisu-internal/certs
+    cp $(which makisu) $CHROOT/makisu-internal/makisu
+    cat $SSL_CERT_DIR/* > $CHROOT/makisu-internal/certs/cacerts.pem
+}
+
+function makisu::prepare_dev () {
+    mkdir -p $CHROOT/dev $CHROOT/shm
+    mknod -m 622 $CHROOT/dev/console c 5 1
+    mknod -m 622 $CHROOT/dev/initctl p
+    mknod -m 666 $CHROOT/dev/full c 1 7
+    mknod -m 666 $CHROOT/dev/null c 1 3
+    mknod -m 666 $CHROOT/dev/ptmx c 5 2
+    mknod -m 666 $CHROOT/dev/random c 1 8
+    mknod -m 666 $CHROOT/dev/tty c 5 0
+    mknod -m 666 $CHROOT/dev/tty0 c 4 0
+    mknod -m 666 $CHROOT/dev/urandom c 1 9
+    mknod -m 666 $CHROOT/dev/zero c 1 5
+    chown root:tty $CHROOT/dev/{console,ptmx,tty}
+
+    # https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/contrib/mkimage-crux.sh
+    # https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/contrib/mkimage-arch.sh
+    # https://github.com/moby/moby/blob/d7ab8ad145fad4c63112f34721663021e5b02707/contrib/mkimage-yum.sh
+}
+
+function makisu::prepare_etc () {
+    mkdir -p $CHROOT/etc
+    cp /etc/*.conf $CHROOT/etc/
+}
+
+function makisu::prepare_volumes () {
+    for vol in $(sed "s/,/ /g" <<< $BUILD_VOLUMES); do
+        from=$(cut -d ':' -f 1 <<< $vol)
+        to=$(cut -d ':' -f 2 <<< $vol)
+        echo "Copying volume $from to chroot directory $CHROOT/$to"
+        mkdir -p $CHROOT/$to
+        cp -r $from/* $CHROOT/$to
+    done
+}
+
+echo "Preparing chroot at $CHROOT"
+rm -rf $CHROOT
+
+makisu::prepare_internals
+makisu::prepare_etc
+makisu::prepare_dev
+makisu::prepare_volumes
+
+makisu_args=${@:1:$#-1}
+echo "Starting Makisu: makisu $makisu_args /context"
+chroot $CHROOT/ /makisu-internal/makisu $makisu_args /context