Create repo with symbolic links

.gitignore

@@ -14,3 +14,4 @@ test/tmp
 !test/fixtures/*
 debian/freight*
 debian/files
+sh

Makefile

@@ -95,5 +95,6 @@ test/tmp/bin/sh: test/tmp/bin
 
 check: test/tmp/bats test/tmp/bats-assert test/tmp/bin/sh
 	PATH=test/tmp/bin/:$$PATH test/tmp/bats/bin/bats test/
+	PATH=test/tmp/bin/:$$PATH FREIGHT_TEST_VARPOOL=1 test/tmp/bats/bin/bats test/
 
 .PHONY: all clean install uninstall build man docs gh-pages check

bin/freight-add

@@ -4,10 +4,11 @@
 # Add a package to the Freight library.
 
 #/ Usage: freight add [-c <conf>] [-v] [-h] <package> <manager>/<distro>...
-#/   -c <conf>, --conf=<conf> config file to parse
-#/   -v, --verbose            verbose mode
-#/   -e, --add-error          exit with error if package already added
-#/   -h, --help               show this help message
+#/   -c <conf>, --conf=<conf>           config file to parse
+#/   -s <subdir>, --subpool=<subdir>    store the file to a subdirectory in symbolic pool
+#/   -v, --verbose                      verbose mode
+#/   -e, --add-error                    exit with error if package already added
+#/   -h, --help                         show this help message
 
 set -e
 
@@ -20,6 +21,9 @@ while [ "$#" -gt 0 ]; do
         -c | --conf) CONF="$2" shift 2 ;;
         -c*) CONF="$(echo "$1" | cut -c"3-")" shift ;;
         --conf=*) CONF="$(echo "$1" | cut -c"8-")" shift ;;
+        -s | --subpool) SUBPOOL="$2" shift 2 ;;
+        -s*) SUBPOOL="$(echo "$1" | cut -c"3-")" shift ;;
+        --subpool=*) SUBPOOL="$(echo "$1" | cut -c"11-")" shift ;;
         -v | --verbose) VERBOSE=1 shift ;;
         -e | --add-error) FREIGHT_ADD_ERROR="YES" shift ;;
         -h | --help) usage 0 ;;
@@ -33,6 +37,10 @@ done
 
 . "$(dirname "$(dirname "$0")")/lib/freight/conf.sh"
 
+if [ -n "$VARPOOL" ] && [ -n "$SUBPOOL" ]; then
+    VARPOOL="$VARPOOL/$SUBPOOL"
+fi
+
 # The non-option argument(s) following the last option are package files.
 # Binary packages have only one but source packages require two or three.
 # When the last of these is found, the remaining arguments are each assumed
@@ -50,6 +58,20 @@ done
 [ -z "$PATHNAMES" ] && usage 1
 [ -z "$*" ] && usage 1
 
+# If we are using symbolic pool, first copy all the packages to this location
+if [ -n "$VARPOOL" ]; then
+    mkdir -p "$VARPOOL"
+    for PATHNAME in $PATHNAMES; do
+        # cp is not atomic, so cp to a tmp name, then mv (which is atomic)
+        # TO-DO: what if packages with the same name and version
+        #        are actually different in differen distros
+        #        when they need to link external libraries?
+        TMP="$(mktemp "$VARPOOL/freight.$$.XXXXXXXXXX")"
+        cp "$PATHNAME" "$TMP"
+        mv "$TMP" "$VARPOOL/$(basename "$PATHNAME")"
+    done
+fi
+
 # Create a working directory on the same device as the Freight library and
 # copy this package there.  This used to be a link but is now a copy because
 # later Freight commands will rely on the link count being reduced to one.
@@ -58,7 +80,12 @@ TMP="$(mktemp -d "$VARLIB/freight.$$.XXXXXXXXXX")"
 # shellcheck disable=SC2064
 trap "rm -rf \"$TMP\"" EXIT INT TERM
 for PATHNAME in $PATHNAMES; do
-    cp "$PATHNAME" "$TMP/"
+    if [ -z "$VARPOOL" ]; then
+        cp "$PATHNAME" "$TMP/"
+    else
+        FILENAME="$(basename "$PATHNAME")"
+        ln "$LINK_OPTION" "$VARPOOL/$FILENAME" "$TMP/"
+    fi
 done
 
 # Enter the Freight library directory so that items in `$@` may be given as
@@ -70,7 +97,7 @@ cd "$VARLIB"
 # PATHNAME.  The first two are given fairly directly to ln(1); the final one
 # is used to create appropriate success or failure messages.
 add() {
-    if ln "$TMP/$1" "$2/$1" 2>"$TMP/ln"; then
+    if ln "$LINK_OPTION" "$TMP/$1" "$2/$1" 2>"$TMP/ln"; then
         echo "# [freight] added $3 to $2${4+" as "}$4" >&2
     else
         if grep -q "File exists" "$TMP/ln"; then
@@ -84,7 +111,7 @@ add() {
     fi
 }
 
-# Hard link this package into every `<manager>/<distro>` given in `$@`.
+# Link this package into every `<manager>/<distro>` given in `$@`.
 # These links will later be used to compile the `Release` and `Packages`
 # files in the Freight cache.
 for PATHNAME in $PATHNAMES; do

bin/freight-init

@@ -7,6 +7,7 @@
 #/   -c<conf>, --conf=<conf>  config file to create (default etc/freight.conf)
 #/   --libdir=<libdir>        library directory (default var/lib/freight)
 #/   --cachedir=<cachedir>    cache directory (default var/cache/freight)
+#/   --pooldir=<pooldir>      symbolic pool directory
 #/   --archs=<archs>          architectures to support (default "i386 amd64")
 #/   --origin=<origin>        Debian archive Origin field (default "Freight")
 #/   --label=<label>          Debian archive Label field (default "Freight")
@@ -56,6 +57,8 @@ while [ "$#" -gt 0 ]; do
         --libdir=*) VARLIB="$(echo "$1" | cut -c"10-")" shift ;;
         --cachedir) VARCACHE="$2" shift 2 ;;
         --cachedir=*) VARCACHE="$(echo "$1" | cut -c"12-")" shift ;;
+        --pooldir) VARPOOL="$2" shift 2 ;;
+        --pooldir=*) VARPOOL="$(echo "$1" | cut -c"12-")" shift ;;
         --archs) ARCHS="$2" shift 2 ;;
         --archs=*) ARCHS="$(echo "$1" | cut -c"9-")" shift ;;
         --origin) ORIGIN="$2" shift 2 ;;
@@ -93,6 +96,7 @@ VARLIB="$VARLIB"
 VARCACHE="$VARCACHE"
 GPG="$GPG"
 EOF
+[ "$VARPOOL" ] && echo "VARPOOL=\"$VARPOOL\"" >>"$CONF"
 [ "$ARCHS" ] && echo "ARCHS=\"$ARCHS\"" >>"$CONF"
 [ "$ORIGIN" ] && echo "ORIGIN=\"$ORIGIN\"" >>"$CONF"
 [ "$LABEL" ] && echo "LABEL=\"$LABEL\"" >>"$CONF"

etc/freight.conf.example

@@ -5,6 +5,11 @@
 VARLIB="/var/lib/freight"
 VARCACHE="/var/cache/freight"
 
+# If the file system does not support hard link, you can use symbolic
+# link instead. In this mode file will be copied to VARPOOL first, then
+# symlinked to VARLIB and finally VARCACHE. Implies SYMLINKS="on".
+VARPOOL=""
+
 # Default `Origin`, `Label`, `NotAutomatic`, and
 # `ButAutomaticUpgrades` fields for `Release` files.
 ORIGIN="Freight"

lib/freight/apt.sh

@@ -337,7 +337,7 @@ EOF
 
     # Link or copy this package into this distro's `.refs` directory.
     mkdir -p "$DISTCACHE/.refs/$COMP"
-    ln "$VARLIB/apt/$DIST/$PATHNAME" "$DISTCACHE/.refs/$COMP" >/dev/null 2>&1 ||
+    ln "$LINK_OPTION" "$VARLIB/apt/$DIST/$PATHNAME" "$DISTCACHE/.refs/$COMP" >/dev/null 2>&1 ||
         cp "$VARLIB/apt/$DIST/$PATHNAME" "$DISTCACHE/.refs/$COMP"
 
     # Package properties.  Remove the epoch from the version number
@@ -358,7 +358,7 @@ EOF
         else
             echo "# [freight] adding $PACKAGE to pool" >&2
         fi
-        ln "$DISTCACHE/.refs/$COMP/$PACKAGE" "$VARCACHE/$POOL/$FILENAME"
+        ln "$LINK_OPTION" "$DISTCACHE/.refs/$COMP/$PACKAGE" "$VARCACHE/$POOL/$FILENAME"
     fi
 
     # Build a list of the one-or-more `Packages` files to append with
@@ -439,7 +439,7 @@ apt_cache_source() {
     for FILENAME in "$DSC_FILENAME" "$ORIG_FILENAME" "$DIFF_FILENAME" "$TAR_FILENAME" "$GIT_FILENAME"; do
         [ -f "$VARLIB/apt/$DIST/$DIRNAME/$FILENAME" ] || continue
         [ -f "$DISTCACHE/.refs/$COMP/$FILENAME" ] ||
-            ln "$VARLIB/apt/$DIST/$DIRNAME/$FILENAME" "$DISTCACHE/.refs/$COMP" >/dev/null 2>&1 ||
+            ln "$LINK_OPTION" "$VARLIB/apt/$DIST/$DIRNAME/$FILENAME" "$DISTCACHE/.refs/$COMP" >/dev/null 2>&1 ||
             cp "$VARLIB/apt/$DIST/$DIRNAME/$FILENAME" "$DISTCACHE/.refs/$COMP"
     done
 
@@ -452,7 +452,7 @@ apt_cache_source() {
     for FILENAME in "$DSC_FILENAME" "$ORIG_FILENAME" "$DIFF_FILENAME" "$TAR_FILENAME" "$GIT_FILENAME"; do
         if [ -f "$DISTCACHE/.refs/$COMP/$FILENAME" ] && ! [ -f "$VARCACHE/$POOL/$FILENAME" ]; then
             echo "# [freight] adding $FILENAME to pool" >&2
-            ln "$DISTCACHE/.refs/$COMP/$FILENAME" "$VARCACHE/$POOL"
+            ln "$LINK_OPTION" "$DISTCACHE/.refs/$COMP/$FILENAME" "$VARCACHE/$POOL"
         fi
     done
 
@@ -514,7 +514,12 @@ apt_cache_source() {
 
 # Clean up old packages in the pool.
 apt_clean() {
-    find "$VARCACHE/pool" -links 1 -type f -delete
+    if [ -z "$VARPOOL" ]; then
+        find "$VARCACHE/pool" -links 1 -type f -delete
+    else
+        find "$VARLIB/apt" -xtype l -delete
+        find "$VARCACHE/pool" -xtype l -delete
+    fi
     find "$VARCACHE/pool" -type d -empty -delete
 }
 

lib/freight/conf.sh

@@ -6,6 +6,11 @@
 VARLIB="/var/lib/freight"
 VARCACHE="/var/cache/freight"
 
+# If the file system does not support hard link, you can use symbolic
+# link instead. In this mode file will be copied to VARPOOL first, then
+# symlinked to VARLIB and finally VARCACHE. Implies SYMLINKS="on".
+VARPOOL=""
+
 # Default architectures.
 # shellcheck disable=SC2034
 ARCHS="i386 amd64"
@@ -55,5 +60,18 @@ fi
 # Normalize directory names.
 VARLIB=${VARLIB%%/}
 VARCACHE=${VARCACHE%%/}
+VARPOOL=${VARPOOL%%/}
+
+# Override options
+if [ -n "$VARPOOL" ]; then
+    # shellcheck disable=SC2034
+    LINK_OPTION="-rsL"
+    # shellcheck disable=SC2034
+    SYMLINKS="on"
+else
+    # For some reasons ln will not work if argv[1] == "" in tests
+    # shellcheck disable=SC2034
+    LINK_OPTION="--"
+fi
 
 # vim: et:ts=4:sw=4

man/man1/freight-add.1

@@ -22,6 +22,12 @@ The package files are organized in the Freight library so \fBfreight\-cache\fR(1
 Use an alternate configuration file\.
 .
 .TP
+\fB\-s\fR \fIsubdir\fR, \fB\-\-subpool=\fR\fIsubdir\fR
+This option is only valid with symbolic pool\.
+Store the file to a subdirectory in symbolic pool\.
+By default, all packages will be saved to the root of the symbolic pool\. Use this option to workaround the package conflict\.
+.
+.TP
 \fB\-v\fR, \fB\-\-verbose\fR
 Verbose mode\.
 .

man/man5/freight.5

@@ -20,6 +20,10 @@ The Freight library directory\. Typically \fI/var/lib/freight\fR\.
 The Freight cache directory\. Typically \fI/var/cache/freight\fR\. This should be the document root of the web server\.
 .
 .TP
+\fBVARPOOL\fR
+The Freight symbolic pool directory\. Forces Freight to use symbolic link instead of hard link, and file will be copied to this folder first so garbage collection can be done later on\.
+.
+.TP
 \fBARCHS\fR
 The architectures to support\. Typically \fIi386 amd64\fR\.
 .

man/man5/freight.5.ronn

@@ -11,6 +11,8 @@ The Freight configuration is a `source`d shell script that defines a few importa
   The Freight library directory.  Typically _/var/lib/freight_.
 * `VARCACHE`:
   The Freight cache directory.  Typically _/var/cache/freight_.  This should be the document root of the web server.
+* `VARPOOL`:
+  The Freight symbolic pool directory. Forces Freight to use symbolic link instead of hard link, and file will be copied to this folder first so garbage collection can be done later on.
 * `ARCHS`:
   The architectures to support.  Typically _i386 amd64_.
 * `ORIGIN`:

test/apt_add.bats

@@ -19,6 +19,9 @@ setup() {
 }
 
 @test "freight-add adds package and hard link to multiple components" {
+    if grep VARPOOL $FREIGHT_CONFIG; then
+        skip "symbolic link mode"
+    fi
     freight_add ${FIXTURES}/test_1.0_all.deb apt/example/comp apt/example/another
     test -e ${FREIGHT_LIB}/apt/example/comp/test_1.0_all.deb
     test -e ${FREIGHT_LIB}/apt/example/another/test_1.0_all.deb

test/apt_cache.bats

@@ -63,6 +63,9 @@ setup() {
 }
 
 @test "freight-cache removes deleted packages from pool" {
+    if grep VARPOOL $FREIGHT_CONFIG; then
+        skip "symbolic link mode"
+    fi
     freight_cache -v
     test -e ${FREIGHT_CACHE}/pool/example/main/t/test/test_1.0_all.deb
     rm -f ${FREIGHT_LIB}/apt/example/test_1.0_all.deb
@@ -74,6 +77,9 @@ setup() {
 }
 
 @test "freight-cache --keep retains deleted packages in pool" {
+    if grep VARPOOL $FREIGHT_CONFIG; then
+        skip "symbolic link mode"
+    fi
     freight_cache -v
     test -e ${FREIGHT_CACHE}/pool/example/main/t/test/test_1.0_all.deb
     rm -f ${FREIGHT_LIB}/apt/example/test_1.0_all.deb

test/freight_helpers.bash

@@ -10,10 +10,15 @@ FREIGHT_HOME=${TMPDIR}/freight
 FREIGHT_CONFIG=${FREIGHT_HOME}/etc/freight.conf
 FREIGHT_CACHE=${FREIGHT_HOME}/var/cache
 FREIGHT_LIB=${FREIGHT_HOME}/var/lib
+FREIGHT_POOL=${FREIGHT_HOME}/var/pool
 
 export GNUPGHOME=${TMPDIR}/gpg
 
 freight_init() {
+    if [ "$FREIGHT_TEST_VARPOOL" = "1" ]; then
+        TEST_VARPOOL=--pooldir=$FREIGHT_POOL
+    fi
+
     gpg_init
     rm -rf $FREIGHT_HOME
     mkdir -p $FREIGHT_CACHE $FREIGHT_LIB
@@ -23,6 +28,7 @@ freight_init() {
         --libdir $FREIGHT_LIB \
         --cachedir $FREIGHT_CACHE \
         --archs "i386 amd64" \
+        $TEST_VARPOOL \
         "$@"
 }