Merge branch 'master' into arm64

Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get -y update && \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
         libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
-        binfmt-support ca-certificates qemu-utils kpartx fdisk gpg \
+        binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

README.md

@@ -15,7 +15,7 @@ To install the required dependencies for `pi-gen` you should run:
 ```bash
 apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
 dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \
-qemu-utils kpartx gpg
+qemu-utils kpartx gpg pigz
 ```
 
 The file `depends` contains a list of tools needed.  The format of this
@@ -116,9 +116,28 @@ The following environment variables are supported:
 
    Output directory for target system images and NOOBS bundles.
 
- * `DEPLOY_ZIP` (Default: `1`)
+ * `DEPLOY_COMPRESSION` (Default: `zip`)
 
-   Setting to `0` will deploy the actual image (`.img`) instead of a zipped image (`.zip`).
+   Set to:
+   * `none` to deploy the actual image (`.img`).
+   * `zip` to deploy a zipped image (`.zip`).
+   * `gz` to deploy a gzipped image (`.img.gz`).
+   * `xz` to deploy a xzipped image (`.img.xz`).
+
+
+ * `DEPLOY_ZIP` (Deprecated)
+
+   This option has been deprecated in favor of `DEPLOY_COMPRESSION`.
+
+   If `DEPLOY_ZIP=0` is still present in your config file, the behavior is the
+   same as with `DEPLOY_COMPRESSION=none`.
+
+ * `COMPRESSION_LEVEL` (Default: `6`)
+
+   Compression level to be used when using `zip`, `gz` or `xz` for
+   `DEPLOY_COMPRESSION`. From 0 to 9 (refer to the tool man page for more
+   information on this. Usually 0 is no compression but very fast, up to 9 with
+   the best compression but very slow ).
 
  * `USE_QEMU` (Default: `"0"`)
 

build.sh

@@ -203,12 +203,22 @@ fi
 export USE_QEMU="${USE_QEMU:-0}"
 export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}"
 export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}"
-export ZIP_FILENAME="${ZIP_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}"
+export ARCHIVE_FILENAME="${ARCHIVE_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}"
 
 export SCRIPT_DIR="${BASE_DIR}/scripts"
 export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_NAME}"}"
 export DEPLOY_DIR=${DEPLOY_DIR:-"${BASE_DIR}/deploy"}
-export DEPLOY_ZIP="${DEPLOY_ZIP:-1}"
+
+# DEPLOY_ZIP was deprecated in favor of DEPLOY_COMPRESSION
+# This preserve the old behavior with DEPLOY_ZIP=0 where no archive was created
+if [ -z "${DEPLOY_COMPRESSION}" ] && [ "${DEPLOY_ZIP:-1}" = "0" ]; then
+	echo "DEPLOY_ZIP has been deprecated in favor of DEPLOY_COMPRESSION"
+	echo "Similar behavior to DEPLOY_ZIP=0 can be obtained with DEPLOY_COMPRESSION=none"
+	echo "Please update your config file"
+	DEPLOY_COMPRESSION=none
+fi
+export DEPLOY_COMPRESSION=${DEPLOY_COMPRESSION:-zip}
+export COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-6}
 export LOG_FILE="${WORK_DIR}/build.log"
 
 export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi}

depends

@@ -20,3 +20,4 @@ bc
 qemu-nbd:qemu-utils
 kpartx
 gpg
+pigz

export-image/04-finalise/01-run.sh

@@ -78,7 +78,7 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
 
 mkdir -p "${DEPLOY_DIR}"
 
-rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip"
+rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*"
 rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
 mv "$INFO_FILE" "$DEPLOY_DIR/"
@@ -95,11 +95,22 @@ else
 	make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
 fi
 
-if [ "${DEPLOY_ZIP}" == "1" ]; then
+case "${DEPLOY_COMPRESSION}" in
+zip)
 	pushd "${STAGE_WORK_DIR}" > /dev/null
-	zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \
+	zip -"${COMPRESSION_LEVEL}" \
-		"$(basename "${IMG_FILE}")"
+	"${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.zip" "$(basename "${IMG_FILE}")"
 	popd > /dev/null
-else
+	;;
-	mv "$IMG_FILE" "$DEPLOY_DIR/"
+gz)
-fi
+	pigz --force -"${COMPRESSION_LEVEL}" "$IMG_FILE" --stdout > \
+	"${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.img.gz"
+	;;
+xz)
+	xz --compress --force --threads 0 --memlimit-compress=50% -"${COMPRESSION_LEVEL}" \
+	--stdout "$IMG_FILE" > "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.img.xz"
+	;;
+none | *)
+	cp "$IMG_FILE" "$DEPLOY_DIR/"
+;;
+esac

export-noobs/prerun.sh

@@ -3,11 +3,7 @@
 NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_NAME}${IMG_SUFFIX}"
 mkdir -p "${STAGE_WORK_DIR}"
 
-if [ "${DEPLOY_ZIP}" == "1" ]; then
+IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img"
-	IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img"
-else
-	IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
-fi
 
 unmount_image "${IMG_FILE}"