diff --git a/buildall b/buildall index 73e612d..b4fcca0 100755 --- a/buildall +++ b/buildall @@ -6,12 +6,23 @@ set -e set -u set -o pipefail -DISTS="jessie +dist="jessie stretch buster unstable " +dist_with_snapshot="buster" -for DIST in $DISTS; do - ./buildone "$DIST" +for i in $dist; do + ./buildone "$i" done + +snapshot_id=$(./snapshot_id) +if [ -n "$snapshot_id" ]; then + for i in $dist_with_snapshot; do + ./buildone_snapshot "$i" "$snapshot_id" + done + + mkdir -p build + echo "$snapshot_id" > build/snapshot_id +fi diff --git a/buildone b/buildone index edbe732..2e61e3b 100755 --- a/buildone +++ b/buildone @@ -43,37 +43,45 @@ log() { build() { DIST=$1 + + debian_snapshot_id=${2:-} + if [ -n "$debian_snapshot_id" ]; then + TAG="${DIST}-snapshot-${debian_snapshot_id}" + else + TAG=$DIST + fi + [ -f "debootstrap/$DIST" ] || (echo "buildall: Unknown distribution: $DIST" && exit 1) current_ts="$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)" - if docker pull "$BASENAME:$DIST" > /dev/null; then - target_ts="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Created")" - pulled_image_id="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Id")" + if docker pull "$BASENAME:$TAG" > /dev/null; then + target_ts="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Created")" + pulled_image_id="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Id")" else target_ts="$current_ts" pulled_image_id= fi log "============================================" - log "Building $BASENAME:$DIST" + log "Building $BASENAME:$TAG" log "============================================" - ./mkimage "build/$DIST.tar" "$DIST" - built_image_id=$(./import "build/$DIST.tar" "$target_ts") + ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}" + built_image_id=$(./import "build/$TAG.tar" "$target_ts") log "============================================" - log "Running tests for $BASENAME:$DIST" + log "Running tests for $BASENAME:$TAG" log "============================================" - ./test "$built_image_id" "$DIST" + ./test "$built_image_id" "$TAG" log "============================================" - log "Rebuilding $BASENAME:$DIST to test reproducibility" + log "Rebuilding $BASENAME:$TAG to test reproducibility" log "============================================" - ./mkimage "build/${DIST}-repro.tar" "$DIST" - repro_image_id=$(./import "build/${DIST}-repro.tar" "$target_ts") + ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}" + repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts") if [ "$repro_image_id" != "$built_image_id" ]; then - log "$BASENAME:$DIST differs after a rebuild. Examine $built_image_id and $repro_image_id" + log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id" log "to find the differences and fix the build to be reproducible again." log "Changes (- first build, + second build):" ./dockerdiff "$built_image_id" "$repro_image_id" || true exit 1 fi - rm "build/${DIST}-repro.tar" + rm "build/${TAG}-repro.tar" if [ -n "$pulled_image_id" ]; then if [ "$built_image_id" != "$pulled_image_id" ]; then log "Image changed $built_image_id (new) != $pulled_image_id (old)" @@ -81,14 +89,14 @@ build() { ./dockerdiff "$pulled_image_id" "$built_image_id" || true # Re-import with the current timestamp so that the image shows # as new - built_image_id="$(./import "build/$DIST.tar" "$current_ts")" + built_image_id="$(./import "build/$TAG.tar" "$current_ts")" else log "Image didn't change" return fi fi - docker tag "$built_image_id" "$BASENAME:$DIST" - log "Tagged $built_image_id as $BASENAME:$DIST" + docker tag "$built_image_id" "$BASENAME:$TAG" + log "Tagged $built_image_id as $BASENAME:$TAG" } if [ -z "$1" ]; then @@ -96,4 +104,4 @@ if [ -z "$1" ]; then exit 1 fi -build "$1" +build "${1}" "${2:-}" diff --git a/buildone_snapshot b/buildone_snapshot new file mode 100755 index 0000000..d2e6847 --- /dev/null +++ b/buildone_snapshot @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + +dist=${1:?dist arg is required} +snapshot_id=${2:-$(./snapshot_id)} + +./buildone "$dist" "$snapshot_id" diff --git a/mkimage b/mkimage index cd9d2c6..3928912 100755 --- a/mkimage +++ b/mkimage @@ -7,6 +7,7 @@ ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} DIST=${2:-stable} +SNAPSHOT_ID=${3:-} LOGFILE=${TARGET}.log @@ -40,9 +41,17 @@ echo "Building base in $rootfsDir" DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir" chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage -echo -e "deb http://deb.debian.org/debian $DIST main" > "$rootfsDir/etc/apt/sources.list" +repo_url="http://deb.debian.org/debian" +sec_repo_url="http://security.debian.org/" + +if [ -n "$SNAPSHOT_ID" ]; then + repo_url="http://snapshot.debian.org/archive/debian/${SNAPSHOT_ID}/" + sec_repo_url="http://snapshot.debian.org/archive/debian-security/${SNAPSHOT_ID}/" +fi + +echo -e "deb ${repo_url} $DIST main" > "$rootfsDir/etc/apt/sources.list" if [ "$DIST" != "unstable" ]; then - echo "deb http://security.debian.org/ $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" + echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" fi chroot "$rootfsDir" apt-get update diff --git a/pushall b/pushall index 85e1909..5d2a561 100755 --- a/pushall +++ b/pushall @@ -9,6 +9,7 @@ stretch buster unstable " +DISTS_WITH_SNAPSHOT="buster" LATEST=buster BASENAME=bitnami/minideb GCR_BASENAME=gcr.io/bitnami-containers/minideb @@ -49,6 +50,18 @@ for DIST in $DISTS; do push "$DIST" done +if [ -f "build/snapshot_id" ]; then + snapshot_id=$(cat build/snapshot_id) + + for DIST in $DISTS_WITH_SNAPSHOT; do + TAG="${DIST}-snapshot-${snapshot_id}" + + docker tag "${BASENAME}:${TAG}" "${QUAY_BASENAME}:${TAG}" + docker tag "${BASENAME}:${TAG}" "${GCR_BASENAME}:${TAG}" + push "${TAG}" + done +fi + docker tag "${BASENAME}:${LATEST}" "${BASENAME}:latest" docker tag "${QUAY_BASENAME}:${LATEST}" "${QUAY_BASENAME}:latest" docker tag "${GCR_BASENAME}:${LATEST}" "${GCR_BASENAME}:latest" diff --git a/snapshot_id b/snapshot_id new file mode 100755 index 0000000..083cd05 --- /dev/null +++ b/snapshot_id @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + +snapshot_tmp_dir=$(mktemp -d) +mkdir -p "${snapshot_tmp_dir}" + +get_latest_month_query() { + local -r snapshot_archive_tmp_file="${snapshot_tmp_dir}/archive.html" + + curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" + + local -r month_query_regex="(\?year=\d\d\d\d&month=\d+)" + local -r month_query=$(grep -Po "${month_query_regex}" "${snapshot_archive_tmp_file}" | tail -1) + + [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: ${month_query_regex}" && return 1 + + echo "$month_query" +} + +get_latest_debian_snapshot_id() { + local -r snapshot_list_tmp_file="${snapshot_tmp_dir}/month-snapshots.html" + + ! month_query=$(get_latest_month_query) && return 1 + + curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" + + local -r snapshot_id_regex="(\d+T.*Z)" + local -r snapshot_id=$(grep -Po "${snapshot_id_regex}" "${snapshot_list_tmp_file}" | tail -1) + + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: ${snapshot_id_regex}" && return 1 + + echo "$snapshot_id" +} + +get_latest_debian_snapshot_id +rm -rf "${snapshot_tmp_dir}"