diff --git a/.travis.yml b/.travis.yml index 4f1d485..e31e017 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: bash sudo: required -script: bash shellcheck && sudo bash buildall +script: bash shellcheck && bash snapshot && sudo bash buildall dist: xenial services: - docker diff --git a/buildall b/buildall index 73e612d..ddd32a4 100755 --- a/buildall +++ b/buildall @@ -9,6 +9,7 @@ set -o pipefail DISTS="jessie stretch buster +buster-snapshot unstable " diff --git a/buildone b/buildone index edbe732..b88e9a5 100755 --- a/buildone +++ b/buildone @@ -41,39 +41,55 @@ log() { echo "$@" >&2 } +is_snapshot() { + local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" + if [[ $1 =~ $dist_snapshot_regex ]]; then + true + else + false + fi +} + build() { DIST=$1 + if is_snapshot "$DIST"; then + ! debian_snapshot_id=$(cat build/snapshot_id) && return + TAG="${DIST}-${debian_snapshot_id}" + else + TAG=$DIST + fi + [ -f "debootstrap/$DIST" ] || (echo "buildall: Unknown distribution: $DIST" && exit 1) current_ts="$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)" - if docker pull "$BASENAME:$DIST" > /dev/null; then - target_ts="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Created")" - pulled_image_id="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Id")" + if docker pull "$BASENAME:$TAG" > /dev/null; then + target_ts="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Created")" + pulled_image_id="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Id")" else target_ts="$current_ts" pulled_image_id= fi log "============================================" - log "Building $BASENAME:$DIST" + log "Building $BASENAME:$TAG" log "============================================" - ./mkimage "build/$DIST.tar" "$DIST" - built_image_id=$(./import "build/$DIST.tar" "$target_ts") + ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}" + built_image_id=$(./import "build/$TAG.tar" "$target_ts") log "============================================" - log "Running tests for $BASENAME:$DIST" + log "Running tests for $BASENAME:$TAG" log "============================================" - ./test "$built_image_id" "$DIST" + ./test "$built_image_id" "$TAG" log "============================================" - log "Rebuilding $BASENAME:$DIST to test reproducibility" + log "Rebuilding $BASENAME:$TAG to test reproducibility" log "============================================" - ./mkimage "build/${DIST}-repro.tar" "$DIST" - repro_image_id=$(./import "build/${DIST}-repro.tar" "$target_ts") + ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}" + repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts") if [ "$repro_image_id" != "$built_image_id" ]; then - log "$BASENAME:$DIST differs after a rebuild. Examine $built_image_id and $repro_image_id" + log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id" log "to find the differences and fix the build to be reproducible again." log "Changes (- first build, + second build):" ./dockerdiff "$built_image_id" "$repro_image_id" || true exit 1 fi - rm "build/${DIST}-repro.tar" + rm "build/${TAG}-repro.tar" if [ -n "$pulled_image_id" ]; then if [ "$built_image_id" != "$pulled_image_id" ]; then log "Image changed $built_image_id (new) != $pulled_image_id (old)" @@ -81,14 +97,14 @@ build() { ./dockerdiff "$pulled_image_id" "$built_image_id" || true # Re-import with the current timestamp so that the image shows # as new - built_image_id="$(./import "build/$DIST.tar" "$current_ts")" + built_image_id="$(./import "build/$TAG.tar" "$current_ts")" else log "Image didn't change" return fi fi - docker tag "$built_image_id" "$BASENAME:$DIST" - log "Tagged $built_image_id as $BASENAME:$DIST" + docker tag "$built_image_id" "$BASENAME:$TAG" + log "Tagged $built_image_id as $BASENAME:$TAG" } if [ -z "$1" ]; then diff --git a/debootstrap/buster-snapshot b/debootstrap/buster-snapshot new file mode 120000 index 0000000..8cfa86a --- /dev/null +++ b/debootstrap/buster-snapshot @@ -0,0 +1 @@ +jessie \ No newline at end of file diff --git a/mkimage b/mkimage index cd9d2c6..76e3fc1 100755 --- a/mkimage +++ b/mkimage @@ -7,6 +7,13 @@ ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} DIST=${2:-stable} +SNAPSHOT_ID=${3:-} + +# TRIM -snapshot from the distro +IFS="-" +read -a DISTARR <<< "$DIST" +DIST="${DISTARR[0]}" +unset IFS LOGFILE=${TARGET}.log @@ -40,9 +47,17 @@ echo "Building base in $rootfsDir" DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir" chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage -echo -e "deb http://deb.debian.org/debian $DIST main" > "$rootfsDir/etc/apt/sources.list" +repo_url="http://deb.debian.org/debian" +sec_repo_url="http://security.debian.org/" + +if [ -n "$SNAPSHOT_ID" ]; then + repo_url="http://snapshot.debian.org/archive/debian/${SNAPSHOT_ID}/" + sec_repo_url="http://snapshot.debian.org/archive/debian-security/${SNAPSHOT_ID}/" +fi + +echo -e "deb ${repo_url} $DIST main" > "$rootfsDir/etc/apt/sources.list" if [ "$DIST" != "unstable" ]; then - echo "deb http://security.debian.org/ $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" + echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" fi chroot "$rootfsDir" apt-get update diff --git a/pushall b/pushall index 85e1909..eb3ba95 100755 --- a/pushall +++ b/pushall @@ -7,6 +7,7 @@ set -o pipefail DISTS="jessie stretch buster +buster-snapshot unstable " LATEST=buster @@ -14,6 +15,15 @@ BASENAME=bitnami/minideb GCR_BASENAME=gcr.io/bitnami-containers/minideb QUAY_BASENAME=quay.io/bitnami/minideb +is_snapshot() { + local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" + if [[ $1 =~ $dist_snapshot_regex ]]; then + true + else + false + fi +} + if [ -n "${DOCKER_PASSWORD:-}" ]; then docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" fi @@ -44,9 +54,16 @@ push() { } for DIST in $DISTS; do - docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}" - docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}" - push "$DIST" + if is_snapshot "$DIST"; then + ! debian_snapshot_id=$(cat build/snapshot_id) && continue + TAG="${DIST}-${debian_snapshot_id}" + else + TAG=$DIST + fi + + docker tag "${BASENAME}:${TAG}" "${QUAY_BASENAME}:${TAG}" + docker tag "${BASENAME}:${TAG}" "${GCR_BASENAME}:${TAG}" + push "$TAG" done docker tag "${BASENAME}:${LATEST}" "${BASENAME}:latest" diff --git a/snapshot b/snapshot new file mode 100755 index 0000000..8e8317f --- /dev/null +++ b/snapshot @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + + +get_debian_snapshot_id() { + local -r year=$(date -u +%G) + local -r month=$(date -u +%m) + local -r day=$(date -u +%d) + + snapshot_list_tmp_dir=$(mktemp -d) + snapshot_list_tmp_file="${snapshot_list_tmp_dir}/${year}-${month}.html" + + ! curl -sSfL "https://snapshot.debian.org/archive/debian/?year=$year&month=$month" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: year=${year} month=${month}" && return 1 + + snapshot_id=$(grep -Po "(${year}${month}${day}T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (${year}${month}${day}T.*Z)" && return 1 + + rm -f "${snapshot_list_tmp_file}" + echo "$snapshot_id" > build/snapshot_id && return 0 +} + +get_debian_snapshot_id