add support for debian snapshot

.travis.yml

@@ -1,6 +1,6 @@
 language: bash
 sudo: required
-script: bash shellcheck && sudo bash buildall
+script: bash shellcheck && bash snapshot && sudo bash buildall
 dist: xenial
 services:
   - docker

buildall

@@ -9,6 +9,7 @@ set -o pipefail
 DISTS="jessie
 stretch
 buster
+buster-snapshot
 unstable
 "
 

buildone

@@ -41,39 +41,55 @@ log() {
     echo "$@" >&2
 }
 
+is_snapshot() {
+    local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot"
+    if [[ $1 =~ $dist_snapshot_regex ]]; then
+        true
+    else
+        false
+    fi
+}
+
 build() {
     DIST=$1
+    if is_snapshot "$DIST"; then
+        ! debian_snapshot_id=$(cat build/snapshot_id) && return
+        TAG="${DIST}-${debian_snapshot_id}"
+    else
+        TAG=$DIST
+    fi
+
     [ -f "debootstrap/$DIST" ] || (echo "buildall: Unknown distribution: $DIST" && exit 1)
     current_ts="$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)"
-    if docker pull "$BASENAME:$DIST" > /dev/null; then
-        target_ts="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Created")"
-        pulled_image_id="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Id")"
+    if docker pull "$BASENAME:$TAG" > /dev/null; then
+        target_ts="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Created")"
+        pulled_image_id="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Id")"
     else
         target_ts="$current_ts"
         pulled_image_id=
     fi
     log "============================================"
-    log "Building $BASENAME:$DIST"
+    log "Building $BASENAME:$TAG"
     log "============================================"
-    ./mkimage "build/$DIST.tar" "$DIST"
-    built_image_id=$(./import "build/$DIST.tar" "$target_ts")
+    ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}"
+    built_image_id=$(./import "build/$TAG.tar" "$target_ts")
     log "============================================"
-    log "Running tests for $BASENAME:$DIST"
+    log "Running tests for $BASENAME:$TAG"
     log "============================================"
-    ./test "$built_image_id" "$DIST"
+    ./test "$built_image_id" "$TAG"
     log "============================================"
-    log "Rebuilding $BASENAME:$DIST to test reproducibility"
+    log "Rebuilding $BASENAME:$TAG to test reproducibility"
     log "============================================"
-    ./mkimage "build/${DIST}-repro.tar" "$DIST"
-    repro_image_id=$(./import "build/${DIST}-repro.tar" "$target_ts")
+    ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}"
+    repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts")
     if [ "$repro_image_id" != "$built_image_id" ]; then
-        log "$BASENAME:$DIST differs after a rebuild. Examine $built_image_id and $repro_image_id"
+        log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id"
         log "to find the differences and fix the build to be reproducible again."
         log "Changes (- first build, + second build):"
         ./dockerdiff "$built_image_id" "$repro_image_id" || true
         exit 1
     fi
-    rm "build/${DIST}-repro.tar"
+    rm "build/${TAG}-repro.tar"
     if [ -n "$pulled_image_id" ]; then
         if [ "$built_image_id" != "$pulled_image_id" ]; then
             log "Image changed $built_image_id (new) != $pulled_image_id (old)"
@@ -81,14 +97,14 @@ build() {
             ./dockerdiff "$pulled_image_id" "$built_image_id" || true
             # Re-import with the current timestamp so that the image shows
             # as new
-            built_image_id="$(./import "build/$DIST.tar" "$current_ts")"
+            built_image_id="$(./import "build/$TAG.tar" "$current_ts")"
         else
             log "Image didn't change"
             return
         fi
     fi
-    docker tag "$built_image_id" "$BASENAME:$DIST"
-    log "Tagged $built_image_id as $BASENAME:$DIST"
+    docker tag "$built_image_id" "$BASENAME:$TAG"
+    log "Tagged $built_image_id as $BASENAME:$TAG"
 }
 
 if [ -z "$1" ]; then

debootstrap/buster-snapshot

@@ -0,0 +1 @@
+jessie

mkimage

@@ -7,6 +7,13 @@ ROOT=$(cd "$(dirname "$0")" && pwd)
 
 TARGET=${1:?Specify the target filename}
 DIST=${2:-stable}
+SNAPSHOT_ID=${3:-}
+
+# TRIM -snapshot from the distro
+IFS="-"
+read -a DISTARR <<< "$DIST"
+DIST="${DISTARR[0]}"
+unset IFS
 
 LOGFILE=${TARGET}.log
 
@@ -40,9 +47,17 @@ echo "Building base in $rootfsDir"
 DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir"
 chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage
 
-echo -e "deb http://deb.debian.org/debian $DIST main" > "$rootfsDir/etc/apt/sources.list"
+repo_url="http://deb.debian.org/debian"
+sec_repo_url="http://security.debian.org/"
+
+if [ -n "$SNAPSHOT_ID" ]; then
+	repo_url="http://snapshot.debian.org/archive/debian/${SNAPSHOT_ID}/"
+	sec_repo_url="http://snapshot.debian.org/archive/debian-security/${SNAPSHOT_ID}/"
+fi
+
+echo -e "deb ${repo_url} $DIST main" > "$rootfsDir/etc/apt/sources.list"
 if [ "$DIST" != "unstable" ]; then
-    echo "deb http://security.debian.org/ $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list"
+    echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list"
 fi
 
 chroot "$rootfsDir" apt-get update

pushall

@@ -7,6 +7,7 @@ set -o pipefail
 DISTS="jessie
 stretch
 buster
+buster-snapshot
 unstable
 "
 LATEST=buster
@@ -14,6 +15,15 @@ BASENAME=bitnami/minideb
 GCR_BASENAME=gcr.io/bitnami-containers/minideb
 QUAY_BASENAME=quay.io/bitnami/minideb
 
+is_snapshot() {
+    local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot"
+    if [[ $1 =~ $dist_snapshot_regex ]]; then
+        true
+    else
+        false
+    fi
+}
+
 if [ -n "${DOCKER_PASSWORD:-}" ]; then
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
 fi
@@ -44,9 +54,16 @@ push() {
 }
 
 for DIST in $DISTS; do
-    docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}"
-    docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}"
-    push "$DIST"
+    if is_snapshot "$DIST"; then
+        ! debian_snapshot_id=$(cat build/snapshot_id) && continue
+        TAG="${DIST}-${debian_snapshot_id}"
+    else
+        TAG=$DIST
+    fi
+
+    docker tag "${BASENAME}:${TAG}" "${QUAY_BASENAME}:${TAG}"
+    docker tag "${BASENAME}:${TAG}" "${GCR_BASENAME}:${TAG}"
+    push "$TAG"
 done
 
 docker tag "${BASENAME}:${LATEST}" "${BASENAME}:latest"

snapshot

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+
+get_debian_snapshot_id() {
+    local -r year=$(date -u +%G)
+    local -r month=$(date -u +%m)
+    local -r day=$(date -u +%d)
+
+    snapshot_list_tmp_dir=$(mktemp -d)
+    snapshot_list_tmp_file="${snapshot_list_tmp_dir}/${year}-${month}.html"
+
+    ! curl -sSfL "https://snapshot.debian.org/archive/debian/?year=$year&month=$month" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: year=${year} month=${month}" && return 1
+
+    snapshot_id=$(grep -Po "(${year}${month}${day}T.*Z)" "${snapshot_list_tmp_file}" | tail -1)
+
+    [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (${year}${month}${day}T.*Z)" && return 1
+
+    rm -f "${snapshot_list_tmp_file}"
+    echo "$snapshot_id" > build/snapshot_id && return 0
+}
+
+get_debian_snapshot_id