From 8fafa0d8996b4677d47e09290a4aa2773bddd4dd Mon Sep 17 00:00:00 2001 From: darteaga Date: Tue, 30 Jun 2020 11:38:19 +0000 Subject: [PATCH 01/15] add support for debian snapshot --- .travis.yml | 2 +- buildall | 1 + buildone | 48 ++++++++++++++++++++++++------------- debootstrap/buster-snapshot | 1 + mkimage | 19 +++++++++++++-- pushall | 23 +++++++++++++++--- snapshot | 26 ++++++++++++++++++++ 7 files changed, 98 insertions(+), 22 deletions(-) create mode 120000 debootstrap/buster-snapshot create mode 100755 snapshot diff --git a/.travis.yml b/.travis.yml index 4f1d485..e31e017 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: bash sudo: required -script: bash shellcheck && sudo bash buildall +script: bash shellcheck && bash snapshot && sudo bash buildall dist: xenial services: - docker diff --git a/buildall b/buildall index 73e612d..ddd32a4 100755 --- a/buildall +++ b/buildall @@ -9,6 +9,7 @@ set -o pipefail DISTS="jessie stretch buster +buster-snapshot unstable " diff --git a/buildone b/buildone index edbe732..b88e9a5 100755 --- a/buildone +++ b/buildone @@ -41,39 +41,55 @@ log() { echo "$@" >&2 } +is_snapshot() { + local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" + if [[ $1 =~ $dist_snapshot_regex ]]; then + true + else + false + fi +} + build() { DIST=$1 + if is_snapshot "$DIST"; then + ! debian_snapshot_id=$(cat build/snapshot_id) && return + TAG="${DIST}-${debian_snapshot_id}" + else + TAG=$DIST + fi + [ -f "debootstrap/$DIST" ] || (echo "buildall: Unknown distribution: $DIST" && exit 1) current_ts="$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)" - if docker pull "$BASENAME:$DIST" > /dev/null; then - target_ts="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Created")" - pulled_image_id="$(docker inspect "$BASENAME:$DIST" | jq --raw-output ".[0].Id")" + if docker pull "$BASENAME:$TAG" > /dev/null; then + target_ts="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Created")" + pulled_image_id="$(docker inspect "$BASENAME:$TAG" | jq --raw-output ".[0].Id")" else target_ts="$current_ts" pulled_image_id= fi log "============================================" - log "Building $BASENAME:$DIST" + log "Building $BASENAME:$TAG" log "============================================" - ./mkimage "build/$DIST.tar" "$DIST" - built_image_id=$(./import "build/$DIST.tar" "$target_ts") + ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}" + built_image_id=$(./import "build/$TAG.tar" "$target_ts") log "============================================" - log "Running tests for $BASENAME:$DIST" + log "Running tests for $BASENAME:$TAG" log "============================================" - ./test "$built_image_id" "$DIST" + ./test "$built_image_id" "$TAG" log "============================================" - log "Rebuilding $BASENAME:$DIST to test reproducibility" + log "Rebuilding $BASENAME:$TAG to test reproducibility" log "============================================" - ./mkimage "build/${DIST}-repro.tar" "$DIST" - repro_image_id=$(./import "build/${DIST}-repro.tar" "$target_ts") + ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}" + repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts") if [ "$repro_image_id" != "$built_image_id" ]; then - log "$BASENAME:$DIST differs after a rebuild. Examine $built_image_id and $repro_image_id" + log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id" log "to find the differences and fix the build to be reproducible again." log "Changes (- first build, + second build):" ./dockerdiff "$built_image_id" "$repro_image_id" || true exit 1 fi - rm "build/${DIST}-repro.tar" + rm "build/${TAG}-repro.tar" if [ -n "$pulled_image_id" ]; then if [ "$built_image_id" != "$pulled_image_id" ]; then log "Image changed $built_image_id (new) != $pulled_image_id (old)" @@ -81,14 +97,14 @@ build() { ./dockerdiff "$pulled_image_id" "$built_image_id" || true # Re-import with the current timestamp so that the image shows # as new - built_image_id="$(./import "build/$DIST.tar" "$current_ts")" + built_image_id="$(./import "build/$TAG.tar" "$current_ts")" else log "Image didn't change" return fi fi - docker tag "$built_image_id" "$BASENAME:$DIST" - log "Tagged $built_image_id as $BASENAME:$DIST" + docker tag "$built_image_id" "$BASENAME:$TAG" + log "Tagged $built_image_id as $BASENAME:$TAG" } if [ -z "$1" ]; then diff --git a/debootstrap/buster-snapshot b/debootstrap/buster-snapshot new file mode 120000 index 0000000..8cfa86a --- /dev/null +++ b/debootstrap/buster-snapshot @@ -0,0 +1 @@ +jessie \ No newline at end of file diff --git a/mkimage b/mkimage index cd9d2c6..76e3fc1 100755 --- a/mkimage +++ b/mkimage @@ -7,6 +7,13 @@ ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} DIST=${2:-stable} +SNAPSHOT_ID=${3:-} + +# TRIM -snapshot from the distro +IFS="-" +read -a DISTARR <<< "$DIST" +DIST="${DISTARR[0]}" +unset IFS LOGFILE=${TARGET}.log @@ -40,9 +47,17 @@ echo "Building base in $rootfsDir" DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir" chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage -echo -e "deb http://deb.debian.org/debian $DIST main" > "$rootfsDir/etc/apt/sources.list" +repo_url="http://deb.debian.org/debian" +sec_repo_url="http://security.debian.org/" + +if [ -n "$SNAPSHOT_ID" ]; then + repo_url="http://snapshot.debian.org/archive/debian/${SNAPSHOT_ID}/" + sec_repo_url="http://snapshot.debian.org/archive/debian-security/${SNAPSHOT_ID}/" +fi + +echo -e "deb ${repo_url} $DIST main" > "$rootfsDir/etc/apt/sources.list" if [ "$DIST" != "unstable" ]; then - echo "deb http://security.debian.org/ $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" + echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" fi chroot "$rootfsDir" apt-get update diff --git a/pushall b/pushall index 85e1909..eb3ba95 100755 --- a/pushall +++ b/pushall @@ -7,6 +7,7 @@ set -o pipefail DISTS="jessie stretch buster +buster-snapshot unstable " LATEST=buster @@ -14,6 +15,15 @@ BASENAME=bitnami/minideb GCR_BASENAME=gcr.io/bitnami-containers/minideb QUAY_BASENAME=quay.io/bitnami/minideb +is_snapshot() { + local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" + if [[ $1 =~ $dist_snapshot_regex ]]; then + true + else + false + fi +} + if [ -n "${DOCKER_PASSWORD:-}" ]; then docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" fi @@ -44,9 +54,16 @@ push() { } for DIST in $DISTS; do - docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}" - docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}" - push "$DIST" + if is_snapshot "$DIST"; then + ! debian_snapshot_id=$(cat build/snapshot_id) && continue + TAG="${DIST}-${debian_snapshot_id}" + else + TAG=$DIST + fi + + docker tag "${BASENAME}:${TAG}" "${QUAY_BASENAME}:${TAG}" + docker tag "${BASENAME}:${TAG}" "${GCR_BASENAME}:${TAG}" + push "$TAG" done docker tag "${BASENAME}:${LATEST}" "${BASENAME}:latest" diff --git a/snapshot b/snapshot new file mode 100755 index 0000000..8e8317f --- /dev/null +++ b/snapshot @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + + +get_debian_snapshot_id() { + local -r year=$(date -u +%G) + local -r month=$(date -u +%m) + local -r day=$(date -u +%d) + + snapshot_list_tmp_dir=$(mktemp -d) + snapshot_list_tmp_file="${snapshot_list_tmp_dir}/${year}-${month}.html" + + ! curl -sSfL "https://snapshot.debian.org/archive/debian/?year=$year&month=$month" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: year=${year} month=${month}" && return 1 + + snapshot_id=$(grep -Po "(${year}${month}${day}T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (${year}${month}${day}T.*Z)" && return 1 + + rm -f "${snapshot_list_tmp_file}" + echo "$snapshot_id" > build/snapshot_id && return 0 +} + +get_debian_snapshot_id From e754737551795f37ac4b8b825d1841fe7b707780 Mon Sep 17 00:00:00 2001 From: darteaga Date: Tue, 30 Jun 2020 11:59:15 +0000 Subject: [PATCH 02/15] fix read options error --- mkimage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage b/mkimage index 76e3fc1..5c7577c 100755 --- a/mkimage +++ b/mkimage @@ -11,7 +11,7 @@ SNAPSHOT_ID=${3:-} # TRIM -snapshot from the distro IFS="-" -read -a DISTARR <<< "$DIST" +read -ra DISTARR <<< "$DIST" DIST="${DISTARR[0]}" unset IFS From b7ed2764cd98b1a1de161a12cee874cc05080ad9 Mon Sep 17 00:00:00 2001 From: darteaga Date: Tue, 30 Jun 2020 12:04:23 +0000 Subject: [PATCH 03/15] fix not directory error --- snapshot | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snapshot b/snapshot index 8e8317f..7e5c9f0 100755 --- a/snapshot +++ b/snapshot @@ -23,4 +23,6 @@ get_debian_snapshot_id() { echo "$snapshot_id" > build/snapshot_id && return 0 } +mkdir -p build + get_debian_snapshot_id From c9c82d6a178ee22adaa658b17e4338decb309e16 Mon Sep 17 00:00:00 2001 From: Daniel Arteaga Date: Tue, 30 Jun 2020 15:31:08 +0200 Subject: [PATCH 04/15] remove unnecessary if/else Co-authored-by: Marko Mikulicic --- buildone | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/buildone b/buildone index b88e9a5..80cabd0 100755 --- a/buildone +++ b/buildone @@ -43,11 +43,7 @@ log() { is_snapshot() { local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" - if [[ $1 =~ $dist_snapshot_regex ]]; then - true - else - false - fi + [[ $1 =~ $dist_snapshot_regex ]] } build() { From 82fec01b8ef8cc89d470cb3a3c617c7568d23590 Mon Sep 17 00:00:00 2001 From: darteaga Date: Tue, 30 Jun 2020 13:32:48 +0000 Subject: [PATCH 05/15] remove unnecessary if/else in pushall --- pushall | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pushall b/pushall index eb3ba95..d9ed057 100755 --- a/pushall +++ b/pushall @@ -17,11 +17,7 @@ QUAY_BASENAME=quay.io/bitnami/minideb is_snapshot() { local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" - if [[ $1 =~ $dist_snapshot_regex ]]; then - true - else - false - fi + [[ $1 =~ $dist_snapshot_regex ]] } if [ -n "${DOCKER_PASSWORD:-}" ]; then From 35fb3b08f53bc6dc4cffbe19d57b7ad94e882ef2 Mon Sep 17 00:00:00 2001 From: darteaga Date: Wed, 1 Jul 2020 10:44:13 +0000 Subject: [PATCH 06/15] avoid using IFS --- mkimage | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mkimage b/mkimage index 5c7577c..fca304a 100755 --- a/mkimage +++ b/mkimage @@ -6,15 +6,9 @@ set -o pipefail ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} -DIST=${2:-stable} +DIST=${2%%-*} SNAPSHOT_ID=${3:-} -# TRIM -snapshot from the distro -IFS="-" -read -ra DISTARR <<< "$DIST" -DIST="${DISTARR[0]}" -unset IFS - LOGFILE=${TARGET}.log :>"$LOGFILE" From eee9f06e77cc54cd4dbda15f6dd0c5dad77b6ffe Mon Sep 17 00:00:00 2001 From: darteaga Date: Wed, 1 Jul 2020 14:44:40 +0000 Subject: [PATCH 07/15] improve snapshot script to always get the latest id --- mkimage | 2 +- snapshot | 35 ++++++++++++++++++++++------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/mkimage b/mkimage index fca304a..95d6714 100755 --- a/mkimage +++ b/mkimage @@ -6,7 +6,7 @@ set -o pipefail ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} -DIST=${2%%-*} +DIST=${2%-*} SNAPSHOT_ID=${3:-} LOGFILE=${TARGET}.log diff --git a/snapshot b/snapshot index 7e5c9f0..2e980d2 100755 --- a/snapshot +++ b/snapshot @@ -4,25 +4,34 @@ set -e set -u set -o pipefail +snapshot_list_tmp_dir=$(mktemp -d) +mkdir -p "${snapshot_list_tmp_dir}" -get_debian_snapshot_id() { - local -r year=$(date -u +%G) - local -r month=$(date -u +%m) - local -r day=$(date -u +%d) +get_latest_month_query(){ + snapshot_archive_tmp_file="${snapshot_list_tmp_dir}/archive.html" - snapshot_list_tmp_dir=$(mktemp -d) - snapshot_list_tmp_file="${snapshot_list_tmp_dir}/${year}-${month}.html" + ! curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" && echo "Error when accessing https://snapshot.debian.org/archive/debian/" && return 1 - ! curl -sSfL "https://snapshot.debian.org/archive/debian/?year=$year&month=$month" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: year=${year} month=${month}" && return 1 + month_query=$(grep -Po "(\?year=\d\d\d\d&month=\d+)" "${snapshot_archive_tmp_file}" | tail -1) - snapshot_id=$(grep -Po "(${year}${month}${day}T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: (?year=\d\d\d\d&month=\d+)" && return 1 - [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (${year}${month}${day}T.*Z)" && return 1 + echo "$month_query" && return 0 +} - rm -f "${snapshot_list_tmp_file}" +get_latest_debian_snapshot_id() { + ! month_query=$(get_latest_month_query) && return 1 + + snapshot_list_tmp_file="${snapshot_list_tmp_dir}/month-snapshots.html" + + ! curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: query=${month_query}" && return 1 + + snapshot_id=$(grep -Po "(\d+T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (\d+T.*Z)" && return 1 + + mkdir -p build echo "$snapshot_id" > build/snapshot_id && return 0 } -mkdir -p build - -get_debian_snapshot_id +get_latest_debian_snapshot_id From 815d32183d85116134c40bef687e7e042f79bb8a Mon Sep 17 00:00:00 2001 From: darteaga Date: Wed, 1 Jul 2020 16:36:28 +0000 Subject: [PATCH 08/15] apply pr feedback --- .travis.yml | 2 +- buildall | 51 ++++++++++++++++++++++++++++++++----- buildone | 14 ++++------ debootstrap/buster-snapshot | 1 - mkimage | 2 +- snapshot | 37 --------------------------- 6 files changed, 51 insertions(+), 56 deletions(-) delete mode 120000 debootstrap/buster-snapshot delete mode 100755 snapshot diff --git a/.travis.yml b/.travis.yml index e31e017..4f1d485 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: bash sudo: required -script: bash shellcheck && bash snapshot && sudo bash buildall +script: bash shellcheck && sudo bash buildall dist: xenial services: - docker diff --git a/buildall b/buildall index ddd32a4..f16321d 100755 --- a/buildall +++ b/buildall @@ -6,13 +6,50 @@ set -e set -u set -o pipefail -DISTS="jessie -stretch -buster -buster-snapshot -unstable -" +DISTS="buster-snapshot" + +# is_snapshot() { +# local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" +# [[ $1 =~ $dist_snapshot_regex ]] +# } + +get_latest_month_query(){ + local -r snapshot_tmp_dir=$(mktemp -d) + local -r snapshot_archive_tmp_file="${snapshot_tmp_dir}/archive.html" + mkdir -p "${snapshot_tmp_dir}" + + ! curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" && echo "Error when accessing https://snapshot.debian.org/archive/debian/" && return 1 + + local month_query=$(grep -Po "(\?year=\d\d\d\d&month=\d+)" "${snapshot_archive_tmp_file}" | tail -1) + + [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: (?year=\d\d\d\d&month=\d+)" && return 1 + + echo "$month_query" +} + +get_latest_debian_snapshot_id() { + local -r snapshot_tmp_dir=$(mktemp -d) + local -r snapshot_list_tmp_file="${snapshot_tmp_dir}/month-snapshots.html" + mkdir -p "${snapshot_tmp_dir}" + + ! month_query=$(get_latest_month_query) && return 1 + + ! curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: query=${month_query}" && return 1 + + local snapshot_id=$(grep -Po "(\d+T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (\d+T.*Z)" && return 1 + + echo "$snapshot_id" +} + +! snapshot_id=$(get_latest_debian_snapshot_id) && exit 1 for DIST in $DISTS; do - ./buildone "$DIST" + DIST=${DIST%-*} + + ./buildone "$DIST" "$snapshot_id" done + +mkdir -p build +echo "$snapshot_id" > build/snapshot_id diff --git a/buildone b/buildone index 80cabd0..43260a9 100755 --- a/buildone +++ b/buildone @@ -41,16 +41,12 @@ log() { echo "$@" >&2 } -is_snapshot() { - local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" - [[ $1 =~ $dist_snapshot_regex ]] -} - build() { DIST=$1 - if is_snapshot "$DIST"; then - ! debian_snapshot_id=$(cat build/snapshot_id) && return - TAG="${DIST}-${debian_snapshot_id}" + + debian_snapshot_id=${2:-} + if [ -n "$debian_snapshot_id" ]; then + TAG="${DIST}-snapshot-${debian_snapshot_id}" else TAG=$DIST fi @@ -108,4 +104,4 @@ if [ -z "$1" ]; then exit 1 fi -build "$1" +build "$1" "$2" diff --git a/debootstrap/buster-snapshot b/debootstrap/buster-snapshot deleted file mode 120000 index 8cfa86a..0000000 --- a/debootstrap/buster-snapshot +++ /dev/null @@ -1 +0,0 @@ -jessie \ No newline at end of file diff --git a/mkimage b/mkimage index 95d6714..3928912 100755 --- a/mkimage +++ b/mkimage @@ -6,7 +6,7 @@ set -o pipefail ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} -DIST=${2%-*} +DIST=${2:-stable} SNAPSHOT_ID=${3:-} LOGFILE=${TARGET}.log diff --git a/snapshot b/snapshot deleted file mode 100755 index 2e980d2..0000000 --- a/snapshot +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e -set -u -set -o pipefail - -snapshot_list_tmp_dir=$(mktemp -d) -mkdir -p "${snapshot_list_tmp_dir}" - -get_latest_month_query(){ - snapshot_archive_tmp_file="${snapshot_list_tmp_dir}/archive.html" - - ! curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" && echo "Error when accessing https://snapshot.debian.org/archive/debian/" && return 1 - - month_query=$(grep -Po "(\?year=\d\d\d\d&month=\d+)" "${snapshot_archive_tmp_file}" | tail -1) - - [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: (?year=\d\d\d\d&month=\d+)" && return 1 - - echo "$month_query" && return 0 -} - -get_latest_debian_snapshot_id() { - ! month_query=$(get_latest_month_query) && return 1 - - snapshot_list_tmp_file="${snapshot_list_tmp_dir}/month-snapshots.html" - - ! curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: query=${month_query}" && return 1 - - snapshot_id=$(grep -Po "(\d+T.*Z)" "${snapshot_list_tmp_file}" | tail -1) - - [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (\d+T.*Z)" && return 1 - - mkdir -p build - echo "$snapshot_id" > build/snapshot_id && return 0 -} - -get_latest_debian_snapshot_id From 135056aa0aa9944b60b3042b550999f9957981d1 Mon Sep 17 00:00:00 2001 From: darteaga Date: Wed, 1 Jul 2020 16:42:29 +0000 Subject: [PATCH 09/15] missing dists --- buildall | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/buildall b/buildall index f16321d..fbc4f34 100755 --- a/buildall +++ b/buildall @@ -6,12 +6,12 @@ set -e set -u set -o pipefail -DISTS="buster-snapshot" - -# is_snapshot() { -# local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" -# [[ $1 =~ $dist_snapshot_regex ]] -# } +DISTS="jessie +stretch +buster +buster-snapshot +unstable +" get_latest_month_query(){ local -r snapshot_tmp_dir=$(mktemp -d) From eb3437cd8ae3bc90dca599fd90f43e24ebc92422 Mon Sep 17 00:00:00 2001 From: darteaga Date: Wed, 1 Jul 2020 17:06:21 +0000 Subject: [PATCH 10/15] improve scripts --- buildall | 15 +++++++++------ buildone | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/buildall b/buildall index fbc4f34..def9b68 100755 --- a/buildall +++ b/buildall @@ -9,10 +9,11 @@ set -o pipefail DISTS="jessie stretch buster -buster-snapshot unstable " +DISTS_WITH_SNAPSHOT="buster" + get_latest_month_query(){ local -r snapshot_tmp_dir=$(mktemp -d) local -r snapshot_archive_tmp_file="${snapshot_tmp_dir}/archive.html" @@ -20,7 +21,7 @@ get_latest_month_query(){ ! curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" && echo "Error when accessing https://snapshot.debian.org/archive/debian/" && return 1 - local month_query=$(grep -Po "(\?year=\d\d\d\d&month=\d+)" "${snapshot_archive_tmp_file}" | tail -1) + local -r month_query=$(grep -Po "(\?year=\d\d\d\d&month=\d+)" "${snapshot_archive_tmp_file}" | tail -1) [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: (?year=\d\d\d\d&month=\d+)" && return 1 @@ -36,18 +37,20 @@ get_latest_debian_snapshot_id() { ! curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: query=${month_query}" && return 1 - local snapshot_id=$(grep -Po "(\d+T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + local -r snapshot_id=$(grep -Po "(\d+T.*Z)" "${snapshot_list_tmp_file}" | tail -1) [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (\d+T.*Z)" && return 1 echo "$snapshot_id" } +for DIST in $DISTS; do + ./buildone "$DIST" +done + ! snapshot_id=$(get_latest_debian_snapshot_id) && exit 1 -for DIST in $DISTS; do - DIST=${DIST%-*} - +for DIST in $DISTS_WITH_SNAPSHOT; do ./buildone "$DIST" "$snapshot_id" done diff --git a/buildone b/buildone index 43260a9..2e61e3b 100755 --- a/buildone +++ b/buildone @@ -104,4 +104,4 @@ if [ -z "$1" ]; then exit 1 fi -build "$1" "$2" +build "${1}" "${2:-}" From ff93bf6411ebc385696a8572dddf5bb269656292 Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 2 Jul 2020 10:38:56 +0000 Subject: [PATCH 11/15] improve scripts --- buildall | 10 ++++++---- pushall | 27 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/buildall b/buildall index def9b68..8689390 100755 --- a/buildall +++ b/buildall @@ -21,9 +21,10 @@ get_latest_month_query(){ ! curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" && echo "Error when accessing https://snapshot.debian.org/archive/debian/" && return 1 - local -r month_query=$(grep -Po "(\?year=\d\d\d\d&month=\d+)" "${snapshot_archive_tmp_file}" | tail -1) + local -r month_query_regex="(\?year=\d\d\d\d&month=\d+)" + local -r month_query=$(grep -Po "${month_query_regex}" "${snapshot_archive_tmp_file}" | tail -1) - [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: (?year=\d\d\d\d&month=\d+)" && return 1 + [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: ${month_query_regex}" && return 1 echo "$month_query" } @@ -37,9 +38,10 @@ get_latest_debian_snapshot_id() { ! curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: query=${month_query}" && return 1 - local -r snapshot_id=$(grep -Po "(\d+T.*Z)" "${snapshot_list_tmp_file}" | tail -1) + local -r snapshot_id_regex="(\d+T.*Z)" + local -r snapshot_id=$(grep -Po "${snapshot_id_regex}" "${snapshot_list_tmp_file}" | tail -1) - [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: (\d+T.*Z)" && return 1 + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: ${snapshot_id_regex}" && return 1 echo "$snapshot_id" } diff --git a/pushall b/pushall index d9ed057..c9ecd82 100755 --- a/pushall +++ b/pushall @@ -7,9 +7,9 @@ set -o pipefail DISTS="jessie stretch buster -buster-snapshot unstable " +DISTS_WITH_SNAPSHOT="buster" LATEST=buster BASENAME=bitnami/minideb GCR_BASENAME=gcr.io/bitnami-containers/minideb @@ -50,18 +50,23 @@ push() { } for DIST in $DISTS; do - if is_snapshot "$DIST"; then - ! debian_snapshot_id=$(cat build/snapshot_id) && continue - TAG="${DIST}-${debian_snapshot_id}" - else - TAG=$DIST - fi - - docker tag "${BASENAME}:${TAG}" "${QUAY_BASENAME}:${TAG}" - docker tag "${BASENAME}:${TAG}" "${GCR_BASENAME}:${TAG}" - push "$TAG" + docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}" + docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}" + push "$DIST" done +if [ -f "build/snapshot_id" ]; then + snapshot_id=$(cat build/snapshot_id) + + for DIST in $DISTS_WITH_SNAPSHOT; do + DIST="${DIST}-snapshot-${snapshot_id}" + + docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}" + docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}" + push "$DIST" + done +fi + docker tag "${BASENAME}:${LATEST}" "${BASENAME}:latest" docker tag "${QUAY_BASENAME}:${LATEST}" "${QUAY_BASENAME}:latest" docker tag "${GCR_BASENAME}:${LATEST}" "${GCR_BASENAME}:latest" From ec594cb8877f0db027c2b388c9f6611d9a0e02b7 Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 2 Jul 2020 12:06:41 +0000 Subject: [PATCH 12/15] refactor vars and remove unused function --- pushall | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pushall b/pushall index c9ecd82..5d2a561 100755 --- a/pushall +++ b/pushall @@ -15,11 +15,6 @@ BASENAME=bitnami/minideb GCR_BASENAME=gcr.io/bitnami-containers/minideb QUAY_BASENAME=quay.io/bitnami/minideb -is_snapshot() { - local -r dist_snapshot_regex="^(jessie|stretch|buster|unstable)-snapshot" - [[ $1 =~ $dist_snapshot_regex ]] -} - if [ -n "${DOCKER_PASSWORD:-}" ]; then docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" fi @@ -59,11 +54,11 @@ if [ -f "build/snapshot_id" ]; then snapshot_id=$(cat build/snapshot_id) for DIST in $DISTS_WITH_SNAPSHOT; do - DIST="${DIST}-snapshot-${snapshot_id}" + TAG="${DIST}-snapshot-${snapshot_id}" - docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}" - docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}" - push "$DIST" + docker tag "${BASENAME}:${TAG}" "${QUAY_BASENAME}:${TAG}" + docker tag "${BASENAME}:${TAG}" "${GCR_BASENAME}:${TAG}" + push "${TAG}" done fi From 8a090bd3e680f6b2b000e3bf9c5be30f023b4589 Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 2 Jul 2020 15:03:12 +0000 Subject: [PATCH 13/15] separate snapshot_id logic --- buildall | 48 ++++++++---------------------------------------- snapshotid | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 40 deletions(-) create mode 100755 snapshotid diff --git a/buildall b/buildall index 8689390..e4b027d 100755 --- a/buildall +++ b/buildall @@ -11,50 +11,18 @@ stretch buster unstable " - DISTS_WITH_SNAPSHOT="buster" -get_latest_month_query(){ - local -r snapshot_tmp_dir=$(mktemp -d) - local -r snapshot_archive_tmp_file="${snapshot_tmp_dir}/archive.html" - mkdir -p "${snapshot_tmp_dir}" - - ! curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" && echo "Error when accessing https://snapshot.debian.org/archive/debian/" && return 1 - - local -r month_query_regex="(\?year=\d\d\d\d&month=\d+)" - local -r month_query=$(grep -Po "${month_query_regex}" "${snapshot_archive_tmp_file}" | tail -1) - - [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: ${month_query_regex}" && return 1 - - echo "$month_query" -} - -get_latest_debian_snapshot_id() { - local -r snapshot_tmp_dir=$(mktemp -d) - local -r snapshot_list_tmp_file="${snapshot_tmp_dir}/month-snapshots.html" - mkdir -p "${snapshot_tmp_dir}" - - ! month_query=$(get_latest_month_query) && return 1 - - ! curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" && echo "Not found snapshots for these parameters: query=${month_query}" && return 1 - - local -r snapshot_id_regex="(\d+T.*Z)" - local -r snapshot_id=$(grep -Po "${snapshot_id_regex}" "${snapshot_list_tmp_file}" | tail -1) - - [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: ${snapshot_id_regex}" && return 1 - - echo "$snapshot_id" -} - for DIST in $DISTS; do ./buildone "$DIST" done -! snapshot_id=$(get_latest_debian_snapshot_id) && exit 1 +snapshot_id=$(./snapshotid) +if [ -n "$snapshot_id" ]; then + for DIST in $DISTS_WITH_SNAPSHOT; do + ./buildone "$DIST" "$snapshot_id" + done -for DIST in $DISTS_WITH_SNAPSHOT; do - ./buildone "$DIST" "$snapshot_id" -done - -mkdir -p build -echo "$snapshot_id" > build/snapshot_id + mkdir -p build + echo "$snapshot_id" > build/snapshot_id +fi diff --git a/snapshotid b/snapshotid new file mode 100755 index 0000000..083cd05 --- /dev/null +++ b/snapshotid @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + +snapshot_tmp_dir=$(mktemp -d) +mkdir -p "${snapshot_tmp_dir}" + +get_latest_month_query() { + local -r snapshot_archive_tmp_file="${snapshot_tmp_dir}/archive.html" + + curl -sSfL "https://snapshot.debian.org/archive/debian/" > "$snapshot_archive_tmp_file" + + local -r month_query_regex="(\?year=\d\d\d\d&month=\d+)" + local -r month_query=$(grep -Po "${month_query_regex}" "${snapshot_archive_tmp_file}" | tail -1) + + [[ -z "$month_query" ]] && echo "Not found snapshots using the following regex: ${month_query_regex}" && return 1 + + echo "$month_query" +} + +get_latest_debian_snapshot_id() { + local -r snapshot_list_tmp_file="${snapshot_tmp_dir}/month-snapshots.html" + + ! month_query=$(get_latest_month_query) && return 1 + + curl -sSfL "https://snapshot.debian.org/archive/debian/$month_query" > "$snapshot_list_tmp_file" + + local -r snapshot_id_regex="(\d+T.*Z)" + local -r snapshot_id=$(grep -Po "${snapshot_id_regex}" "${snapshot_list_tmp_file}" | tail -1) + + [[ -z "$snapshot_id" ]] && echo "Not found snapshot id using the following regex: ${snapshot_id_regex}" && return 1 + + echo "$snapshot_id" +} + +get_latest_debian_snapshot_id +rm -rf "${snapshot_tmp_dir}" From eb9110745c9e16b126c40cce6ac75ed51d2e10dc Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 2 Jul 2020 16:11:43 +0000 Subject: [PATCH 14/15] add build_latest_snapshot script --- build_latest_snapshot | 10 ++++++++++ buildall | 14 +++++++------- snapshotid => snapshot_id | 0 3 files changed, 17 insertions(+), 7 deletions(-) create mode 100755 build_latest_snapshot rename snapshotid => snapshot_id (100%) diff --git a/build_latest_snapshot b/build_latest_snapshot new file mode 100755 index 0000000..d2e6847 --- /dev/null +++ b/build_latest_snapshot @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + +dist=${1:?dist arg is required} +snapshot_id=${2:-$(./snapshot_id)} + +./buildone "$dist" "$snapshot_id" diff --git a/buildall b/buildall index e4b027d..8ee87a6 100755 --- a/buildall +++ b/buildall @@ -6,21 +6,21 @@ set -e set -u set -o pipefail -DISTS="jessie +dist="jessie stretch buster unstable " -DISTS_WITH_SNAPSHOT="buster" +dist_with_snapshot="buster" -for DIST in $DISTS; do - ./buildone "$DIST" +for i in $dist; do + ./buildone "$i" done -snapshot_id=$(./snapshotid) +snapshot_id=$(./snapshot_id) if [ -n "$snapshot_id" ]; then - for DIST in $DISTS_WITH_SNAPSHOT; do - ./buildone "$DIST" "$snapshot_id" + for i in $dist_with_snapshot; do + ./build_latest_snapshot "$i" "$snapshot_id" done mkdir -p build diff --git a/snapshotid b/snapshot_id similarity index 100% rename from snapshotid rename to snapshot_id From 3fc1c3a0a80331cd313b42cb089b3ef51e2fd5f3 Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 2 Jul 2020 16:32:35 +0000 Subject: [PATCH 15/15] rename buildone_snapshot --- buildall | 2 +- build_latest_snapshot => buildone_snapshot | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename build_latest_snapshot => buildone_snapshot (100%) diff --git a/buildall b/buildall index 8ee87a6..b4fcca0 100755 --- a/buildall +++ b/buildall @@ -20,7 +20,7 @@ done snapshot_id=$(./snapshot_id) if [ -n "$snapshot_id" ]; then for i in $dist_with_snapshot; do - ./build_latest_snapshot "$i" "$snapshot_id" + ./buildone_snapshot "$i" "$snapshot_id" done mkdir -p build diff --git a/build_latest_snapshot b/buildone_snapshot similarity index 100% rename from build_latest_snapshot rename to buildone_snapshot