diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 0000000..b83c537 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,52 @@ +inputs: + dist: + description: 'Dist to build' + default: 'buster' + platform: + description: 'Platform to build' + default: 'amd64' + is_latest: + description: The created dist is also latest + default: false + build_snapshot: + description: Build snapshot build + default: false +outputs: + snapshot-id: + description: "Created snapshot id if requested to build it" + value: ${{ steps.snapshot-id.outputs.snapshot-id }} +runs: + using: "composite" + steps: + - name: Fix for Ubuntu Xenial apt-daily.service triggering + run: | + while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do + sleep 1 + done + shell: bash + - run: sudo make .installed-requirements + shell: bash + - name: Build snapshot id + id: snapshot-id + run: | + if ${{ inputs.build_snapshot }} ; then + echo "::set-output name=snapshot-id::$(./snapshot_id)" + fi + shell: bash + - name: "Build image" + run: | + set -x + echo Building ${{ inputs.dist }} - ${{ inputs.platform }} + if [[ "${{ inputs.platform }}" == "arm64" ]]; then + sudo apt-get update -qq && sudo apt-get install -y qemu-user-static + fi + sudo -E bash -x buildone "${{ inputs.dist }}" "${{ inputs.platform }}" + if ${{ inputs.build_snapshot }} ; then + sudo -E bash -x buildone_snapshot "${{ inputs.dist }}" "${{ steps.snapshot-id.outputs.snapshot-id }}" "${{ inputs.platform }}" + fi + if ${{ inputs.is_latest }} ; then + BASENAME=${BASENAME:?Undefined or empty BASENAME} + echo "Tagging latest" + docker tag $BASENAME:${{ inputs.dist }}-${{ inputs.platform }} $BASENAME:latest-${{ inputs.platform }} + fi + shell: bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..8ca8ac9 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,122 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: + - master + pull_request: + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: +env: + BASENAME: bitnami/minideb + LATEST: buster +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + shellcheck: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + name: Shellcheck + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Install Dependencies + run: | + sudo apt-get -qq update + sudo apt-get install -y shellcheck + - name: Verify scripts with shellcheck + run: | + bash shellcheck + build_jessie: + runs-on: ubuntu-20.04 + needs: [ shellcheck ] + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Use local build action + uses: ./.github/actions/build + with: + dist: "jessie" + platform: "amd64" + - name: Push + if: github.ref == 'refs/heads/master' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + GCR_EMAIL: ${{ secrets.GCR_EMAIL }} + GCR_KEY: ${{ secrets.GCR_KEY }} + GCR_TOKEN: ${{ secrets.GCR_TOKEN }} + DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }} + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }} + run: | + bash pushone jessie amd64 + + build_multiarch: + runs-on: ubuntu-20.04 + needs: [ shellcheck ] + strategy: + matrix: + dist: [stretch, buster] + arch: [amd64, arm64] + + name: Build ${{ matrix.dist }} on ${{ matrix.arch }} + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Use local build action + id: build + uses: ./.github/actions/build + with: + dist: "${{ matrix.dist }}" + platform: "${{ matrix.arch }}" + is_latest: ${{ matrix.dist == env.LATEST }} + build_snapshot: ${{ matrix.dist == env.LATEST }} + - name: Push + if: github.ref == 'refs/heads/master' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + GCR_EMAIL: ${{ secrets.GCR_EMAIL }} + GCR_KEY: ${{ secrets.GCR_KEY }} + GCR_TOKEN: ${{ secrets.GCR_TOKEN }} + DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }} + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }} + run: | + bash pushone "${{ matrix.dist }}" "${{ matrix.arch }}" + if ${{ matrix.dist == env.LATEST }} ; then + bash pushone "latest" "${{ matrix.arch }}" + fi + if ${{ matrix.dist == env.LATEST }} ; then + bash pushone "${{ matrix.dist }}-snapshot-${{ steps.build.outputs.snapshot-id }}" "${{ matrix.arch }}" + fi + + deploy_manifests: + runs-on: ubuntu-20.04 + needs: [ build_multiarch, build_jessie ] + if: github.ref == 'refs/heads/master' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + GCR_EMAIL: ${{ secrets.GCR_EMAIL }} + GCR_KEY: ${{ secrets.GCR_KEY }} + GCR_TOKEN: ${{ secrets.GCR_TOKEN }} + DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }} + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }} + steps: + - uses: actions/checkout@v2 + - name: Push Manifests + run: | + DISTS="stretch buster latest" bash pushmanifest + DISTS=jessie PLATFORMS=amd64 bash pushmanifest diff --git a/.gitignore b/.gitignore index d945355..64d9240 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -build +/build .kvm-images .installed-requirements .installed-qemu diff --git a/buildone b/buildone index cebb24e..bdf5554 100755 --- a/buildone +++ b/buildone @@ -64,16 +64,16 @@ build() { log "============================================" log "Building $BASENAME:$TAG" log "============================================" - ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}" + ./mkimage "build/$TAG.tar" "$DIST" "$PLATFORM" "${debian_snapshot_id:-}" built_image_id=$(./import "build/$TAG.tar" "$target_ts" "$PLATFORM") log "============================================" log "Running tests for $BASENAME:$TAG" log "============================================" - ./test "$built_image_id" "$TAG" + ./test "$built_image_id" "$TAG" "$PLATFORM" log "============================================" log "Rebuilding $BASENAME:$TAG to test reproducibility" log "============================================" - ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}" + ./mkimage "build/${TAG}-repro.tar" "$DIST" "$PLATFORM" "${debian_snapshot_id:-}" repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts" "$PLATFORM") if [ "$repro_image_id" != "$built_image_id" ]; then log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id" diff --git a/mkimage b/mkimage index 3928912..ec92171 100755 --- a/mkimage +++ b/mkimage @@ -7,7 +7,8 @@ ROOT=$(cd "$(dirname "$0")" && pwd) TARGET=${1:?Specify the target filename} DIST=${2:-stable} -SNAPSHOT_ID=${3:-} +PLATFORM=${3:-$(dpkg --print-architecture)} +SNAPSHOT_ID=${4:-} LOGFILE=${TARGET}.log @@ -26,6 +27,10 @@ if [ -f "${ROOT}/keys/${DIST}.gpg" ]; then gpg --no-default-keyring --keyring "$KEYRING" --import "${ROOT}/keys/${DIST}.gpg" fi +use_qemu_static() { + [[ "$PLATFORM" == "arm64" && ! ( "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ) ]] +} + export DEBIAN_FRONTEND=noninteractive DIRS_TO_TRIM="/usr/share/man @@ -36,10 +41,39 @@ DIRS_TO_TRIM="/usr/share/man /usr/share/info " +debootstrap_arch_args=( ) + +if use_qemu_static ; then + debootstrap_arch_args+=( --arch "$PLATFORM" ) +fi + rootfsDir=$(mktemp -d) echo "Building base in $rootfsDir" -DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir" -chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage +DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap "${debootstrap_arch_args[@]}" --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir" + +# get path to "chroot" in our current PATH +chrootPath="$(type -P chroot)" +rootfs_chroot() { + # "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately! + # set PATH and chroot away! + PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ + "$chrootPath" "$rootfsDir" "$@" + +} + +if use_qemu_static ; then + echo "Setting up qemu static in chroot" + usr_bin_modification_time=$(stat -c %y "$rootfsDir"/usr/bin) + if [ -f "/usr/bin/qemu-aarch64-static" ]; then + find /usr/bin/ -type f -name 'qemu-*-static' -exec cp {} "$rootfsDir"/usr/bin/. \; + else + echo "Cannot find aarch64 qemu static. Aborting..." >&2 + exit 1 + fi + touch -d "$usr_bin_modification_time" "$rootfsDir"/usr/bin +fi + +rootfs_chroot bash debootstrap/debootstrap --second-stage repo_url="http://deb.debian.org/debian" sec_repo_url="http://security.debian.org/" @@ -54,10 +88,10 @@ if [ "$DIST" != "unstable" ]; then echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list" fi -chroot "$rootfsDir" apt-get update -chroot "$rootfsDir" apt-get upgrade -y -o Dpkg::Options::="--force-confdef" +rootfs_chroot apt-get update +rootfs_chroot apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -chroot "$rootfsDir" dpkg -l | tee "$TARGET.manifest" +rootfs_chroot dpkg -l | tee "$TARGET.manifest" echo "Applying docker-specific tweaks" # These are copied from the docker contrib/mkimage/debootstrap script. @@ -67,15 +101,6 @@ echo "Applying docker-specific tweaks" # interested in building versions that this guard would apply to, # so simply apply the tweak unconditionally. -# get path to "chroot" in our current PATH -chrootPath="$(type -P chroot)" -rootfs_chroot() { - # "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately! - # set PATH and chroot away! - PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ - "$chrootPath" "$rootfsDir" "$@" - -} # prevent init scripts from running during install/update echo >&2 "+ echo exit 101 > '$rootfsDir/usr/sbin/policy-rc.d'" @@ -221,7 +246,16 @@ echo "host" > "$rootfsDir/etc/hostname" # Capture the most recent date that a package in the image was changed. # We don't care about the particular date, or which package it comes from, # we just need a date that isn't very far in the past. + +# We get multiple errors like: +# gzip: stdout: Broken pipe +# dpkg-parsechangelog: error: gunzip gave error exit status 1 +# +# TODO: Why? +set +o pipefail BUILD_DATE="$(find "$rootfsDir/usr/share/doc" -name changelog.Debian.gz -print0 | xargs -0 -n1 -I{} dpkg-parsechangelog -SDate -l'{}' | xargs -l -i date --date="{}" +%s | sort -n | tail -n 1)" +set -o pipefail + echo "Trimming down" for DIR in $DIRS_TO_TRIM; do @@ -252,6 +286,13 @@ echo "Largest dirs" du "$rootfsDir" | sort -n | tail -n 20 echo "Built in $rootfsDir" +if use_qemu_static ; then + echo "Cleaning up qemu static files from image" + usr_bin_modification_time=$(stat -c %y "$rootfsDir"/usr/bin) + rm -rf "$rootfsDir"/usr/bin/qemu-*-static + touch -d "$usr_bin_modification_time" "$rootfsDir"/usr/bin +fi + tar cf "$TARGET" -C "$rootfsDir" . rm -r "$rootfsDir" rm -r "$DEBOOTSTRAP_DIR" diff --git a/pre-build.sh b/pre-build.sh index 18c911b..9439b4c 100755 --- a/pre-build.sh +++ b/pre-build.sh @@ -10,10 +10,16 @@ if [[ ! -f /etc/debian_version ]]; then fi apt-get update -apt-get install -y debootstrap debian-archive-keyring jq dpkg-dev gnupg apt-transport-https ca-certificates +apt-get install -y debootstrap debian-archive-keyring jq dpkg-dev gnupg apt-transport-https ca-certificates curl gpg -echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +if ! command -v gcloud &> /dev/null +then + echo "Installing gcloud" + echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -apt-get update -apt-get install -y google-cloud-sdk + apt-get update + apt-get install -y google-cloud-sdk +else + echo "gcloud is installed" +fi diff --git a/pushmanifest b/pushmanifest index 72eee11..582a962 100755 --- a/pushmanifest +++ b/pushmanifest @@ -15,7 +15,7 @@ GCR_BASENAME=gcr.io/bitnami-containers/minideb QUAY_BASENAME=quay.io/bitnami/minideb PLATFORMS=${PLATFORMS:-amd64 arm64} DRY_RUN=${DRY_RUN:-} - +SNAPSHOT_ID=${SNAPSHOT_ID:-} read -r -a ARCHS <<<"$PLATFORMS" run_docker() { @@ -42,16 +42,18 @@ list_includes() { } if [ -n "${DOCKER_PASSWORD:-}" ]; then - run_docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + echo "$DOCKER_PASSWORD" | run_docker login -u "$DOCKER_USERNAME" --password-stdin fi if [ -n "${QUAY_PASSWORD:-}" ]; then - run_docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io + echo "${QUAY_PASSWORD}" | run_docker login -u "$QUAY_USERNAME" --password-stdin quay.io fi if [ -n "${GCR_KEY:-}" ]; then gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY") gcloud auth print-access-token | run_docker login -u oauth2accesstoken --password-stdin gcr.io +elif [ -n "${GCR_TOKEN:-}" ]; then + echo "${GCR_TOKEN:-}" | run_docker login -u oauth2accesstoken --password-stdin gcr.io fi push_manifest() { @@ -72,7 +74,7 @@ tags=() for DIST in $DISTS; do tags+=("$DIST") if list_includes "$DISTS_WITH_SNAPSHOT" "$DIST" ; then - tags+=("$DIST-snapshot-$(./snapshot_id)") + tags+=("$DIST-snapshot-${SNAPSHOT_ID:-$(./snapshot_id)}") fi done @@ -83,7 +85,7 @@ if [[ -n "${QUAY_PASSWORD:-}" ]]; then else echo "Skipping repository quay.io (empty password)" fi -if [[ -n "${GCR_KEY:-}" ]]; then +if [[ -n "${GCR_KEY:-}" || -n "${GCR_TOKEN:-}" ]]; then repositories+=("$GCR_BASENAME") else echo "Skipping repository gcr.io (empty password)" diff --git a/pushone b/pushone index 5b7213f..d43516d 100755 --- a/pushone +++ b/pushone @@ -12,15 +12,18 @@ GCR_BASENAME=gcr.io/bitnami-containers/minideb QUAY_BASENAME=quay.io/bitnami/minideb if [ -n "${DOCKER_PASSWORD:-}" ]; then - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin fi if [ -n "${QUAY_PASSWORD:-}" ]; then - docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io + echo "$QUAY_PASSWORD" | docker login -u "$QUAY_USERNAME" --password-stdin quay.io fi if [ -n "${GCR_KEY:-}" ]; then gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY") + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin gcr.io +elif [ -n "${GCR_TOKEN:-}" ]; then + echo "${GCR_TOKEN:-}" | docker login -u oauth2accesstoken --password-stdin gcr.io fi ENABLE_DOCKER_CONTENT_TRUST=0 @@ -37,10 +40,11 @@ push() { local dist="$1" DOCKER_CONTENT_TRUST=${ENABLE_DOCKER_CONTENT_TRUST} docker push "${BASENAME}:${dist}" docker push "${QUAY_BASENAME}:${dist}" - gcloud docker -- push "${GCR_BASENAME}:${dist}" + docker push "${GCR_BASENAME}:${dist}" } docker tag "${BASENAME}:${DIST}-${PLATFORM}" "${QUAY_BASENAME}:${DIST}-${PLATFORM}" docker tag "${BASENAME}:${DIST}-${PLATFORM}" "${GCR_BASENAME}:${DIST}-${PLATFORM}" push "$DIST-${PLATFORM}" + diff --git a/test b/test index 5f6ae10..a0bae2e 100755 --- a/test +++ b/test @@ -4,7 +4,7 @@ set -eu IMAGE_ID=$1 DIST=$2 - +PLATFORM=${3:-amd64} function desc() { echo "================================" echo -n "TEST: " @@ -16,10 +16,24 @@ function test() { test_extra_args '' "$@" } +bind_mounts=( ) +docker_platform_args=( ) +if [[ "$PLATFORM" == "arm64" ]]; then + if [[ "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ]]; then + echo "Running in arm host. QEMU is not needed" + else + echo "Setting up qemu static" + for qemu_static_file in /usr/bin/qemu-*-static; do + bind_mounts+=( -v="$qemu_static_file:$qemu_static_file" ) + done + docker_platform_args+=( --platform "linux/arm64" ) + fi +fi + function test_extra_args() { local extra_args=$1 shift - docker run --rm $extra_args -e DEBIAN_FRONTEND=noninteractive "$IMAGE_ID" "$@" + docker run "${docker_platform_args[@]}" --rm "${bind_mounts[@]}" $extra_args -e DEBIAN_FRONTEND=noninteractive "$IMAGE_ID" "$@" echo "" echo TEST: OK echo "" @@ -28,6 +42,15 @@ function test_extra_args() { desc "Checking that apt is installed" test dpkg -l apt +desc "Arch matches" +if [[ "$PLATFORM" == "amd64" ]]; then + test bash -c 'echo "$(uname -m)" && [[ "$(uname -m)" == *x86_64* ]]' +elif [[ "$PLATFORM" == "arm64" ]]; then + test bash -c 'echo "$(uname -m)" && [[ "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ]]' +else + echo Unknown platform $PLATFORM >&2 + exit 1 +fi desc "Checking that a package can be installed with apt" test bash -c 'apt-get update && apt-get -y install less && less --help >/dev/null'