Migrate to GitHub actions (#103)

.github/actions/build/action.yml

@@ -0,0 +1,52 @@
+inputs:
+  dist:
+    description: 'Dist to build'
+    default: 'buster'
+  platform:
+    description: 'Platform to build'
+    default: 'amd64'
+  is_latest:
+    description: The created dist is also latest
+    default: false
+  build_snapshot:
+    description: Build snapshot build
+    default: false
+outputs:
+  snapshot-id:
+    description: "Created snapshot id if requested to build it"
+    value: ${{ steps.snapshot-id.outputs.snapshot-id }}
+runs:
+  using: "composite"
+  steps:
+    - name: Fix for Ubuntu Xenial apt-daily.service triggering
+      run: |
+        while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
+          sleep 1
+        done
+      shell: bash
+    - run: sudo make .installed-requirements
+      shell: bash
+    - name: Build snapshot id
+      id: snapshot-id
+      run: |
+        if ${{ inputs.build_snapshot }} ; then
+           echo "::set-output name=snapshot-id::$(./snapshot_id)"
+        fi
+      shell: bash
+    - name: "Build image"
+      run: |
+        set -x
+        echo Building  ${{ inputs.dist }} - ${{ inputs.platform }}
+        if [[ "${{ inputs.platform }}" == "arm64" ]]; then
+          sudo apt-get update -qq && sudo apt-get install -y qemu-user-static
+        fi
+        sudo -E bash -x buildone "${{ inputs.dist }}" "${{ inputs.platform }}"
+        if ${{ inputs.build_snapshot }} ; then
+             sudo -E bash -x buildone_snapshot "${{ inputs.dist }}" "${{ steps.snapshot-id.outputs.snapshot-id }}" "${{ inputs.platform }}"
+        fi
+        if ${{ inputs.is_latest }} ; then
+          BASENAME=${BASENAME:?Undefined or empty BASENAME}
+          echo "Tagging latest"
+          docker tag $BASENAME:${{ inputs.dist }}-${{ inputs.platform }} $BASENAME:latest-${{ inputs.platform }}
+        fi
+      shell: bash

.github/workflows/main.yml

@@ -0,0 +1,122 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches:
+      - master
+  pull_request:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+env:
+  BASENAME: bitnami/minideb
+  LATEST: buster
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  shellcheck:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-20.04
+    name: Shellcheck
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Install Dependencies
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get install -y shellcheck
+      - name: Verify scripts with shellcheck
+        run: |
+          bash shellcheck
+  build_jessie:
+    runs-on: ubuntu-20.04
+    needs: [ shellcheck ]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Use local build action
+        uses: ./.github/actions/build
+        with:
+          dist: "jessie"
+          platform: "amd64"
+      - name: Push
+        if: github.ref == 'refs/heads/master'
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+          QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+          GCR_EMAIL: ${{ secrets.GCR_EMAIL }}
+          GCR_KEY: ${{ secrets.GCR_KEY }}
+          GCR_TOKEN: ${{ secrets.GCR_TOKEN }}
+          DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }}
+          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }}
+        run: |
+          bash pushone jessie amd64
+
+  build_multiarch:
+    runs-on: ubuntu-20.04
+    needs: [ shellcheck ]
+    strategy:
+      matrix:
+        dist: [stretch, buster]
+        arch: [amd64, arm64]
+
+    name: Build ${{ matrix.dist }} on ${{ matrix.arch }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Use local build action
+        id: build
+        uses: ./.github/actions/build
+        with:
+          dist: "${{ matrix.dist }}"
+          platform: "${{ matrix.arch }}"
+          is_latest: ${{ matrix.dist == env.LATEST }}
+          build_snapshot: ${{ matrix.dist == env.LATEST }}
+      - name: Push
+        if: github.ref == 'refs/heads/master'
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+          QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+          GCR_EMAIL: ${{ secrets.GCR_EMAIL }}
+          GCR_KEY: ${{ secrets.GCR_KEY }}
+          GCR_TOKEN: ${{ secrets.GCR_TOKEN }}
+          DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }}
+          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }}
+        run: |
+          bash pushone "${{ matrix.dist }}" "${{ matrix.arch }}"
+          if ${{ matrix.dist == env.LATEST }} ; then
+             bash pushone "latest" "${{ matrix.arch }}"
+          fi
+          if ${{ matrix.dist == env.LATEST }} ; then
+             bash pushone "${{ matrix.dist }}-snapshot-${{ steps.build.outputs.snapshot-id }}" "${{ matrix.arch }}"
+          fi
+
+  deploy_manifests:
+    runs-on: ubuntu-20.04
+    needs: [ build_multiarch, build_jessie ]
+    if: github.ref == 'refs/heads/master'
+    env:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+      GCR_EMAIL: ${{ secrets.GCR_EMAIL }}
+      GCR_KEY: ${{ secrets.GCR_KEY }}
+      GCR_TOKEN: ${{ secrets.GCR_TOKEN }}
+      DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }}
+      DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Push Manifests
+        run: |
+          DISTS="stretch buster latest" bash pushmanifest
+          DISTS=jessie PLATFORMS=amd64 bash pushmanifest

.gitignore

@@ -1,4 +1,4 @@
-build
+/build
 .kvm-images
 .installed-requirements
 .installed-qemu

buildone

@@ -64,16 +64,16 @@ build() {
     log "============================================"
     log "Building $BASENAME:$TAG"
     log "============================================"
-    ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}"
+    ./mkimage "build/$TAG.tar" "$DIST" "$PLATFORM" "${debian_snapshot_id:-}"
     built_image_id=$(./import "build/$TAG.tar" "$target_ts" "$PLATFORM")
     log "============================================"
     log "Running tests for $BASENAME:$TAG"
     log "============================================"
-    ./test "$built_image_id" "$TAG"
+    ./test "$built_image_id" "$TAG" "$PLATFORM"
     log "============================================"
     log "Rebuilding $BASENAME:$TAG to test reproducibility"
     log "============================================"
-    ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}"
+    ./mkimage "build/${TAG}-repro.tar" "$DIST" "$PLATFORM" "${debian_snapshot_id:-}"
     repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts" "$PLATFORM")
     if [ "$repro_image_id" != "$built_image_id" ]; then
         log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id"

mkimage

@@ -7,7 +7,8 @@ ROOT=$(cd "$(dirname "$0")" && pwd)
 
 TARGET=${1:?Specify the target filename}
 DIST=${2:-stable}
-SNAPSHOT_ID=${3:-}
+PLATFORM=${3:-$(dpkg --print-architecture)}
+SNAPSHOT_ID=${4:-}
 
 LOGFILE=${TARGET}.log
 
@@ -26,6 +27,10 @@ if [ -f "${ROOT}/keys/${DIST}.gpg" ]; then
     gpg --no-default-keyring --keyring "$KEYRING" --import "${ROOT}/keys/${DIST}.gpg"
 fi
 
+use_qemu_static() {
+    [[ "$PLATFORM" == "arm64" && ! ( "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ) ]]
+}
+
 export DEBIAN_FRONTEND=noninteractive
 
 DIRS_TO_TRIM="/usr/share/man
@@ -36,10 +41,39 @@ DIRS_TO_TRIM="/usr/share/man
 /usr/share/info
 "
 
+debootstrap_arch_args=( )
+
+if use_qemu_static ; then
+    debootstrap_arch_args+=( --arch "$PLATFORM" )
+fi
+
 rootfsDir=$(mktemp -d)
 echo "Building base in $rootfsDir"
-DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir"
-chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage
+DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap "${debootstrap_arch_args[@]}"  --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir"
+
+# get path to "chroot" in our current PATH
+chrootPath="$(type -P chroot)"
+rootfs_chroot() {
+    # "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately!
+    # set PATH and chroot away!
+    PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
+            "$chrootPath" "$rootfsDir" "$@"
+
+}
+
+if use_qemu_static ; then
+    echo "Setting up qemu static in chroot"
+    usr_bin_modification_time=$(stat -c %y "$rootfsDir"/usr/bin)
+    if [ -f "/usr/bin/qemu-aarch64-static" ]; then
+        find /usr/bin/ -type f -name 'qemu-*-static' -exec cp {} "$rootfsDir"/usr/bin/. \;
+    else
+        echo "Cannot find aarch64 qemu static. Aborting..." >&2
+        exit 1
+    fi
+    touch -d "$usr_bin_modification_time" "$rootfsDir"/usr/bin
+fi
+
+rootfs_chroot bash debootstrap/debootstrap --second-stage
 
 repo_url="http://deb.debian.org/debian"
 sec_repo_url="http://security.debian.org/"
@@ -54,10 +88,10 @@ if [ "$DIST" != "unstable" ]; then
     echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list"
 fi
 
-chroot "$rootfsDir" apt-get update
-chroot "$rootfsDir" apt-get upgrade -y -o Dpkg::Options::="--force-confdef"
+rootfs_chroot apt-get update
+rootfs_chroot apt-get upgrade -y -o Dpkg::Options::="--force-confdef"
 
-chroot "$rootfsDir" dpkg -l | tee "$TARGET.manifest"
+rootfs_chroot dpkg -l | tee "$TARGET.manifest"
 
 echo "Applying docker-specific tweaks"
 # These are copied from the docker contrib/mkimage/debootstrap script.
@@ -67,15 +101,6 @@ echo "Applying docker-specific tweaks"
 #     interested in building versions that this guard would apply to,
 #     so simply apply the tweak unconditionally.
 
-# get path to "chroot" in our current PATH
-chrootPath="$(type -P chroot)"
-rootfs_chroot() {
-    # "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately!
-    # set PATH and chroot away!
-    PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
-            "$chrootPath" "$rootfsDir" "$@"
-
-}
 
 # prevent init scripts from running during install/update
 echo >&2 "+ echo exit 101 > '$rootfsDir/usr/sbin/policy-rc.d'"
@@ -221,7 +246,16 @@ echo "host" > "$rootfsDir/etc/hostname"
 # Capture the most recent date that a package in the image was changed.
 # We don't care about the particular date, or which package it comes from,
 # we just need a date that isn't very far in the past.
+
+# We get multiple errors like:
+# gzip: stdout: Broken pipe
+# dpkg-parsechangelog: error: gunzip gave error exit status 1
+#
+# TODO: Why?
+set +o pipefail
 BUILD_DATE="$(find "$rootfsDir/usr/share/doc" -name changelog.Debian.gz -print0 | xargs -0 -n1 -I{} dpkg-parsechangelog -SDate -l'{}' | xargs -l -i date --date="{}" +%s | sort -n | tail -n 1)"
+set -o pipefail
+
 
 echo "Trimming down"
 for DIR in $DIRS_TO_TRIM; do
@@ -252,6 +286,13 @@ echo "Largest dirs"
 du "$rootfsDir" | sort -n | tail -n 20
 echo "Built in $rootfsDir"
 
+if use_qemu_static ; then
+    echo "Cleaning up qemu static files from image"
+    usr_bin_modification_time=$(stat -c %y "$rootfsDir"/usr/bin)
+    rm -rf "$rootfsDir"/usr/bin/qemu-*-static
+    touch -d "$usr_bin_modification_time" "$rootfsDir"/usr/bin
+fi
+
 tar cf "$TARGET" -C "$rootfsDir" .
 rm -r "$rootfsDir"
 rm -r "$DEBOOTSTRAP_DIR"

pre-build.sh

@@ -10,10 +10,16 @@ if [[ ! -f /etc/debian_version ]]; then
 fi
 
 apt-get update
-apt-get install -y debootstrap debian-archive-keyring jq dpkg-dev gnupg apt-transport-https ca-certificates
+apt-get install -y debootstrap debian-archive-keyring jq dpkg-dev gnupg apt-transport-https ca-certificates curl gpg
 
-echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+if ! command -v gcloud &> /dev/null
+then
+    echo "Installing gcloud"
+    echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 
-apt-get update
-apt-get install -y google-cloud-sdk
+    apt-get update
+    apt-get install -y google-cloud-sdk
+else
+    echo "gcloud is installed"
+fi

pushmanifest

@@ -15,7 +15,7 @@ GCR_BASENAME=gcr.io/bitnami-containers/minideb
 QUAY_BASENAME=quay.io/bitnami/minideb
 PLATFORMS=${PLATFORMS:-amd64 arm64}
 DRY_RUN=${DRY_RUN:-}
-
+SNAPSHOT_ID=${SNAPSHOT_ID:-}
 read -r -a ARCHS <<<"$PLATFORMS"
 
 run_docker() {
@@ -42,16 +42,18 @@ list_includes() {
 }
 
 if [ -n "${DOCKER_PASSWORD:-}" ]; then
-    run_docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
+    echo "$DOCKER_PASSWORD" | run_docker login -u "$DOCKER_USERNAME" --password-stdin
 fi
 
 if [ -n "${QUAY_PASSWORD:-}" ]; then
-    run_docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io
+    echo "${QUAY_PASSWORD}" | run_docker login -u "$QUAY_USERNAME" --password-stdin quay.io
 fi
 
 if [ -n "${GCR_KEY:-}" ]; then
     gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY")
     gcloud auth print-access-token | run_docker login -u oauth2accesstoken --password-stdin gcr.io
+elif [ -n "${GCR_TOKEN:-}" ]; then
+    echo "${GCR_TOKEN:-}" | run_docker login -u oauth2accesstoken --password-stdin gcr.io
 fi
 
 push_manifest() {
@@ -72,7 +74,7 @@ tags=()
 for DIST in $DISTS; do
     tags+=("$DIST")
     if list_includes "$DISTS_WITH_SNAPSHOT" "$DIST" ; then
-        tags+=("$DIST-snapshot-$(./snapshot_id)")
+        tags+=("$DIST-snapshot-${SNAPSHOT_ID:-$(./snapshot_id)}")
     fi
 done
 
@@ -83,7 +85,7 @@ if [[ -n "${QUAY_PASSWORD:-}" ]]; then
 else
     echo "Skipping repository quay.io (empty password)"
 fi
-if [[ -n "${GCR_KEY:-}" ]]; then
+if [[ -n "${GCR_KEY:-}" || -n "${GCR_TOKEN:-}" ]]; then
     repositories+=("$GCR_BASENAME")
 else
     echo "Skipping repository gcr.io (empty password)"

pushone

@@ -12,15 +12,18 @@ GCR_BASENAME=gcr.io/bitnami-containers/minideb
 QUAY_BASENAME=quay.io/bitnami/minideb
 
 if [ -n "${DOCKER_PASSWORD:-}" ]; then
-    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
+    echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 fi
 
 if [ -n "${QUAY_PASSWORD:-}" ]; then
-    docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io
+    echo "$QUAY_PASSWORD" | docker login -u "$QUAY_USERNAME" --password-stdin quay.io
 fi
 
 if [ -n "${GCR_KEY:-}" ]; then
     gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY")
+    gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin gcr.io
+elif [ -n "${GCR_TOKEN:-}" ]; then
+    echo "${GCR_TOKEN:-}" | docker login -u oauth2accesstoken --password-stdin gcr.io
 fi
 
 ENABLE_DOCKER_CONTENT_TRUST=0
@@ -37,10 +40,11 @@ push() {
     local dist="$1"
     DOCKER_CONTENT_TRUST=${ENABLE_DOCKER_CONTENT_TRUST} docker push "${BASENAME}:${dist}"
     docker push "${QUAY_BASENAME}:${dist}"
-    gcloud docker -- push "${GCR_BASENAME}:${dist}"
+    docker push "${GCR_BASENAME}:${dist}"
 }
 
 docker tag "${BASENAME}:${DIST}-${PLATFORM}" "${QUAY_BASENAME}:${DIST}-${PLATFORM}"
 docker tag "${BASENAME}:${DIST}-${PLATFORM}" "${GCR_BASENAME}:${DIST}-${PLATFORM}"
 
 push "$DIST-${PLATFORM}"
+

test

@@ -4,7 +4,7 @@ set -eu
 
 IMAGE_ID=$1
 DIST=$2
-
+PLATFORM=${3:-amd64}
 function desc() {
     echo "================================"
     echo -n "TEST: "
@@ -16,10 +16,24 @@ function test() {
     test_extra_args '' "$@"
 }
 
+bind_mounts=( )
+docker_platform_args=( )
+if [[ "$PLATFORM" == "arm64" ]]; then
+    if [[ "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ]]; then
+        echo "Running in arm host. QEMU is not needed"
+    else
+        echo "Setting up qemu static"
+        for qemu_static_file in /usr/bin/qemu-*-static; do
+            bind_mounts+=( -v="$qemu_static_file:$qemu_static_file" )
+        done
+        docker_platform_args+=( --platform "linux/arm64" )
+    fi
+fi
+
 function test_extra_args() {
     local extra_args=$1
     shift
-    docker run --rm $extra_args -e DEBIAN_FRONTEND=noninteractive "$IMAGE_ID" "$@"
+    docker run "${docker_platform_args[@]}" --rm "${bind_mounts[@]}" $extra_args -e DEBIAN_FRONTEND=noninteractive "$IMAGE_ID" "$@"
     echo ""
     echo TEST: OK
     echo ""
@@ -28,6 +42,15 @@ function test_extra_args() {
 desc "Checking that apt is installed"
 test dpkg -l apt
 
+desc "Arch matches"
+if [[ "$PLATFORM" == "amd64" ]]; then
+    test bash -c 'echo "$(uname -m)" && [[ "$(uname -m)" == *x86_64* ]]'
+elif [[ "$PLATFORM" == "arm64" ]]; then
+    test bash -c 'echo "$(uname -m)" && [[ "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ]]'
+else
+    echo Unknown platform $PLATFORM >&2
+    exit 1
+fi
 desc "Checking that a package can be installed with apt"
 test bash -c 'apt-get update && apt-get -y install less && less --help >/dev/null'