Migrate to GitHub actions (#103)

This commit is contained in:
Juan Jose Medina
2021-03-10 16:37:37 +01:00
committed by GitHub
parent 092c369201
commit 98de255d46
9 changed files with 284 additions and 34 deletions
+52
View File
@@ -0,0 +1,52 @@
inputs:
dist:
description: 'Dist to build'
default: 'buster'
platform:
description: 'Platform to build'
default: 'amd64'
is_latest:
description: The created dist is also latest
default: false
build_snapshot:
description: Build snapshot build
default: false
outputs:
snapshot-id:
description: "Created snapshot id if requested to build it"
value: ${{ steps.snapshot-id.outputs.snapshot-id }}
runs:
using: "composite"
steps:
- name: Fix for Ubuntu Xenial apt-daily.service triggering
run: |
while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
sleep 1
done
shell: bash
- run: sudo make .installed-requirements
shell: bash
- name: Build snapshot id
id: snapshot-id
run: |
if ${{ inputs.build_snapshot }} ; then
echo "::set-output name=snapshot-id::$(./snapshot_id)"
fi
shell: bash
- name: "Build image"
run: |
set -x
echo Building ${{ inputs.dist }} - ${{ inputs.platform }}
if [[ "${{ inputs.platform }}" == "arm64" ]]; then
sudo apt-get update -qq && sudo apt-get install -y qemu-user-static
fi
sudo -E bash -x buildone "${{ inputs.dist }}" "${{ inputs.platform }}"
if ${{ inputs.build_snapshot }} ; then
sudo -E bash -x buildone_snapshot "${{ inputs.dist }}" "${{ steps.snapshot-id.outputs.snapshot-id }}" "${{ inputs.platform }}"
fi
if ${{ inputs.is_latest }} ; then
BASENAME=${BASENAME:?Undefined or empty BASENAME}
echo "Tagging latest"
docker tag $BASENAME:${{ inputs.dist }}-${{ inputs.platform }} $BASENAME:latest-${{ inputs.platform }}
fi
shell: bash
+122
View File
@@ -0,0 +1,122 @@
# This is a basic workflow to help you get started with Actions
name: CI
# Controls when the action will run.
on:
# Triggers the workflow on push or pull request events but only for the master branch
push:
branches:
- master
pull_request:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
env:
BASENAME: bitnami/minideb
LATEST: buster
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
shellcheck:
# The type of runner that the job will run on
runs-on: ubuntu-20.04
name: Shellcheck
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- name: Install Dependencies
run: |
sudo apt-get -qq update
sudo apt-get install -y shellcheck
- name: Verify scripts with shellcheck
run: |
bash shellcheck
build_jessie:
runs-on: ubuntu-20.04
needs: [ shellcheck ]
steps:
- name: Check out repository
uses: actions/checkout@v2
- name: Use local build action
uses: ./.github/actions/build
with:
dist: "jessie"
platform: "amd64"
- name: Push
if: github.ref == 'refs/heads/master'
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
GCR_EMAIL: ${{ secrets.GCR_EMAIL }}
GCR_KEY: ${{ secrets.GCR_KEY }}
GCR_TOKEN: ${{ secrets.GCR_TOKEN }}
DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }}
DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }}
run: |
bash pushone jessie amd64
build_multiarch:
runs-on: ubuntu-20.04
needs: [ shellcheck ]
strategy:
matrix:
dist: [stretch, buster]
arch: [amd64, arm64]
name: Build ${{ matrix.dist }} on ${{ matrix.arch }}
steps:
- name: Check out repository
uses: actions/checkout@v2
- name: Use local build action
id: build
uses: ./.github/actions/build
with:
dist: "${{ matrix.dist }}"
platform: "${{ matrix.arch }}"
is_latest: ${{ matrix.dist == env.LATEST }}
build_snapshot: ${{ matrix.dist == env.LATEST }}
- name: Push
if: github.ref == 'refs/heads/master'
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
GCR_EMAIL: ${{ secrets.GCR_EMAIL }}
GCR_KEY: ${{ secrets.GCR_KEY }}
GCR_TOKEN: ${{ secrets.GCR_TOKEN }}
DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }}
DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }}
run: |
bash pushone "${{ matrix.dist }}" "${{ matrix.arch }}"
if ${{ matrix.dist == env.LATEST }} ; then
bash pushone "latest" "${{ matrix.arch }}"
fi
if ${{ matrix.dist == env.LATEST }} ; then
bash pushone "${{ matrix.dist }}-snapshot-${{ steps.build.outputs.snapshot-id }}" "${{ matrix.arch }}"
fi
deploy_manifests:
runs-on: ubuntu-20.04
needs: [ build_multiarch, build_jessie ]
if: github.ref == 'refs/heads/master'
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
GCR_EMAIL: ${{ secrets.GCR_EMAIL }}
GCR_KEY: ${{ secrets.GCR_KEY }}
GCR_TOKEN: ${{ secrets.GCR_TOKEN }}
DOCKER_CONTENT_TRUST_REPOSITORY_KEY: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_KEY }}
DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE }}
steps:
- uses: actions/checkout@v2
- name: Push Manifests
run: |
DISTS="stretch buster latest" bash pushmanifest
DISTS=jessie PLATFORMS=amd64 bash pushmanifest
+1 -1
View File
@@ -1,4 +1,4 @@
build
/build
.kvm-images
.installed-requirements
.installed-qemu
+3 -3
View File
@@ -64,16 +64,16 @@ build() {
log "============================================"
log "Building $BASENAME:$TAG"
log "============================================"
./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}"
./mkimage "build/$TAG.tar" "$DIST" "$PLATFORM" "${debian_snapshot_id:-}"
built_image_id=$(./import "build/$TAG.tar" "$target_ts" "$PLATFORM")
log "============================================"
log "Running tests for $BASENAME:$TAG"
log "============================================"
./test "$built_image_id" "$TAG"
./test "$built_image_id" "$TAG" "$PLATFORM"
log "============================================"
log "Rebuilding $BASENAME:$TAG to test reproducibility"
log "============================================"
./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}"
./mkimage "build/${TAG}-repro.tar" "$DIST" "$PLATFORM" "${debian_snapshot_id:-}"
repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts" "$PLATFORM")
if [ "$repro_image_id" != "$built_image_id" ]; then
log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id"
+56 -15
View File
@@ -7,7 +7,8 @@ ROOT=$(cd "$(dirname "$0")" && pwd)
TARGET=${1:?Specify the target filename}
DIST=${2:-stable}
SNAPSHOT_ID=${3:-}
PLATFORM=${3:-$(dpkg --print-architecture)}
SNAPSHOT_ID=${4:-}
LOGFILE=${TARGET}.log
@@ -26,6 +27,10 @@ if [ -f "${ROOT}/keys/${DIST}.gpg" ]; then
gpg --no-default-keyring --keyring "$KEYRING" --import "${ROOT}/keys/${DIST}.gpg"
fi
use_qemu_static() {
[[ "$PLATFORM" == "arm64" && ! ( "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ) ]]
}
export DEBIAN_FRONTEND=noninteractive
DIRS_TO_TRIM="/usr/share/man
@@ -36,10 +41,39 @@ DIRS_TO_TRIM="/usr/share/man
/usr/share/info
"
debootstrap_arch_args=( )
if use_qemu_static ; then
debootstrap_arch_args+=( --arch "$PLATFORM" )
fi
rootfsDir=$(mktemp -d)
echo "Building base in $rootfsDir"
DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir"
chroot "$rootfsDir" bash debootstrap/debootstrap --second-stage
DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" debootstrap "${debootstrap_arch_args[@]}" --keyring "$KEYRING" --variant container --foreign "${DIST}" "$rootfsDir"
# get path to "chroot" in our current PATH
chrootPath="$(type -P chroot)"
rootfs_chroot() {
# "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately!
# set PATH and chroot away!
PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
"$chrootPath" "$rootfsDir" "$@"
}
if use_qemu_static ; then
echo "Setting up qemu static in chroot"
usr_bin_modification_time=$(stat -c %y "$rootfsDir"/usr/bin)
if [ -f "/usr/bin/qemu-aarch64-static" ]; then
find /usr/bin/ -type f -name 'qemu-*-static' -exec cp {} "$rootfsDir"/usr/bin/. \;
else
echo "Cannot find aarch64 qemu static. Aborting..." >&2
exit 1
fi
touch -d "$usr_bin_modification_time" "$rootfsDir"/usr/bin
fi
rootfs_chroot bash debootstrap/debootstrap --second-stage
repo_url="http://deb.debian.org/debian"
sec_repo_url="http://security.debian.org/"
@@ -54,10 +88,10 @@ if [ "$DIST" != "unstable" ]; then
echo "deb ${sec_repo_url} $DIST/updates main" >> "$rootfsDir/etc/apt/sources.list"
fi
chroot "$rootfsDir" apt-get update
chroot "$rootfsDir" apt-get upgrade -y -o Dpkg::Options::="--force-confdef"
rootfs_chroot apt-get update
rootfs_chroot apt-get upgrade -y -o Dpkg::Options::="--force-confdef"
chroot "$rootfsDir" dpkg -l | tee "$TARGET.manifest"
rootfs_chroot dpkg -l | tee "$TARGET.manifest"
echo "Applying docker-specific tweaks"
# These are copied from the docker contrib/mkimage/debootstrap script.
@@ -67,15 +101,6 @@ echo "Applying docker-specific tweaks"
# interested in building versions that this guard would apply to,
# so simply apply the tweak unconditionally.
# get path to "chroot" in our current PATH
chrootPath="$(type -P chroot)"
rootfs_chroot() {
# "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately!
# set PATH and chroot away!
PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
"$chrootPath" "$rootfsDir" "$@"
}
# prevent init scripts from running during install/update
echo >&2 "+ echo exit 101 > '$rootfsDir/usr/sbin/policy-rc.d'"
@@ -221,7 +246,16 @@ echo "host" > "$rootfsDir/etc/hostname"
# Capture the most recent date that a package in the image was changed.
# We don't care about the particular date, or which package it comes from,
# we just need a date that isn't very far in the past.
# We get multiple errors like:
# gzip: stdout: Broken pipe
# dpkg-parsechangelog: error: gunzip gave error exit status 1
#
# TODO: Why?
set +o pipefail
BUILD_DATE="$(find "$rootfsDir/usr/share/doc" -name changelog.Debian.gz -print0 | xargs -0 -n1 -I{} dpkg-parsechangelog -SDate -l'{}' | xargs -l -i date --date="{}" +%s | sort -n | tail -n 1)"
set -o pipefail
echo "Trimming down"
for DIR in $DIRS_TO_TRIM; do
@@ -252,6 +286,13 @@ echo "Largest dirs"
du "$rootfsDir" | sort -n | tail -n 20
echo "Built in $rootfsDir"
if use_qemu_static ; then
echo "Cleaning up qemu static files from image"
usr_bin_modification_time=$(stat -c %y "$rootfsDir"/usr/bin)
rm -rf "$rootfsDir"/usr/bin/qemu-*-static
touch -d "$usr_bin_modification_time" "$rootfsDir"/usr/bin
fi
tar cf "$TARGET" -C "$rootfsDir" .
rm -r "$rootfsDir"
rm -r "$DEBOOTSTRAP_DIR"
+11 -5
View File
@@ -10,10 +10,16 @@ if [[ ! -f /etc/debian_version ]]; then
fi
apt-get update
apt-get install -y debootstrap debian-archive-keyring jq dpkg-dev gnupg apt-transport-https ca-certificates
apt-get install -y debootstrap debian-archive-keyring jq dpkg-dev gnupg apt-transport-https ca-certificates curl gpg
echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
if ! command -v gcloud &> /dev/null
then
echo "Installing gcloud"
echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
apt-get update
apt-get install -y google-cloud-sdk
apt-get update
apt-get install -y google-cloud-sdk
else
echo "gcloud is installed"
fi
+7 -5
View File
@@ -15,7 +15,7 @@ GCR_BASENAME=gcr.io/bitnami-containers/minideb
QUAY_BASENAME=quay.io/bitnami/minideb
PLATFORMS=${PLATFORMS:-amd64 arm64}
DRY_RUN=${DRY_RUN:-}
SNAPSHOT_ID=${SNAPSHOT_ID:-}
read -r -a ARCHS <<<"$PLATFORMS"
run_docker() {
@@ -42,16 +42,18 @@ list_includes() {
}
if [ -n "${DOCKER_PASSWORD:-}" ]; then
run_docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
echo "$DOCKER_PASSWORD" | run_docker login -u "$DOCKER_USERNAME" --password-stdin
fi
if [ -n "${QUAY_PASSWORD:-}" ]; then
run_docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io
echo "${QUAY_PASSWORD}" | run_docker login -u "$QUAY_USERNAME" --password-stdin quay.io
fi
if [ -n "${GCR_KEY:-}" ]; then
gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY")
gcloud auth print-access-token | run_docker login -u oauth2accesstoken --password-stdin gcr.io
elif [ -n "${GCR_TOKEN:-}" ]; then
echo "${GCR_TOKEN:-}" | run_docker login -u oauth2accesstoken --password-stdin gcr.io
fi
push_manifest() {
@@ -72,7 +74,7 @@ tags=()
for DIST in $DISTS; do
tags+=("$DIST")
if list_includes "$DISTS_WITH_SNAPSHOT" "$DIST" ; then
tags+=("$DIST-snapshot-$(./snapshot_id)")
tags+=("$DIST-snapshot-${SNAPSHOT_ID:-$(./snapshot_id)}")
fi
done
@@ -83,7 +85,7 @@ if [[ -n "${QUAY_PASSWORD:-}" ]]; then
else
echo "Skipping repository quay.io (empty password)"
fi
if [[ -n "${GCR_KEY:-}" ]]; then
if [[ -n "${GCR_KEY:-}" || -n "${GCR_TOKEN:-}" ]]; then
repositories+=("$GCR_BASENAME")
else
echo "Skipping repository gcr.io (empty password)"
+7 -3
View File
@@ -12,15 +12,18 @@ GCR_BASENAME=gcr.io/bitnami-containers/minideb
QUAY_BASENAME=quay.io/bitnami/minideb
if [ -n "${DOCKER_PASSWORD:-}" ]; then
docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
fi
if [ -n "${QUAY_PASSWORD:-}" ]; then
docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io
echo "$QUAY_PASSWORD" | docker login -u "$QUAY_USERNAME" --password-stdin quay.io
fi
if [ -n "${GCR_KEY:-}" ]; then
gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY")
gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin gcr.io
elif [ -n "${GCR_TOKEN:-}" ]; then
echo "${GCR_TOKEN:-}" | docker login -u oauth2accesstoken --password-stdin gcr.io
fi
ENABLE_DOCKER_CONTENT_TRUST=0
@@ -37,10 +40,11 @@ push() {
local dist="$1"
DOCKER_CONTENT_TRUST=${ENABLE_DOCKER_CONTENT_TRUST} docker push "${BASENAME}:${dist}"
docker push "${QUAY_BASENAME}:${dist}"
gcloud docker -- push "${GCR_BASENAME}:${dist}"
docker push "${GCR_BASENAME}:${dist}"
}
docker tag "${BASENAME}:${DIST}-${PLATFORM}" "${QUAY_BASENAME}:${DIST}-${PLATFORM}"
docker tag "${BASENAME}:${DIST}-${PLATFORM}" "${GCR_BASENAME}:${DIST}-${PLATFORM}"
push "$DIST-${PLATFORM}"
+25 -2
View File
@@ -4,7 +4,7 @@ set -eu
IMAGE_ID=$1
DIST=$2
PLATFORM=${3:-amd64}
function desc() {
echo "================================"
echo -n "TEST: "
@@ -16,10 +16,24 @@ function test() {
test_extra_args '' "$@"
}
bind_mounts=( )
docker_platform_args=( )
if [[ "$PLATFORM" == "arm64" ]]; then
if [[ "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ]]; then
echo "Running in arm host. QEMU is not needed"
else
echo "Setting up qemu static"
for qemu_static_file in /usr/bin/qemu-*-static; do
bind_mounts+=( -v="$qemu_static_file:$qemu_static_file" )
done
docker_platform_args+=( --platform "linux/arm64" )
fi
fi
function test_extra_args() {
local extra_args=$1
shift
docker run --rm $extra_args -e DEBIAN_FRONTEND=noninteractive "$IMAGE_ID" "$@"
docker run "${docker_platform_args[@]}" --rm "${bind_mounts[@]}" $extra_args -e DEBIAN_FRONTEND=noninteractive "$IMAGE_ID" "$@"
echo ""
echo TEST: OK
echo ""
@@ -28,6 +42,15 @@ function test_extra_args() {
desc "Checking that apt is installed"
test dpkg -l apt
desc "Arch matches"
if [[ "$PLATFORM" == "amd64" ]]; then
test bash -c 'echo "$(uname -m)" && [[ "$(uname -m)" == *x86_64* ]]'
elif [[ "$PLATFORM" == "arm64" ]]; then
test bash -c 'echo "$(uname -m)" && [[ "$(uname -m)" == *arm* || "$(uname -m)" == *aarch64* ]]'
else
echo Unknown platform $PLATFORM >&2
exit 1
fi
desc "Checking that a package can be installed with apt"
test bash -c 'apt-get update && apt-get -y install less && less --help >/dev/null'