From 755ae001f9eaf1a29983f2e377e16725660b35d8 Mon Sep 17 00:00:00 2001 From: Eoin McAfee <83226740+eoinmcafee00@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:41:06 +0100 Subject: [PATCH] ci-3011- split out gar/gcr (#415) * split out gar/gcr --- .drone.yml | 186 ++++++++++++++++++- README.md | 12 +- cmd/drone-gar/main.go | 165 ++++++++++++++++ cmd/drone-gcr/main.go | 81 +------- docker/gar/Dockerfile.linux.amd64 | 4 + docker/gar/Dockerfile.linux.arm64 | 4 + docker/gar/Dockerfile.windows.amd64.1809 | 10 + docker/gar/Dockerfile.windows.amd64.ltsc2022 | 10 + docker/gar/manifest.tmpl | 31 ++++ internal/gcp/tokenutil.go | 65 +++++++ 10 files changed, 484 insertions(+), 84 deletions(-) create mode 100644 cmd/drone-gar/main.go create mode 100644 docker/gar/Dockerfile.linux.amd64 create mode 100644 docker/gar/Dockerfile.linux.arm64 create mode 100644 docker/gar/Dockerfile.windows.amd64.1809 create mode 100644 docker/gar/Dockerfile.windows.amd64.ltsc2022 create mode 100644 docker/gar/manifest.tmpl create mode 100644 internal/gcp/tokenutil.go diff --git a/.drone.yml b/.drone.yml index 065e6f8..b930a4d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -63,6 +63,8 @@ steps: - go build -o release/windows/amd64/drone-ecr.exe ./cmd/drone-ecr - go build -o release/windows/amd64/drone-gcr.exe ./cmd/drone-gcr - go build -o release/windows/amd64/drone-acr.exe ./cmd/drone-acr + - go build -o release/windows/amd64/drone-gcr.exe ./cmd/drone-gar + - name: build docker plugin image: plugins/docker@sha256:f0233d950ae87ee6cb5500b2d5497fe02aa338201c0bdce2619f443fd174cfa4 settings: @@ -123,7 +125,21 @@ steps: purge: false when: event: [push, tag] - + - name: build gar plugin + image: plugins/docker@sha256:f0233d950ae87ee6cb5500b2d5497fe02aa338201c0bdce2619f443fd174cfa4 + pull: never + settings: + dockerfile: docker/gar/Dockerfile.windows.amd64.1809 + repo: plugins/gar + username: + from_secret: docker_username + password: + from_secret: docker_password + auto_tag: true + auto_tag_suffix: windows-1809-amd64 + purge: false + when: + event: [push, tag] depends_on: - testing @@ -154,6 +170,7 @@ steps: - go build -o release/windows/amd64/drone-ecr.exe ./cmd/drone-ecr - go build -o release/windows/amd64/drone-gcr.exe ./cmd/drone-gcr - go build -o release/windows/amd64/drone-acr.exe ./cmd/drone-acr + - go build -o release/windows/amd64/drone-gcr.exe ./cmd/drone-gar - name: build docker plugin image: plugins/docker settings: @@ -210,7 +227,20 @@ steps: purge: false when: event: [push, tag] - + - name: build gar plugin + image: plugins/docker + settings: + dockerfile: docker/gar/Dockerfile.windows.amd64.ltsc2022 + repo: plugins/gar + username: + from_secret: docker_username + password: + from_secret: docker_password + auto_tag: true + auto_tag_suffix: windows-ltsc2022-amd64 + purge: false + when: + event: [push, tag] depends_on: - testing @@ -444,7 +474,6 @@ trigger: depends_on: - linux-amd64-docker - --- kind: pipeline name: linux-arm64-gcr @@ -539,7 +568,158 @@ depends_on: - windows-ltsc2022 - linux-amd64-gcr - linux-arm64-gcr +--- +kind: pipeline +name: linux-amd64-gar +type: vm +pool: + use: ubuntu + +platform: + os: linux + arch: amd64 + +steps: + - name: build-push + image: golang:1.21 + commands: + - 'go build -v -ldflags "-X main.version=${DRONE_COMMIT_SHA:0:8}" -a -tags netgo -o release/linux/amd64/drone-gar ./cmd/drone-gar' + environment: + CGO_ENABLED: 0 + when: + event: + exclude: + - tag + + - name: build-tag + image: golang:1.21 + commands: + - 'go build -v -ldflags "-X main.version=${DRONE_TAG##v}" -a -tags netgo -o release/linux/amd64/drone-gar ./cmd/drone-gar' + environment: + CGO_ENABLED: 0 + when: + event: + - tag + + - name: publish + image: plugins/docker:18 + settings: + auto_tag: true + auto_tag_suffix: linux-amd64 + daemon_off: false + dockerfile: docker/gar/Dockerfile.linux.amd64 + password: + from_secret: docker_password + repo: plugins/gar + username: + from_secret: docker_username + when: + event: + exclude: + - pull_request + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" + - "refs/pull/**" + +depends_on: + - linux-amd64-docker +--- +kind: pipeline +name: linux-arm64-gar +type: vm + +pool: + use: ubuntu_arm64 + +platform: + os: linux + arch: arm64 + +steps: + - name: build-push + image: golang:1.21 + commands: + - 'go build -v -ldflags "-X main.version=${DRONE_COMMIT_SHA:0:8}" -a -tags netgo -o release/linux/arm64/drone-gar ./cmd/drone-gar' + environment: + CGO_ENABLED: 0 + when: + event: + exclude: + - tag + + - name: build-tag + image: golang:1.21 + commands: + - 'go build -v -ldflags "-X main.version=${DRONE_TAG##v}" -a -tags netgo -o release/linux/arm64/drone-gar ./cmd/drone-gar' + environment: + CGO_ENABLED: 0 + when: + event: + - tag + + - name: publish + image: plugins/docker:18 + settings: + auto_tag: true + auto_tag_suffix: linux-arm64 + daemon_off: false + dockerfile: docker/gar/Dockerfile.linux.arm64 + password: + from_secret: docker_password + repo: plugins/gar + username: + from_secret: docker_username + when: + event: + exclude: + - pull_request + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" + - "refs/pull/**" + +depends_on: + - linux-arm64-docker +--- +kind: pipeline +name: notifications-gar +type: vm + +pool: + use: ubuntu + +platform: + os: linux + arch: amd64 + +steps: + - name: manifest + image: plugins/manifest + settings: + auto_tag: true + ignore_missing: true + password: + from_secret: docker_password + spec: docker/gar/manifest.tmpl + username: + from_secret: docker_username + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" + +depends_on: + - windows-1809 + - windows-ltsc2022 + - linux-amd64-gar + - linux-arm64-gar --- kind: pipeline name: linux-amd64-ecr diff --git a/README.md b/README.md index bef4c14..b40d7c3 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ go build -v -a -tags netgo -o release/linux/amd64/drone-gcr ./cmd/drone-gcr go build -v -a -tags netgo -o release/linux/amd64/drone-ecr ./cmd/drone-ecr go build -v -a -tags netgo -o release/linux/amd64/drone-acr ./cmd/drone-acr go build -v -a -tags netgo -o release/linux/amd64/drone-heroku ./cmd/drone-heroku +go build -v -a -tags netgo -o release/linux/amd64/drone-gar ./cmd/drone-gar ``` ## Docker @@ -56,6 +57,11 @@ docker build \ --label org.label-schema.build-date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ --label org.label-schema.vcs-ref=$(git rev-parse --short HEAD) \ --file docker/heroku/Dockerfile.linux.amd64 --tag plugins/heroku . + +docker build \ + --label org.label-schema.build-date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --label org.label-schema.vcs-ref=$(git rev-parse --short HEAD) \ + --file docker/gar/Dockerfile.linux.amd64 --tag plugins/gar . ``` ## Usage @@ -122,12 +128,11 @@ type: docker steps: - name: push-to-gar - image: plugins/gcr + image: plugins/gar pull: never settings: tag: latest repo: project-id/repo/image-name - registry_type: GAR location: us json_key: from_secret: gcr_json_key @@ -138,12 +143,11 @@ steps: ```yaml steps: - name: push-to-gar - image: plugins/gcr + image: plugins/gar pull: never settings: tag: latest repo: project-id/repo/image-name - registry_type: GAR location: europe project_number: project-number pool_id: workload identity pool id diff --git a/cmd/drone-gar/main.go b/cmd/drone-gar/main.go new file mode 100644 index 0000000..235b075 --- /dev/null +++ b/cmd/drone-gar/main.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "log" + "os" + "os/exec" + "path" + "strconv" + "strings" + + docker "github.com/drone-plugins/drone-docker" + "github.com/drone-plugins/drone-docker/internal/gcp" + + "github.com/joho/godotenv" + "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" +) + +type Config struct { + Repo string + Registry string + Password string + WorkloadIdentity bool + Username string + AccessToken string +} + +type staticTokenSource struct { + token *oauth2.Token +} + +func (s *staticTokenSource) Token() (*oauth2.Token, error) { + return s.token, nil +} + +func loadConfig() Config { + // Default username + username := "_json_key" + var config Config + + // Load env-file if it exists + if env := os.Getenv("PLUGIN_ENV_FILE"); env != "" { + if err := godotenv.Load(env); err != nil { + log.Fatalf("Error loading .env file: %v", err) + } + } + + idToken := getenv("PLUGIN_OIDC_TOKEN_ID") + projectId := getenv("PLUGIN_PROJECT_NUMBER") + poolId := getenv("PLUGIN_POOL_ID") + providerId := getenv("PLUGIN_PROVIDER_ID") + serviceAccountEmail := getenv("PLUGIN_SERVICE_ACCOUNT_EMAIL") + + if idToken != "" && projectId != "" && poolId != "" && providerId != "" && serviceAccountEmail != "" { + federalToken, err := gcp.GetFederalToken(idToken, projectId, poolId, providerId) + if err != nil { + logrus.Fatalf("Error (getFederalToken): %s", err) + } + accessToken, err := gcp.GetGoogleCloudAccessToken(federalToken, serviceAccountEmail) + if err != nil { + logrus.Fatalf("Error (getGoogleCloudAccessToken): %s", err) + } + config.AccessToken = accessToken + } else { + password := getenv( + "PLUGIN_JSON_KEY", + "GCR_JSON_KEY", + "GOOGLE_CREDENTIALS", + "TOKEN", + ) + config.WorkloadIdentity = parseBoolOrDefault(false, getenv("PLUGIN_WORKLOAD_IDENTITY")) + config.Username, config.Password = setUsernameAndPassword(username, password, config.WorkloadIdentity) + } + + location := getenv("PLUGIN_LOCATION") + repo := getenv("PLUGIN_REPO") + + registry := getenv("PLUGIN_REGISTRY") + if registry == "" { + registry = fmt.Sprintf("%s-docker.pkg.dev", location) + } + + if !strings.HasPrefix(repo, registry) { + repo = path.Join(registry, repo) + } + config.Repo = repo + config.Registry = registry + return config +} + +func main() { + config := loadConfig() + if config.AccessToken != "" { + os.Setenv("ACCESS_TOKEN", config.AccessToken) + } else if config.Username != "" && config.Password != "" { + os.Setenv("DOCKER_USERNAME", config.Username) + os.Setenv("DOCKER_PASSWORD", config.Password) + } + + os.Setenv("PLUGIN_REPO", config.Repo) + os.Setenv("PLUGIN_REGISTRY", config.Registry) + + // invoke the base docker plugin binary + cmd := exec.Command(docker.GetDroneDockerExecCmd()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + logrus.Fatal(err) + } +} + +func getOauthToken(data []byte) (s string) { + scopes := []string{ + "https://www.googleapis.com/auth/cloud-platform", + } + ctx := context.Background() + credentials, err := google.CredentialsFromJSON(ctx, data, scopes...) + if err == nil { + token, err := credentials.TokenSource.Token() + if err == nil { + return token.AccessToken + } + } + return +} + +func setUsernameAndPassword(user string, pass string, workloadIdentity bool) (u string, p string) { + // decode the token if base64 encoded + decoded, err := base64.StdEncoding.DecodeString(pass) + if err == nil { + pass = string(decoded) + } + // get oauth token and set username if using workload identity + if workloadIdentity { + data := []byte(pass) + pass = getOauthToken(data) + user = "oauth2accesstoken" + } + return user, pass +} + +func parseBoolOrDefault(defaultValue bool, s string) (result bool) { + var err error + result, err = strconv.ParseBool(s) + if err != nil { + result = defaultValue + } + + return +} + +func getenv(key ...string) (s string) { + for _, k := range key { + s = os.Getenv(k) + if s != "" { + return + } + } + return +} diff --git a/cmd/drone-gcr/main.go b/cmd/drone-gcr/main.go index 8bfacb0..6b099f8 100644 --- a/cmd/drone-gcr/main.go +++ b/cmd/drone-gcr/main.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/base64" - "fmt" "log" "os" "os/exec" @@ -12,14 +11,11 @@ import ( "strings" docker "github.com/drone-plugins/drone-docker" + "github.com/drone-plugins/drone-docker/internal/gcp" "github.com/joho/godotenv" "github.com/sirupsen/logrus" - "golang.org/x/oauth2" "golang.org/x/oauth2/google" - "google.golang.org/api/iamcredentials/v1" - "google.golang.org/api/option" - "google.golang.org/api/sts/v1" ) type Config struct { @@ -28,18 +24,9 @@ type Config struct { Password string WorkloadIdentity bool Username string - RegistryType string AccessToken string } -type staticTokenSource struct { - token *oauth2.Token -} - -func (s *staticTokenSource) Token() (*oauth2.Token, error) { - return s.token, nil -} - func loadConfig() Config { // Default username username := "_json_key" @@ -59,11 +46,11 @@ func loadConfig() Config { serviceAccountEmail := getenv("PLUGIN_SERVICE_ACCOUNT_EMAIL") if idToken != "" && projectId != "" && poolId != "" && providerId != "" && serviceAccountEmail != "" { - federalToken, err := getFederalToken(idToken, projectId, poolId, providerId) + federalToken, err := gcp.GetFederalToken(idToken, projectId, poolId, providerId) if err != nil { logrus.Fatalf("Error (getFederalToken): %s", err) } - accessToken, err := getGoogleCloudAccessToken(federalToken, serviceAccountEmail) + accessToken, err := gcp.GetGoogleCloudAccessToken(federalToken, serviceAccountEmail) if err != nil { logrus.Fatalf("Error (getGoogleCloudAccessToken): %s", err) } @@ -79,9 +66,7 @@ func loadConfig() Config { config.Username, config.Password = setUsernameAndPassword(username, password, config.WorkloadIdentity) } - location := getenv("PLUGIN_LOCATION") repo := getenv("PLUGIN_REPO") - registryType := getenv("PLUGIN_REGISTRY_TYPE") if registryType == "" { registryType = "GCR" @@ -89,17 +74,7 @@ func loadConfig() Config { registry := getenv("PLUGIN_REGISTRY") if registry == "" { - switch registryType { - case "GCR": - registry = "gcr.io" - case "GAR": - if location == "" { - logrus.Fatalf("Error: For REGISTRY_TYPE of GAR, LOCATION must be set") - } - registry = fmt.Sprintf("%s-docker.pkg.dev", location) - default: - logrus.Fatalf("Unsupported registry type: %s", registryType) - } + registry = "gcr.io" } if !strings.HasPrefix(repo, registry) { @@ -107,7 +82,6 @@ func loadConfig() Config { } config.Repo = repo config.Registry = registry - config.RegistryType = registryType return config } @@ -122,7 +96,6 @@ func main() { os.Setenv("PLUGIN_REPO", config.Repo) os.Setenv("PLUGIN_REGISTRY", config.Registry) - os.Setenv("PLUGIN_REGISTRY_TYPE", config.RegistryType) // invoke the base docker plugin binary cmd := exec.Command(docker.GetDroneDockerExecCmd()) @@ -183,49 +156,3 @@ func getenv(key ...string) (s string) { } return } - -func getFederalToken(idToken, projectNumber, poolId, providerId string) (string, error) { - ctx := context.Background() - stsService, err := sts.NewService(ctx, option.WithoutAuthentication()) - if err != nil { - return "", err - } - audience := fmt.Sprintf("//iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s/providers/%s", projectNumber, poolId, providerId) - tokenRequest := &sts.GoogleIdentityStsV1ExchangeTokenRequest{ - GrantType: "urn:ietf:params:oauth:grant-type:token-exchange", - SubjectToken: idToken, - Audience: audience, - Scope: "https://www.googleapis.com/auth/cloud-platform", - RequestedTokenType: "urn:ietf:params:oauth:token-type:access_token", - SubjectTokenType: "urn:ietf:params:oauth:token-type:id_token", - } - tokenResponse, err := stsService.V1.Token(tokenRequest).Do() - if err != nil { - return "", err - } - - return tokenResponse.AccessToken, nil -} - -func getGoogleCloudAccessToken(federatedToken string, serviceAccountEmail string) (string, error) { - ctx := context.Background() - tokenSource := &staticTokenSource{ - token: &oauth2.Token{AccessToken: federatedToken}, - } - service, err := iamcredentials.NewService(ctx, option.WithTokenSource(tokenSource)) - if err != nil { - return "", err - } - - name := "projects/-/serviceAccounts/" + serviceAccountEmail - rb := &iamcredentials.GenerateAccessTokenRequest{ - Scope: []string{"https://www.googleapis.com/auth/cloud-platform"}, - } - - resp, err := service.Projects.ServiceAccounts.GenerateAccessToken(name, rb).Do() - if err != nil { - return "", err - } - - return resp.AccessToken, nil -} diff --git a/docker/gar/Dockerfile.linux.amd64 b/docker/gar/Dockerfile.linux.amd64 new file mode 100644 index 0000000..2e390d8 --- /dev/null +++ b/docker/gar/Dockerfile.linux.amd64 @@ -0,0 +1,4 @@ +FROM plugins/docker:linux-amd64 + +ADD release/linux/amd64/drone-gar /bin/ +ENTRYPOINT ["/usr/local/bin/dockerd-entrypoint.sh", "/bin/drone-gar"] diff --git a/docker/gar/Dockerfile.linux.arm64 b/docker/gar/Dockerfile.linux.arm64 new file mode 100644 index 0000000..809525e --- /dev/null +++ b/docker/gar/Dockerfile.linux.arm64 @@ -0,0 +1,4 @@ +FROM plugins/docker:linux-arm64 + +ADD release/linux/arm64/drone-gar /bin/ +ENTRYPOINT ["/usr/local/bin/dockerd-entrypoint.sh", "/bin/drone-gar"] diff --git a/docker/gar/Dockerfile.windows.amd64.1809 b/docker/gar/Dockerfile.windows.amd64.1809 new file mode 100644 index 0000000..e324210 --- /dev/null +++ b/docker/gar/Dockerfile.windows.amd64.1809 @@ -0,0 +1,10 @@ +# escape=` +FROM plugins/docker:windows-1809-amd64 + +LABEL maintainer="Drone.IO Community " ` + org.label-schema.name="Drone GAR" ` + org.label-schema.vendor="Drone.IO Community" ` + org.label-schema.schema-version="1.0" + +ADD release/windows/amd64/drone-gar.exe C:/bin/drone-gar.exe +ENTRYPOINT [ "C:\\bin\\drone-gar.exe" ] diff --git a/docker/gar/Dockerfile.windows.amd64.ltsc2022 b/docker/gar/Dockerfile.windows.amd64.ltsc2022 new file mode 100644 index 0000000..9891099 --- /dev/null +++ b/docker/gar/Dockerfile.windows.amd64.ltsc2022 @@ -0,0 +1,10 @@ +# escape=` +FROM plugins/docker:windows-ltsc2022-amd64 + +LABEL maintainer="Drone.IO Community " ` + org.label-schema.name="Drone GAR" ` + org.label-schema.vendor="Drone.IO Community" ` + org.label-schema.schema-version="1.0" + +ADD release/windows/amd64/drone-gar.exe C:/bin/drone-gar.exe +ENTRYPOINT [ "C:\\bin\\drone-gar.exe" ] diff --git a/docker/gar/manifest.tmpl b/docker/gar/manifest.tmpl new file mode 100644 index 0000000..8946956 --- /dev/null +++ b/docker/gar/manifest.tmpl @@ -0,0 +1,31 @@ +image: plugins/gar:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - + image: plugins/gar:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 + platform: + architecture: amd64 + os: linux + - + image: plugins/gar:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - + image: plugins/gar:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}windows-1809-amd64 + platform: + architecture: amd64 + os: windows + version: 1809 + - + image: plugins/gar:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}windows-ltsc2022-amd64 + platform: + architecture: amd64 + os: windows + version: ltsc2022 diff --git a/internal/gcp/tokenutil.go b/internal/gcp/tokenutil.go new file mode 100644 index 0000000..baa07b4 --- /dev/null +++ b/internal/gcp/tokenutil.go @@ -0,0 +1,65 @@ +package gcp + +import ( + "context" + "fmt" + + "golang.org/x/oauth2" + "google.golang.org/api/iamcredentials/v1" + "google.golang.org/api/option" + "google.golang.org/api/sts/v1" +) + +type staticTokenSource struct { + token *oauth2.Token +} + +func (s *staticTokenSource) Token() (*oauth2.Token, error) { + return s.token, nil +} + +func GetFederalToken(idToken, projectNumber, poolId, providerId string) (string, error) { + ctx := context.Background() + stsService, err := sts.NewService(ctx, option.WithoutAuthentication()) + if err != nil { + return "", err + } + audience := fmt.Sprintf("//iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s/providers/%s", projectNumber, poolId, providerId) + tokenRequest := &sts.GoogleIdentityStsV1ExchangeTokenRequest{ + GrantType: "urn:ietf:params:oauth:grant-type:token-exchange", + SubjectToken: idToken, + Audience: audience, + Scope: "https://www.googleapis.com/auth/cloud-platform", + RequestedTokenType: "urn:ietf:params:oauth:token-type:access_token", + SubjectTokenType: "urn:ietf:params:oauth:token-type:id_token", + } + tokenResponse, err := stsService.V1.Token(tokenRequest).Do() + if err != nil { + return "", err + } + + return tokenResponse.AccessToken, nil +} + +func GetGoogleCloudAccessToken(federatedToken string, serviceAccountEmail string) (string, error) { + ctx := context.Background() + tokenSource := &staticTokenSource{ + token: &oauth2.Token{AccessToken: federatedToken}, + } + service, err := iamcredentials.NewService(ctx, option.WithTokenSource(tokenSource)) + if err != nil { + return "", err + } + + name := "projects/-/serviceAccounts/" + serviceAccountEmail + rb := &iamcredentials.GenerateAccessTokenRequest{ + Scope: []string{"https://www.googleapis.com/auth/cloud-platform"}, + } + + resp, err := service.Projects.ServiceAccounts.GenerateAccessToken(name, rb).Do() + if err != nil { + return "", err + } + + return resp.AccessToken, nil +}