Allow remote caching of docker layers in kaniko build (#16)

Co-authored-by: beniamin.calota <beniamin.calota@emag.ro>

.drone.yml

@@ -14,30 +14,102 @@ steps:
   settings:
     repo: plugins/kaniko
     auto_tag: true
-    dockerfile: docker/gcr/Dockerfile.linux.amd64
+    auto_tag_suffix: linux-amd64
+    daemon_off: false
+    dockerfile: docker/docker/Dockerfile.linux.amd64
     username:
       from_secret: docker_username
     password:
       from_secret: docker_password
+  when:
+    event:
+      exclude:
+      - pull_request
 
 - name: gcr
   image: plugins/docker
   settings:
     repo: plugins/kaniko-gcr
     auto_tag: true
+    auto_tag_suffix: linux-amd64
+    daemon_off: false
     dockerfile: docker/gcr/Dockerfile.linux.amd64
     username:
       from_secret: docker_username
     password:
       from_secret: docker_password
+  when:
+    event:
+      exclude:
+      - pull_request
 
 - name: ecr
   image: plugins/docker
   settings:
     repo: plugins/kaniko-ecr
     auto_tag: true
+    auto_tag_suffix: linux-amd64
+    daemon_off: false
     dockerfile: docker/ecr/Dockerfile.linux.amd64
     username:
       from_secret: docker_username
     password:
       from_secret: docker_password
+  when:
+    event:
+      exclude:
+      - pull_request
+
+---
+kind: pipeline
+type: docker
+name: notifications-docker
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: manifest-docker
+  pull: always
+  image: plugins/manifest
+  settings:
+    auto_tag: true
+    ignore_missing: true
+    password:
+      from_secret: docker_password
+    spec: docker/docker/manifest.tmpl
+    username:
+      from_secret: docker_username
+
+- name: manifest-gcr
+  pull: always
+  image: plugins/manifest
+  settings:
+    auto_tag: true
+    ignore_missing: true
+    password:
+      from_secret: docker_password
+    spec: docker/gcr/manifest.tmpl
+    username:
+      from_secret: docker_username
+
+- name: manifest-ecr
+  pull: always
+  image: plugins/manifest
+  settings:
+    auto_tag: true
+    ignore_missing: true
+    password:
+      from_secret: docker_password
+    spec: docker/ecr/manifest.tmpl
+    username:
+      from_secret: docker_username
+
+trigger:
+  ref:
+  - refs/heads/main
+  - "refs/tags/**"
+
+depends_on:
+- default

README.md

@@ -25,7 +25,7 @@ Build the Docker images with the following commands:
 docker build \
   --label org.label-schema.build-date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
   --label org.label-schema.vcs-ref=$(git rev-parse --short HEAD) \
-  --file docker/docker/Dockerfile.linux.amd64 --tag plugins/kaniko-docker .
+  --file docker/docker/Dockerfile.linux.amd64 --tag plugins/kaniko .
 
 docker build \
   --label org.label-schema.build-date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
@@ -49,5 +49,5 @@ docker run --rm \
     -e PLUGIN_PASSWORD=bar \
     -v $(pwd):/drone \
     -w /drone \
-    plugins/kaniko-docker
+    plugins/kaniko:linux-amd64
 ```

cmd/kaniko-docker/main.go

@@ -18,6 +18,9 @@ const (
 	// Docker file path
 	dockerPath       string = "/kaniko/.docker"
 	dockerConfigPath string = "/kaniko/.docker/config.json"
+
+	v1Registry string = "https://index.docker.io/v1/" // Default registry
+	v2Registry string = "https://index.docker.io/v2/" // v2 registry is not supported
 )
 
 var (
@@ -78,7 +81,7 @@ func main() {
 		cli.StringFlag{
 			Name:   "registry",
 			Usage:  "docker registry",
-			Value:  "https://index.docker.io/v1/",
+			Value:  v1Registry,
 			EnvVar: "PLUGIN_REGISTRY",
 		},
 		cli.StringFlag{
@@ -91,6 +94,31 @@ func main() {
 			Usage:  "docker password",
 			EnvVar: "PLUGIN_PASSWORD",
 		},
+		cli.BoolFlag{
+			Name:   "skip-tls-verify",
+			Usage:  "Skip registry tls verify",
+			EnvVar: "PLUGIN_SKIP_TLS_VERIFY",
+		},
+		cli.StringFlag{
+			Name:   "snapshot-mode",
+			Usage:  "Specify one of full, redo or time as snapshot mode",
+			EnvVar: "PLUGIN_SNAPSHOT_MODE",
+		},
+		cli.BoolFlag{
+			Name:   "enable-cache",
+			Usage:  "Set this flag to opt into caching with kaniko",
+			EnvVar: "PLUGIN_ENABLE_CACHE",
+		},
+		cli.StringFlag{
+			Name:   "cache-repo",
+			Usage:  "Remote repository that will be used to store cached layers. enable-cache needs to be set to use this flag",
+			EnvVar: "PLUGIN_CACHE_REPO",
+		},
+		cli.IntFlag{
+			Name:   "cache-ttl",
+			Usage:  "Cache timeout in hours. Defaults to two weeks.",
+			EnvVar: "PLUGIN_CACHE_TTL",
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -106,13 +134,18 @@ func run(c *cli.Context) error {
 
 	plugin := kaniko.Plugin{
 		Build: kaniko.Build{
-			Dockerfile: c.String("dockerfile"),
-			Context:    c.String("context"),
-			Tags:       c.StringSlice("tags"),
-			Args:       c.StringSlice("args"),
-			Target:     c.String("target"),
-			Repo:       c.String("repo"),
-			Labels:     c.StringSlice("custom-labels"),
+			Dockerfile:    c.String("dockerfile"),
+			Context:       c.String("context"),
+			Tags:          c.StringSlice("tags"),
+			Args:          c.StringSlice("args"),
+			Target:        c.String("target"),
+			Repo:          c.String("repo"),
+			Labels:        c.StringSlice("custom-labels"),
+			SkipTlsVerify: c.Bool("skip-tls-verify"),
+			SnapshotMode:  c.String("snapshot-mode"),
+			EnableCache:   c.Bool("enable-cache"),
+			CacheRepo:     c.String("cache-repo"),
+			CacheTTL:      c.Int("cache-ttl"),
 		},
 	}
 	return plugin.Exec()
@@ -130,6 +163,12 @@ func createDockerCfgFile(username, password, registry string) error {
 		return fmt.Errorf("Registry must be specified")
 	}
 
+	if registry == v2Registry {
+		fmt.Println("Docker v2 registry is not supported in kaniko. Refer issue: https://github.com/GoogleContainerTools/kaniko/issues/1209")
+		fmt.Printf("Using v1 registry instead: %s\n", v1Registry)
+		registry = v1Registry
+	}
+
 	err := os.MkdirAll(dockerPath, 0600)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to create %s directory", dockerPath))

cmd/kaniko-ecr/main.go

@@ -89,6 +89,26 @@ func main() {
 			Usage:  "ECR secret key",
 			EnvVar: "PLUGIN_SECRET_KEY",
 		},
+		cli.StringFlag{
+			Name:   "snapshot-mode",
+			Usage:  "Specify one of full, redo or time as snapshot mode",
+			EnvVar: "PLUGIN_SNAPSHOT_MODE",
+		},
+		cli.BoolFlag{
+			Name:   "enable-cache",
+			Usage:  "Set this flag to opt into caching with kaniko",
+			EnvVar: "PLUGIN_ENABLE_CACHE",
+		},
+		cli.StringFlag{
+			Name:   "cache-repo",
+			Usage:  "Remote repository that will be used to store cached layers. enable-cache needs to be set to use this flag",
+			EnvVar: "PLUGIN_CACHE_REPO",
+		},
+		cli.IntFlag{
+			Name:   "cache-ttl",
+			Usage:  "Cache timeout in hours. Defaults to two weeks.",
+			EnvVar: "PLUGIN_CACHE_TTL",
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -104,41 +124,42 @@ func run(c *cli.Context) error {
 
 	plugin := kaniko.Plugin{
 		Build: kaniko.Build{
-			Dockerfile: c.String("dockerfile"),
-			Context:    c.String("context"),
-			Tags:       c.StringSlice("tags"),
-			Args:       c.StringSlice("args"),
-			Target:     c.String("target"),
-			Repo:       c.String("repo"),
-			Labels:     c.StringSlice("custom-labels"),
+			Dockerfile:   c.String("dockerfile"),
+			Context:      c.String("context"),
+			Tags:         c.StringSlice("tags"),
+			Args:         c.StringSlice("args"),
+			Target:       c.String("target"),
+			Repo:         fmt.Sprintf("%s/%s", c.String("registry"), c.String("repo")),
+			Labels:       c.StringSlice("custom-labels"),
+			SnapshotMode: c.String("snapshot-mode"),
+			EnableCache:  c.Bool("enable-cache"),
+			CacheRepo:    c.String("cache-repo"),
+			CacheTTL:     c.Int("cache-ttl"),
 		},
 	}
 	return plugin.Exec()
 }
 
 func setupECRAuth(accessKey, secretKey, registry string) error {
-	if accessKey == "" {
-		return fmt.Errorf("Access key must be specified")
-	}
-	if secretKey == "" {
-		return fmt.Errorf("Secret key must be specified")
-	}
 	if registry == "" {
 		return fmt.Errorf("Registry must be specified")
 	}
 
-	err := os.Setenv(accessKeyEnv, accessKey)
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("failed to set %s environment variable", accessKeyEnv))
-	}
+	// If IAM role is used, access key & secret key are not required
+	if accessKey != "" && secretKey != "" {
+		err := os.Setenv(accessKeyEnv, accessKey)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to set %s environment variable", accessKeyEnv))
+		}
 
-	err = os.Setenv(secretKeyEnv, secretKey)
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("failed to set %s environment variable", secretKeyEnv))
+		err = os.Setenv(secretKeyEnv, secretKey)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to set %s environment variable", secretKeyEnv))
+		}
 	}
 
 	jsonBytes := []byte(fmt.Sprintf(`{"credStore": "ecr-login", "credHelpers": {"%s": "ecr-login"}}`, registry))
-	err = ioutil.WriteFile(dockerConfigPath, jsonBytes, 0644)
+	err := ioutil.WriteFile(dockerConfigPath, jsonBytes, 0644)
 	if err != nil {
 		return errors.Wrap(err, "failed to create docker config file")
 	}

cmd/kaniko-gcr/main.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	// GCR JSON key file path
-	gcrKeyPath     string = "/kaniko/gcr.json"
+	gcrKeyPath     string = "/kaniko/config.json"
 	gcrEnvVariable string = "GOOGLE_APPLICATION_CREDENTIALS"
 )
 
@@ -85,6 +85,26 @@ func main() {
 			Usage:  "docker username",
 			EnvVar: "PLUGIN_JSON_KEY",
 		},
+		cli.StringFlag{
+			Name:   "snapshot-mode",
+			Usage:  "Specify one of full, redo or time as snapshot mode",
+			EnvVar: "PLUGIN_SNAPSHOT_MODE",
+		},
+		cli.BoolFlag{
+			Name:   "enable-cache",
+			Usage:  "Set this flag to opt into caching with kaniko",
+			EnvVar: "PLUGIN_ENABLE_CACHE",
+		},
+		cli.StringFlag{
+			Name:   "cache-repo",
+			Usage:  "Remote repository that will be used to store cached layers. enable-cache needs to be set to use this flag",
+			EnvVar: "PLUGIN_CACHE_REPO",
+		},
+		cli.IntFlag{
+			Name:   "cache-ttl",
+			Usage:  "Cache timeout in hours. Defaults to two weeks.",
+			EnvVar: "PLUGIN_CACHE_TTL",
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -104,13 +124,17 @@ func run(c *cli.Context) error {
 
 	plugin := kaniko.Plugin{
 		Build: kaniko.Build{
-			Dockerfile: c.String("dockerfile"),
-			Context:    c.String("context"),
-			Tags:       c.StringSlice("tags"),
-			Args:       c.StringSlice("args"),
-			Target:     c.String("target"),
-			Repo:       fmt.Sprintf("%s/%s", c.String("registry"), c.String("repo")),
-			Labels:     c.StringSlice("custom-labels"),
+			Dockerfile:   c.String("dockerfile"),
+			Context:      c.String("context"),
+			Tags:         c.StringSlice("tags"),
+			Args:         c.StringSlice("args"),
+			Target:       c.String("target"),
+			Repo:         fmt.Sprintf("%s/%s", c.String("registry"), c.String("repo")),
+			Labels:       c.StringSlice("custom-labels"),
+			SnapshotMode: c.String("snapshot-mode"),
+			EnableCache:  c.Bool("enable-cache"),
+			CacheRepo:    c.String("cache-repo"),
+			CacheTTL:     c.Int("cache-ttl"),
 		},
 	}
 	return plugin.Exec()

docker/docker/Dockerfile.linux.amd64

@@ -1,7 +1,4 @@
-FROM gcr.io/kaniko-project/executor:amd64-v1.3.0
-
-ENV HOME /root
-ENV USER root
+FROM gcr.io/kaniko-project/executor:v1.3.0
 
 ADD release/linux/amd64/kaniko-docker /kaniko/
 ENTRYPOINT ["/kaniko/kaniko-docker"]

docker/ecr/Dockerfile.linux.amd64

@@ -1,7 +1,4 @@
-FROM gcr.io/kaniko-project/executor:amd64-v1.3.0
-
-ENV HOME /root
-ENV USER root
+FROM gcr.io/kaniko-project/executor:v1.3.0
 
 ADD release/linux/amd64/kaniko-ecr /kaniko/
 ENTRYPOINT ["/kaniko/kaniko-ecr"]

docker/gcr/Dockerfile.linux.amd64

@@ -1,7 +1,4 @@
-FROM gcr.io/kaniko-project/executor:amd64-v1.3.0
-
-ENV HOME /root
-ENV USER root
+FROM gcr.io/kaniko-project/executor:v1.3.0
 
 ADD release/linux/amd64/kaniko-gcr /kaniko/
 ENTRYPOINT ["/kaniko/kaniko-gcr"]

kaniko.go

@@ -10,13 +10,18 @@ import (
 type (
 	// Build defines Docker build parameters.
 	Build struct {
-		Dockerfile string   // Docker build Dockerfile
-		Context    string   // Docker build context
-		Tags       []string // Docker build tags
-		Args       []string // Docker build args
-		Target     string   // Docker build target
-		Repo       string   // Docker build repository
-		Labels     []string // Label map
+		Dockerfile    string   // Docker build Dockerfile
+		Context       string   // Docker build context
+		Tags          []string // Docker build tags
+		Args          []string // Docker build args
+		Target        string   // Docker build target
+		Repo          string   // Docker build repository
+		Labels        []string // Label map
+		SkipTlsVerify bool     // Docker skip tls certificate verify for registry
+		SnapshotMode  string   // Kaniko snapshot mode
+		EnableCache   bool     // Whether to enable kaniko cache
+		CacheRepo     string   // Remote repository that will be used to store cached layers
+		CacheTTL      int      // Cache timeout in hours
 	}
 
 	// Plugin defines the Docker plugin parameters.
@@ -31,6 +36,10 @@ func (p Plugin) Exec() error {
 		return fmt.Errorf("repository name to publish image must be specified")
 	}
 
+	if _, err := os.Stat(p.Build.Dockerfile); os.IsNotExist(err) {
+		return fmt.Errorf("dockerfile does not exist at path: %s", p.Build.Dockerfile)
+	}
+
 	cmdArgs := []string{
 		fmt.Sprintf("--dockerfile=%s", p.Build.Dockerfile),
 		fmt.Sprintf("--context=dir://%s", p.Build.Context),
@@ -46,13 +55,33 @@ func (p Plugin) Exec() error {
 	}
 	// Set the labels
 	for _, label := range p.Build.Labels {
-		cmdArgs = append(cmdArgs, fmt.Sprintf("--label %s", label))
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--label=%s", label))
 	}
 
 	if p.Build.Target != "" {
 		cmdArgs = append(cmdArgs, fmt.Sprintf("--target=%s", p.Build.Target))
 	}
 
+	if p.Build.SkipTlsVerify {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--skip-tls-verify=true"))
+	}
+
+	if p.Build.SnapshotMode != "" {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--snapshotMode=%s", p.Build.SnapshotMode))
+	}
+
+	if p.Build.EnableCache == true {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--cache=true"))
+	}
+
+	if p.Build.CacheRepo != "" {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--cache-repo=%s", p.Build.CacheRepo))
+	}
+
+	if p.Build.CacheTTL != 0 {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--cache-ttl=%d", p.Build.CacheTTL))
+	}
+
 	cmd := exec.Command("/kaniko/executor", cmdArgs...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr

scripts/build.sh

@@ -10,14 +10,14 @@ set -e
 set -x
 
 # linux
-GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/kaniko-gcr
-GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/kaniko-ecr
-GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/kaniko-docker
+GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/kaniko-gcr    ./cmd/kaniko-gcr
+GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/kaniko-ecr    ./cmd/kaniko-ecr
+GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/kaniko-docker ./cmd/kaniko-docker
 
-GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/kaniko-gcr
-GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/kaniko-ecr
-GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/kaniko-docker
+GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/kaniko-gcr    ./cmd/kaniko-gcr
+GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/kaniko-ecr    ./cmd/kaniko-ecr
+GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/kaniko-docker ./cmd/kaniko-docker
 
-GOOS=linux GOARCH=arm   go build -o release/linux/arm/kaniko-gcr
-GOOS=linux GOARCH=arm   go build -o release/linux/arm/kaniko-ecr
-GOOS=linux GOARCH=arm   go build -o release/linux/arm/kaniko-docker
+GOOS=linux GOARCH=arm   go build -o release/linux/arm/kaniko-gcr      ./cmd/kaniko-gcr
+GOOS=linux GOARCH=arm   go build -o release/linux/arm/kaniko-ecr      ./cmd/kaniko-ecr
+GOOS=linux GOARCH=arm   go build -o release/linux/arm/kaniko-docker   ./cmd/kaniko-docker

scripts/docker.sh

@@ -14,9 +14,9 @@ set -e
 set -x
 
 # build the binary
-go build -o release/linux/amd64/kaniko-gcr
-go build -o release/linux/amd64/kaniko-ecr
-go build -o release/linux/amd64/kaniko-docker
+go build -o release/linux/amd64/kaniko-gcr    ./cmd/kaniko-gcr
+go build -o release/linux/amd64/kaniko-ecr    ./cmd/kaniko-ecr
+go build -o release/linux/amd64/kaniko-docker ./cmd/kaniko-docker
 
 # build the docker image
 docker build -f docker/gcr/Dockerfile.linux.amd64    -t plugins/kaniko-gcr .