Jdoss/more buildkite container fixes (#11503)

* Fix bug on creating the testing container for PRs and create a function to
run docker image prune -f that will get run when the script exits cleanly or
uncleanly. This prevent junk containers from building up over time.

* Add in some labels and keys. Add in building containers on master branch.

* Let's also build on main branch for when we eventually move away from master.

* Let's use a previous builder image as a cache as it might speed things up.

.buildkite/pipeline.containers.yml

@@ -1,5 +1,17 @@
 steps:
   - command: ./scripts/build_containers.sh
+    label: ":hammer: Build containers!"
+    key: build-containers
+    plugins:
+      - docker-login#v2.0.1:
+          username: forem+buildkite
+          password-env: QUAY_LOGIN_PASSWORD
+          server: quay.io
+
+  - command: ./scripts/build_containers.sh
+    label: ":rocket: Build production containers!"
+    branches: "master main"
+    key: build-production-containers
     plugins:
       - docker-login#v2.0.1:
           username: forem+buildkite

scripts/build_containers.sh

@@ -11,6 +11,7 @@ function create_pr_containers {
 
   # Pull images if available for caching
   echo "Pulling pull request #${PULL_REQUEST} containers from registry..."
+  docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder ||:
   docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" ||:
   docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" ||:
   docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" ||:
@@ -18,6 +19,7 @@ function create_pr_containers {
   # Build the builder image
   echo "Building builder-${PULL_REQUEST} container..."
   docker build --target builder \
+               --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" .
 
@@ -34,7 +36,7 @@ function create_pr_containers {
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" \
-               --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing .
+               --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" .
 
   # Push images to Quay
   echo "Pushing pull request #${PULL_REQUEST} containers to registry..."
@@ -92,6 +94,15 @@ function create_production_containers {
 
 }
 
+
+function prune_containers {
+
+  docker image prune -f
+
+}
+
+trap prune_containers ERR INT EXIT
+
 if [ ! -v BUILDKITE_BRANCH ]; then
 
     echo "Not running in Buildkite. Building Production Containers..."
@@ -124,6 +135,7 @@ else
 
         echo "Building containers for pull request #${BUILDKITE_PULL_REQUEST}..."
         create_pr_containers "${BUILDKITE_PULL_REQUEST}"
+
   fi
 
 fi