Multistage container builds (#11468)

* WIP multi-stage container build.

* Move to a multistage build process in the Containerfile.

* Enable testing container

* Finalize the multistage build script and Containerfile. Ooof!

I got blocked for a bit with this bugger:

Step 27/32 : COPY --from=builder --chown=${APP_USER}:${APP_USER} ${APP_HOME} ${APP_HOME}
failed to copy files: failed to copy directory: Error processing tar file(exit status 1): Container ID 100999 cannot be mapped to a host ID

but https://github.com/phusion/passenger-docker/issues/235#issuecomment-636318827

was a good shove in the right direction!

* Fix bugs on container build script.

* Adjust compose files to use multistage builds.

* Add in Buildkite pipeline for building containers.

* Update CODEOWNERS to claim ownership over things that Systems Eng cares about
This commit is contained in:
Joe Doss 2020-11-19 10:03:35 -06:00 committed by GitHub
parent c837834d6a
commit 1d4685219d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 252 additions and 40 deletions

View file

@ -0,0 +1,7 @@
steps:
- command: ./scripts/build_containers.sh
plugins:
- docker-login#v2.0.1:
username: QUAY_LOGIN_USER
password-env: QUAY_LOGIN_PASSWORD
server: QUAY_LOGIN_SERVER

8
.github/CODEOWNERS vendored
View file

@ -17,8 +17,14 @@
/lib/sidekiq/ @forem/sre
/spec/rails_helper.rb @forem/sre
/spec/support/ @forem/sre
.buildkite/ @forem/systems @forem/sre
.travis.yml @forem/sre
Containerfile @forem/systems
docker-compose.yml @forem/systems
Dockerfile @forem/systems
Gemfile @forem/sre @forem/oss
Gemfile.lock @forem/sre @forem/oss
package.json @forem/oss @nickytonline
podman-compose.yml @forem/systems
scripts/ @forem/systems
yarn.lock @forem/oss @nickytonline
.travis.yml @forem/sre

View file

@ -1,49 +1,42 @@
FROM quay.io/forem/ruby:2.7.2
FROM quay.io/forem/ruby:2.7.2 as builder
USER root
RUN curl -sL https://dl.yarnpkg.com/rpm/yarn.repo -o /etc/yum.repos.d/yarn.repo && \
dnf install -y bash curl git ImageMagick iproute jemalloc less libcurl libcurl-devel \
libffi-devel libxml2-devel libxslt-devel nodejs pcre-devel \
postgresql postgresql-devel ruby-devel tzdata yarn \
&& dnf -y clean all \
&& rm -rf /var/cache/yum
dnf install --setopt install_weak_deps=false -y \
ImageMagick iproute jemalloc less libcurl libcurl-devel \
libffi-devel libxml2-devel libxslt-devel nodejs pcre-devel \
postgresql postgresql-devel tzdata yarn && \
dnf -y clean all && \
rm -rf /var/cache/yum
ENV APP_USER=forem
ENV APP_UID=1000
ENV APP_GID=1000
ENV APP_HOME=/opt/apps/forem
ENV LD_PRELOAD=/usr/lib64/libjemalloc.so.2
RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}"
RUN groupadd -g "${APP_GID}" "${APP_USER}" && \
adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}"
ENV BUNDLER_VERSION=2.1.4
ENV BUNDLER_VERSION=2.1.4 BUNDLE_SILENCE_ROOT_WARNING=1
RUN gem install bundler:"${BUNDLER_VERSION}"
ENV GEM_HOME=/opt/apps/bundle
ENV BUNDLE_SILENCE_ROOT_WARNING=1 BUNDLE_APP_CONFIG="${GEM_HOME}"
ENV PATH "${GEM_HOME}"/bin:$PATH
RUN mkdir -p "${GEM_HOME}" && chown "${APP_UID}":"${APP_GID}" "${GEM_HOME}"
ENV APP_USER=forem APP_UID=1000 APP_GID=1000 APP_HOME=/opt/apps/forem \
LD_PRELOAD=/usr/lib64/libjemalloc.so.2
RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" && \
groupadd -g "${APP_GID}" "${APP_USER}" && \
adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}"
ENV DOCKERIZE_VERSION=v0.6.1
RUN wget https://github.com/jwilder/dockerize/releases/download/"${DOCKERIZE_VERSION}"/dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \
&& tar -C /usr/local/bin -xzvf dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \
&& rm dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz
&& rm dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \
&& chown root:root /usr/local/bin/dockerize
WORKDIR "${APP_HOME}"
# Comment out running as the forem user due to this issue with podman-compose:
# https://github.com/containers/podman-compose/issues/166
# USER "${APP_USER}"
COPY ./.ruby-version "${APP_HOME}"/
COPY ./Gemfile ./Gemfile.lock "${APP_HOME}"/
COPY ./vendor/cache "${APP_HOME}"/vendor/cache
# Fixes https://github.com/sass/sassc-ruby/issues/146
RUN bundle config build.sassc --disable-march-tune-native
RUN bundle check || bundle install --jobs 20 --retry 5
RUN bundle config build.sassc --disable-march-tune-native && \
bundle config set deployment 'true' && \
bundle config set without 'development test' && \
bundle install --jobs 4 --retry 5 && \
find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
RUN mkdir -p "${APP_HOME}"/public/{assets,images,packs,podcasts,uploads}
@ -55,8 +48,91 @@ RUN echo $(date -u +'%Y-%m-%dT%H:%M:%SZ') >> "${APP_HOME}"/FOREM_BUILD_DATE && \
echo $(git rev-parse --short HEAD) >> "${APP_HOME}"/FOREM_BUILD_SHA && \
rm -rf "${APP_HOME}"/.git/
RUN rm -rf node_modules app/assets vendor/assets vendor/cache spec
## Production
FROM quay.io/forem/ruby:2.7.2 as production
USER root
RUN dnf install --setopt install_weak_deps=false -y bash curl ImageMagick \
iproute jemalloc less libcurl \
postgresql tzdata \
&& dnf -y clean all \
&& rm -rf /var/cache/yum
ENV BUNDLER_VERSION=2.1.4 BUNDLE_SILENCE_ROOT_WARNING=1
RUN gem install bundler:"${BUNDLER_VERSION}"
ENV APP_USER=forem APP_UID=1000 APP_GID=1000 APP_HOME=/opt/apps/forem \
LD_PRELOAD=/usr/lib64/libjemalloc.so.2
RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" && \
groupadd -g "${APP_GID}" "${APP_USER}" && \
adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}"
COPY --from=builder --chown="${APP_USER}":"${APP_USER}" ${APP_HOME} ${APP_HOME}
USER "${APP_USER}"
WORKDIR "${APP_HOME}"
VOLUME "${APP_HOME}"/public/
ENTRYPOINT ["./scripts/entrypoint.sh"]
CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
## Testing
FROM builder AS testing
USER root
RUN dnf install --setopt install_weak_deps=false -y \
chromium-headless chromedriver && \
yum clean all && \
rm -rf /var/cache/yum
COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets
COPY --chown="${APP_USER}":"${APP_USER}" ./vendor/cache "${APP_HOME}"/vendor/cache
COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}"
USER "${APP_USER}"
RUN bundle config build.sassc --disable-march-tune-native && \
bundle config set deployment 'false' && \
bundle config --delete without 'development test' && \
bundle install --jobs 4 --retry 5 && \
find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
RUN yarn install --frozen-lockfile && RAILS_ENV=test NODE_ENV=test bundle exec rails webpacker:compile
ENTRYPOINT ["./scripts/entrypoint-dev.sh"]
CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
## Development
FROM builder AS development
COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets
COPY --chown="${APP_USER}":"${APP_USER}" ./vendor/cache "${APP_HOME}"/vendor/cache
COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}"
USER "${APP_USER}"
RUN bundle config build.sassc --disable-march-tune-native && \
bundle config set deployment 'false' && \
bundle config --delete without 'development test' && \
bundle install --jobs 4 --retry 5 && \
find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
ENTRYPOINT ["./scripts/entrypoint-dev.sh"]
CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]

View file

@ -2,6 +2,7 @@ version: "3"
services:
rails:
build: .
target: development
image: forem-rails:latest
container_name: forem_rails
ports:
@ -24,7 +25,6 @@ services:
APP_DOMAIN: rails
volumes:
- .:/opt/apps/forem:z
- ./.gems:/opt/apps/bundle:z
entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "300s", "./scripts/entrypoint-dev.sh"]
command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"]
@ -38,7 +38,6 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:z
- ./.gems:/opt/apps/bundle:z
entrypoint: ["bash"]
command: ["./scripts/bundle.sh"]
@ -68,7 +67,6 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:z
- ./.gems:/opt/apps/bundle:z
entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-timeout", "300s"]
command: ["./bin/webpack-dev-server"]
@ -89,7 +87,6 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:z
- ./.gems:/opt/apps/bundle:z
entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "300s"]
command: ["bundle", "exec", "rake","db:seed"]
@ -108,7 +105,6 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:z
- ./.gems:/opt/apps/bundle:z
entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "300s"]
command: ["bundle", "exec", "sidekiq","-c","2"]

View file

@ -2,6 +2,7 @@ version: "3"
services:
rails:
build: .
target: development
image: forem-rails:latest
container_name: forem_rails
ports:
@ -24,7 +25,6 @@ services:
APP_DOMAIN: rails
volumes:
- .:/opt/apps/forem:delegated
- ./.gems:/opt/apps/bundle:delegated
entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "2700s", "./scripts/entrypoint-dev.sh"]
command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"]
@ -38,7 +38,6 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:delegated
- ./.gems:/opt/apps/bundle:delegated
entrypoint: ["bash"]
command: ["./scripts/bundle.sh"]
@ -68,7 +67,6 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:delegated
- ./.gems:/opt/apps/bundle:delegated
entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-timeout", "2700s"]
command: ["./bin/webpack-dev-server"]
@ -89,7 +87,7 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:delegated
- ./.gems:/opt/apps/bundle:delegated
entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"]
command: ["bundle", "exec", "rake","db:seed"]
@ -108,7 +106,7 @@ services:
ELASTICSEARCH_URL: http://elasticsearch:9200
volumes:
- .:/opt/apps/forem:delegated
- ./.gems:/opt/apps/bundle:delegated
entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"]
command: ["bundle", "exec", "sidekiq","-c","2"]

129
scripts/build_containers.sh Normal file
View file

@ -0,0 +1,129 @@
#!/bin/bash
set -euo pipefail
: ${CONTAINER_REPO:="quay.io/forem"}
: ${CONTAINER_APP:=forem}
function create_pr_containers {
PULL_REQUEST=$1
# Pull images if available for caching
echo "Pulling pull request #${PULL_REQUEST} containers from registry..."
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" ||:
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" ||:
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" ||:
# Build the builder image
echo "Building builder-${PULL_REQUEST} container..."
docker build --target builder \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" .
# Build the pull request image
echo "Building pr-${PULL_REQUEST} container..."
docker build --target production \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" .
# Build the testing image
echo "Building testing-$"${PULL_REQUEST}" container..."
docker build --target testing \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing .
# Push images to Quay
echo "Pushing pull request #${PULL_REQUEST} containers to registry..."
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}"
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}"
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}"
}
function create_production_containers {
# Pull images if available for caching
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder ||:
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":production ||:
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing ||:
docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":development ||:
# Build the builder image
docker build --target builder \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder .
# Build the production image
docker build --target production \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":$(date +%Y%m%d) \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_COMMIT:0:7} \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":latest .
# Build the testing image
docker build --target testing \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing .
# Build the development image
docker build --target development \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \
--cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":development \
--tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":development .
# Push images to Quay
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":production
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":development
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":$(date +%Y%m%d)
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_COMMIT:0:7}
docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":latest
}
if [ ! -v BUILDKITE_BRANCH ]; then
echo "Not running in Buildkite. Building Production Containers..."
BUILDKITE_COMMIT=$(git rev-parse --short HEAD)
create_production_containers
elif [ -z "$BUILDKITE_BRANCH" ]; then
echo "BUILDKITE_BRANCH is set to an empty string! Exiting..."
exit 1
elif [[ "${BUILDKITE_BRANCH}" = "master" || "${BUILDKITE_BRANCH}" = "main" ]]; then
echo "Building Production Containers..."
create_production_containers
else
if [ ! -v BUILDKITE_PULL_REQUEST ]; then
echo "BUILDKITE_PULL_REQUEST is unset! Exiting..."
exit 1
elif [ -z "$BUILDKITE_PULL_REQUEST" ]; then
echo "BUILDKITE_PULL_REQUEST is set to an empty string! Exiting..."
exit 1
else
echo "Building containers for pull request #${BUILDKITE_PULL_REQUEST}..."
create_pr_containers "${BUILDKITE_PULL_REQUEST}"
fi
fi