diff --git a/.buildkite/pipeline.containers.yml b/.buildkite/pipeline.containers.yml new file mode 100644 index 000000000..c33fdae63 --- /dev/null +++ b/.buildkite/pipeline.containers.yml @@ -0,0 +1,7 @@ +steps: + - command: ./scripts/build_containers.sh + plugins: + - docker-login#v2.0.1: + username: QUAY_LOGIN_USER + password-env: QUAY_LOGIN_PASSWORD + server: QUAY_LOGIN_SERVER diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 93931e84d..6286b9bb2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,8 +17,14 @@ /lib/sidekiq/ @forem/sre /spec/rails_helper.rb @forem/sre /spec/support/ @forem/sre +.buildkite/ @forem/systems @forem/sre +.travis.yml @forem/sre +Containerfile @forem/systems +docker-compose.yml @forem/systems +Dockerfile @forem/systems Gemfile @forem/sre @forem/oss Gemfile.lock @forem/sre @forem/oss package.json @forem/oss @nickytonline +podman-compose.yml @forem/systems +scripts/ @forem/systems yarn.lock @forem/oss @nickytonline -.travis.yml @forem/sre diff --git a/Containerfile b/Containerfile index d4215b5fe..22c2c53bc 100644 --- a/Containerfile +++ b/Containerfile @@ -1,49 +1,42 @@ -FROM quay.io/forem/ruby:2.7.2 +FROM quay.io/forem/ruby:2.7.2 as builder USER root RUN curl -sL https://dl.yarnpkg.com/rpm/yarn.repo -o /etc/yum.repos.d/yarn.repo && \ - dnf install -y bash curl git ImageMagick iproute jemalloc less libcurl libcurl-devel \ - libffi-devel libxml2-devel libxslt-devel nodejs pcre-devel \ - postgresql postgresql-devel ruby-devel tzdata yarn \ - && dnf -y clean all \ - && rm -rf /var/cache/yum + dnf install --setopt install_weak_deps=false -y \ + ImageMagick iproute jemalloc less libcurl libcurl-devel \ + libffi-devel libxml2-devel libxslt-devel nodejs pcre-devel \ + postgresql postgresql-devel tzdata yarn && \ + dnf -y clean all && \ + rm -rf /var/cache/yum -ENV APP_USER=forem -ENV APP_UID=1000 -ENV APP_GID=1000 -ENV APP_HOME=/opt/apps/forem -ENV LD_PRELOAD=/usr/lib64/libjemalloc.so.2 -RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" -RUN groupadd -g "${APP_GID}" "${APP_USER}" && \ - adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}" - -ENV BUNDLER_VERSION=2.1.4 +ENV BUNDLER_VERSION=2.1.4 BUNDLE_SILENCE_ROOT_WARNING=1 RUN gem install bundler:"${BUNDLER_VERSION}" -ENV GEM_HOME=/opt/apps/bundle -ENV BUNDLE_SILENCE_ROOT_WARNING=1 BUNDLE_APP_CONFIG="${GEM_HOME}" -ENV PATH "${GEM_HOME}"/bin:$PATH -RUN mkdir -p "${GEM_HOME}" && chown "${APP_UID}":"${APP_GID}" "${GEM_HOME}" + +ENV APP_USER=forem APP_UID=1000 APP_GID=1000 APP_HOME=/opt/apps/forem \ + LD_PRELOAD=/usr/lib64/libjemalloc.so.2 +RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" && \ + groupadd -g "${APP_GID}" "${APP_USER}" && \ + adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}" ENV DOCKERIZE_VERSION=v0.6.1 RUN wget https://github.com/jwilder/dockerize/releases/download/"${DOCKERIZE_VERSION}"/dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \ - && rm dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz + && rm dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \ + && chown root:root /usr/local/bin/dockerize WORKDIR "${APP_HOME}" -# Comment out running as the forem user due to this issue with podman-compose: -# https://github.com/containers/podman-compose/issues/166 -# USER "${APP_USER}" - COPY ./.ruby-version "${APP_HOME}"/ COPY ./Gemfile ./Gemfile.lock "${APP_HOME}"/ COPY ./vendor/cache "${APP_HOME}"/vendor/cache -# Fixes https://github.com/sass/sassc-ruby/issues/146 -RUN bundle config build.sassc --disable-march-tune-native - -RUN bundle check || bundle install --jobs 20 --retry 5 +RUN bundle config build.sassc --disable-march-tune-native && \ + bundle config set deployment 'true' && \ + bundle config set without 'development test' && \ + bundle install --jobs 4 --retry 5 && \ + find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ + find "${APP_HOME}"/vendor/bundle -name "*.o" -delete RUN mkdir -p "${APP_HOME}"/public/{assets,images,packs,podcasts,uploads} @@ -55,8 +48,91 @@ RUN echo $(date -u +'%Y-%m-%dT%H:%M:%SZ') >> "${APP_HOME}"/FOREM_BUILD_DATE && \ echo $(git rev-parse --short HEAD) >> "${APP_HOME}"/FOREM_BUILD_SHA && \ rm -rf "${APP_HOME}"/.git/ +RUN rm -rf node_modules app/assets vendor/assets vendor/cache spec + +## Production +FROM quay.io/forem/ruby:2.7.2 as production + +USER root + +RUN dnf install --setopt install_weak_deps=false -y bash curl ImageMagick \ + iproute jemalloc less libcurl \ + postgresql tzdata \ + && dnf -y clean all \ + && rm -rf /var/cache/yum + +ENV BUNDLER_VERSION=2.1.4 BUNDLE_SILENCE_ROOT_WARNING=1 +RUN gem install bundler:"${BUNDLER_VERSION}" + +ENV APP_USER=forem APP_UID=1000 APP_GID=1000 APP_HOME=/opt/apps/forem \ + LD_PRELOAD=/usr/lib64/libjemalloc.so.2 +RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" && \ + groupadd -g "${APP_GID}" "${APP_USER}" && \ + adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}" + +COPY --from=builder --chown="${APP_USER}":"${APP_USER}" ${APP_HOME} ${APP_HOME} + +USER "${APP_USER}" +WORKDIR "${APP_HOME}" + VOLUME "${APP_HOME}"/public/ ENTRYPOINT ["./scripts/entrypoint.sh"] CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] + +## Testing +FROM builder AS testing + +USER root + +RUN dnf install --setopt install_weak_deps=false -y \ + chromium-headless chromedriver && \ + yum clean all && \ + rm -rf /var/cache/yum + +COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets +COPY --chown="${APP_USER}":"${APP_USER}" ./vendor/cache "${APP_HOME}"/vendor/cache +COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec +COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize + +RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}" + +USER "${APP_USER}" + +RUN bundle config build.sassc --disable-march-tune-native && \ + bundle config set deployment 'false' && \ + bundle config --delete without 'development test' && \ + bundle install --jobs 4 --retry 5 && \ + find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ + find "${APP_HOME}"/vendor/bundle -name "*.o" -delete + +RUN yarn install --frozen-lockfile && RAILS_ENV=test NODE_ENV=test bundle exec rails webpacker:compile + +ENTRYPOINT ["./scripts/entrypoint-dev.sh"] + +CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] + +## Development +FROM builder AS development + +COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets +COPY --chown="${APP_USER}":"${APP_USER}" ./vendor/cache "${APP_HOME}"/vendor/cache +COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec +COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize + +RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}" + +USER "${APP_USER}" + +RUN bundle config build.sassc --disable-march-tune-native && \ + bundle config set deployment 'false' && \ + bundle config --delete without 'development test' && \ + bundle install --jobs 4 --retry 5 && \ + find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ + find "${APP_HOME}"/vendor/bundle -name "*.o" -delete + +ENTRYPOINT ["./scripts/entrypoint-dev.sh"] + +CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] + diff --git a/container-compose.yml b/container-compose.yml index 6331b7557..9bd87e2d5 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -2,6 +2,7 @@ version: "3" services: rails: build: . + target: development image: forem-rails:latest container_name: forem_rails ports: @@ -24,7 +25,6 @@ services: APP_DOMAIN: rails volumes: - .:/opt/apps/forem:z - - ./.gems:/opt/apps/bundle:z entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "300s", "./scripts/entrypoint-dev.sh"] command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"] @@ -38,7 +38,6 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:z - - ./.gems:/opt/apps/bundle:z entrypoint: ["bash"] command: ["./scripts/bundle.sh"] @@ -68,7 +67,6 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:z - - ./.gems:/opt/apps/bundle:z entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-timeout", "300s"] command: ["./bin/webpack-dev-server"] @@ -89,7 +87,6 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:z - - ./.gems:/opt/apps/bundle:z entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "300s"] command: ["bundle", "exec", "rake","db:seed"] @@ -108,7 +105,6 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:z - - ./.gems:/opt/apps/bundle:z entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "300s"] command: ["bundle", "exec", "sidekiq","-c","2"] diff --git a/docker-compose.yml b/docker-compose.yml index 80aa4bca3..fd0c1c583 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3" services: rails: build: . + target: development image: forem-rails:latest container_name: forem_rails ports: @@ -24,7 +25,6 @@ services: APP_DOMAIN: rails volumes: - .:/opt/apps/forem:delegated - - ./.gems:/opt/apps/bundle:delegated entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "2700s", "./scripts/entrypoint-dev.sh"] command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"] @@ -38,7 +38,6 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - - ./.gems:/opt/apps/bundle:delegated entrypoint: ["bash"] command: ["./scripts/bundle.sh"] @@ -68,7 +67,6 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - - ./.gems:/opt/apps/bundle:delegated entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-timeout", "2700s"] command: ["./bin/webpack-dev-server"] @@ -89,7 +87,7 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - - ./.gems:/opt/apps/bundle:delegated + entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"] command: ["bundle", "exec", "rake","db:seed"] @@ -108,7 +106,7 @@ services: ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - - ./.gems:/opt/apps/bundle:delegated + entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"] command: ["bundle", "exec", "sidekiq","-c","2"] diff --git a/scripts/build_containers.sh b/scripts/build_containers.sh new file mode 100644 index 000000000..3d6d8473e --- /dev/null +++ b/scripts/build_containers.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +set -euo pipefail + +: ${CONTAINER_REPO:="quay.io/forem"} +: ${CONTAINER_APP:=forem} + +function create_pr_containers { + + PULL_REQUEST=$1 + + # Pull images if available for caching + echo "Pulling pull request #${PULL_REQUEST} containers from registry..." + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" ||: + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" ||: + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" ||: + + # Build the builder image + echo "Building builder-${PULL_REQUEST} container..." + docker build --target builder \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" . + + # Build the pull request image + echo "Building pr-${PULL_REQUEST} container..." + docker build --target production \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" . + + # Build the testing image + echo "Building testing-$"${PULL_REQUEST}" container..." + docker build --target testing \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing . + + # Push images to Quay + echo "Pushing pull request #${PULL_REQUEST} containers to registry..." + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" + +} + +function create_production_containers { + + # Pull images if available for caching + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder ||: + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":production ||: + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing ||: + docker pull "${CONTAINER_REPO}"/"${CONTAINER_APP}":development ||: + + # Build the builder image + docker build --target builder \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder . + + # Build the production image + docker build --target production \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":$(date +%Y%m%d) \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_COMMIT:0:7} \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":production \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":latest . + + # Build the testing image + docker build --target testing \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing . + + # Build the development image + docker build --target development \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \ + --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":development \ + --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":development . + + # Push images to Quay + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":production + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":development + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":$(date +%Y%m%d) + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_COMMIT:0:7} + docker push "${CONTAINER_REPO}"/"${CONTAINER_APP}":latest + +} + +if [ ! -v BUILDKITE_BRANCH ]; then + + echo "Not running in Buildkite. Building Production Containers..." + BUILDKITE_COMMIT=$(git rev-parse --short HEAD) + create_production_containers + +elif [ -z "$BUILDKITE_BRANCH" ]; then + + echo "BUILDKITE_BRANCH is set to an empty string! Exiting..." + exit 1 + +elif [[ "${BUILDKITE_BRANCH}" = "master" || "${BUILDKITE_BRANCH}" = "main" ]]; then + + echo "Building Production Containers..." + create_production_containers + +else + + if [ ! -v BUILDKITE_PULL_REQUEST ]; then + + echo "BUILDKITE_PULL_REQUEST is unset! Exiting..." + exit 1 + + elif [ -z "$BUILDKITE_PULL_REQUEST" ]; then + + echo "BUILDKITE_PULL_REQUEST is set to an empty string! Exiting..." + exit 1 + + else + + echo "Building containers for pull request #${BUILDKITE_PULL_REQUEST}..." + create_pr_containers "${BUILDKITE_PULL_REQUEST}" + fi + +fi