From 44569605d2690d299c481e15e4a334c08cff6cf9 Mon Sep 17 00:00:00 2001 From: Joe Doss Date: Mon, 23 Nov 2020 15:15:21 -0600 Subject: [PATCH] Fix container composes w/r/t multistage container builds (#11531) * Use premade development image. * Use prebuilt container for podman-compose too! * Fix bundle bundle_finished path. * Fix docker-compose to work with the Multistage Containerfile Add root user for Forem containers, fix bundle_finished path and make webpacker wait for the bundle to finish. Switch Redis to redis:6.0.9-alpine * Remove build steps and fix bundle path. * Merge dev entrypoint into the main one. * Use merged entrypoint.sh, remove old entrypoint directives, and fix a dockerize bug. * Make sure unset BUNDLE_WITHOUT and remove without in bundle config. * Unset BUNDLE_WITHOUT="development:test" as a global ENV. Also ensure that we remove the without directives for development and testing. This will make sure that the right gems are installed for each environment. Removed some unneeded COPYs and remove a commented out RUN. * Fix ENTRYPOINT on development and testing containers. * Fix entrypoint and command on rails container. * Ignore bundle_finished file. * Fix REDIS_ env vars and adjust podman-compose file. * Change bundle_finished path and update Yarn docker-compose command. --- .gitignore | 3 +++ Containerfile | 13 +++++------ bin/container-setup | 17 ++------------- container-compose.yml | 37 +++++++++++++++++-------------- docker-compose.yml | 46 ++++++++++++++++++++++++--------------- scripts/bundle.sh | 10 +++++---- scripts/entrypoint-dev.sh | 16 -------------- scripts/entrypoint.sh | 14 ++++++++++-- 8 files changed, 77 insertions(+), 79 deletions(-) delete mode 100755 scripts/entrypoint-dev.sh diff --git a/.gitignore b/.gitignore index 8de99e873..a997124bf 100644 --- a/.gitignore +++ b/.gitignore @@ -82,5 +82,8 @@ db_data es_data gem_cache +# Compose generated files +bundle_finished + # Auto-generated Storybook stories app/javascript/generated_stories/ diff --git a/Containerfile b/Containerfile index 86b679e8b..d0b56911c 100644 --- a/Containerfile +++ b/Containerfile @@ -31,9 +31,8 @@ COPY ./.ruby-version "${APP_HOME}"/ COPY ./Gemfile ./Gemfile.lock "${APP_HOME}"/ COPY ./vendor/cache "${APP_HOME}"/vendor/cache -ENV BUNDLE_WITHOUT="development:test" RUN bundle config --local build.sassc --disable-march-tune-native && \ - bundle install --deployment --jobs 4 --retry 5 && \ + BUNDLE_WITHOUT="development:test" bundle install --deployment --jobs 4 --retry 5 && \ find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ find "${APP_HOME}"/vendor/bundle -name "*.o" -delete @@ -90,7 +89,6 @@ RUN dnf install --setopt install_weak_deps=false -y \ yum clean all && \ rm -rf /var/cache/yum -COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize @@ -99,20 +97,18 @@ RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}" USER "${APP_USER}" RUN bundle config --local build.sassc --disable-march-tune-native && \ + bundle config --delete without && \ bundle install --deployment --jobs 4 --retry 5 && \ find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ find "${APP_HOME}"/vendor/bundle -name "*.o" -delete -#RUN yarn install --frozen-lockfile && RAILS_ENV=test NODE_ENV=test bundle exec rails webpacker:compile - -ENTRYPOINT ["./scripts/entrypoint-dev.sh"] +ENTRYPOINT ["./scripts/entrypoint.sh"] CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] ## Development FROM builder AS development -COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize @@ -121,11 +117,12 @@ RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}" USER "${APP_USER}" RUN bundle config --local build.sassc --disable-march-tune-native && \ + bundle config --delete without && \ bundle install --deployment --jobs 4 --retry 5 && \ find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ find "${APP_HOME}"/vendor/bundle -name "*.o" -delete -ENTRYPOINT ["./scripts/entrypoint-dev.sh"] +ENTRYPOINT ["./scripts/entrypoint.sh"] CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] diff --git a/bin/container-setup b/bin/container-setup index ad5dd6e6f..4e433a2df 100755 --- a/bin/container-setup +++ b/bin/container-setup @@ -64,25 +64,12 @@ else fi -echo "Building the Forem container image!" - -if [ "${CONTAINER_RUNTIME}" = "docker" ]; then - echo "This will take a while if you are building the container for the first time." - - docker-compose build - -elif [ "${CONTAINER_RUNTIME}" = "podman" ]; then - - podman-compose -f container-compose.yml build - -fi - echo "Starting the Forem container stack..." if [ "${CONTAINER_RUNTIME}" = "docker" ]; then - if [[ ! -d .gems/ || ! "$(ls -A .gems)" ]]; then - echo "The .gems directory is empty or does not exist." + if [[ ! -d vendor/bundle || ! "$(ls -A vendor/bundle)" ]]; then + echo "The vendor/bundle directory is empty or does not exist." echo "This will cause bundle install to run which takes a long time with ${CONTAINER_RUNTIME}." echo "Depending on your hardware specs, the initial setup can take" echo "30 to 45 minutes. Please stand by..." diff --git a/container-compose.yml b/container-compose.yml index 9bd87e2d5..f21c99aca 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -1,9 +1,7 @@ version: "3" services: rails: - build: . - target: development - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_rails ports: - "3000:3000" @@ -17,52 +15,57 @@ services: RAILS_ENV: development DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 - REDIS_URL: redis://redis:6379 REDIS_SESSIONS_URL: redis://redis:6379 + REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 RACK_TIMEOUT_WAIT_TIMEOUT: 10000 RACK_TIMEOUT_SERVICE_TIMEOUT: 10000 STATEMENT_TIMEOUT: 10000 APP_DOMAIN: rails volumes: - .:/opt/apps/forem:z - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "300s", "./scripts/entrypoint-dev.sh"] - command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"] + entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/forem/vendor/bundle/.bundle_finished", "-timeout", "2700s"] + command: [ "bash", "-c", "./scripts/entrypoint.sh bootstrap && bundle exec rails server -b 0.0.0.0 -p 3000"] bundle: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_bundle environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:z - entrypoint: ["bash"] command: ["./scripts/bundle.sh"] yarn: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_yarn environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:z - entrypoint: ["bash", "-c"] - command: ["yarn install --dev"] + command: [ "bash", "-c", "yarn install --dev"] webpacker: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_webpacker depends_on: - rails - yarn environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: @@ -71,7 +74,7 @@ services: command: ["./bin/webpack-dev-server"] seed: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_seed depends_on: - rails @@ -80,9 +83,9 @@ services: - elasticsearch environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 REDIS_URL: redis://redis:6379 - REDIS_SESSIONS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: @@ -91,7 +94,7 @@ services: command: ["bundle", "exec", "rake","db:seed"] sidekiq: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_sidekiq depends_on: - rails @@ -100,7 +103,9 @@ services: - elasticsearch environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: @@ -121,7 +126,7 @@ services: - db_data:/var/lib/postgresql/data:Z redis: - image: "redis" + image: redis:6.0.9-alpine container_name: forem_redis ports: - "6379:6379" diff --git a/docker-compose.yml b/docker-compose.yml index fd0c1c583..068dad4f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,9 @@ version: "3" services: rails: - build: . - target: development - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_rails + user: root ports: - "3000:3000" depends_on: @@ -17,62 +16,72 @@ services: RAILS_ENV: development DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 - REDIS_URL: redis://redis:6379 REDIS_SESSIONS_URL: redis://redis:6379 + REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 RACK_TIMEOUT_WAIT_TIMEOUT: 10000 RACK_TIMEOUT_SERVICE_TIMEOUT: 10000 STATEMENT_TIMEOUT: 10000 APP_DOMAIN: rails volumes: - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "2700s", "./scripts/entrypoint-dev.sh"] - command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"] + entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/forem/bundle_finished", "-timeout", "2700s"] + command: [ "bash", "-c", "./scripts/entrypoint.sh bootstrap && bundle exec rails server -b 0.0.0.0 -p 3000"] bundle: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_bundle + user: root environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - entrypoint: ["bash"] command: ["./scripts/bundle.sh"] yarn: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_yarn + user: root environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - entrypoint: ["bash", "-c"] command: ["yarn install --dev"] webpacker: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_webpacker + user: root depends_on: + - bundle - rails - yarn environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-timeout", "2700s"] + entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-wait", "file:///opt/apps/forem/bundle_finished", "-timeout", "2700s"] command: ["./bin/webpack-dev-server"] seed: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_seed + user: root depends_on: - rails - redis @@ -80,20 +89,20 @@ services: - elasticsearch environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 REDIS_URL: redis://redis:6379 - REDIS_SESSIONS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"] command: ["bundle", "exec", "rake","db:seed"] sidekiq: - image: forem-rails:latest + image: quay.io/forem/forem:development container_name: forem_sidekiq + user: root depends_on: - rails - redis @@ -101,12 +110,13 @@ services: - elasticsearch environment: RAILS_ENV: development + REDIS_SESSIONS_URL: redis://redis:6379 REDIS_SIDEKIQ_URL: redis://redis:6379 + REDIS_URL: redis://redis:6379 DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development ELASTICSEARCH_URL: http://elasticsearch:9200 volumes: - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"] command: ["bundle", "exec", "sidekiq","-c","2"] @@ -123,7 +133,7 @@ services: - db_data:/var/lib/postgresql/data:delegated redis: - image: "redis" + image: redis:6.0.9-alpine container_name: forem_redis ports: - "6379:6379" diff --git a/scripts/bundle.sh b/scripts/bundle.sh index 1060ac791..6153bf787 100755 --- a/scripts/bundle.sh +++ b/scripts/bundle.sh @@ -2,9 +2,11 @@ set -e -if [ -f /opt/apps/bundle/bundle_finished ]; then - rm -f /opt/apps/bundle/bundle_finished +if [ -f /opt/apps/forem/vendor/bundle/.bundle_finished ]; then + rm -f /opt/apps/forem/vendor/bundle/.bundle_finished fi -bundle install --local --jobs 20 --retry 5 -echo $(date --utc +%FT%T%Z) > /opt/apps/bundle/bundle_finished +unset BUNDLE_WITHOUT +bundle config --delete without +bundle install --local --jobs 20 --retry 5 --path vendor/bundle +echo $(date --utc +%FT%T%Z) > /opt/apps/forem/vendor/bundle/.bundle_finished diff --git a/scripts/entrypoint-dev.sh b/scripts/entrypoint-dev.sh deleted file mode 100755 index 4e7394f8e..000000000 --- a/scripts/entrypoint-dev.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ -f tmp/pids/server.pid ]; then - rm -f tmp/pids/server.pid -fi - -export RELEASE_FOOTPRINT=$(date -u +'%Y-%m-%dT%H:%M:%SZ') -export FOREM_BUILD_DATE=$(cat FOREM_BUILD_DATE) -export FOREM_BUILD_SHA=$(cat FOREM_BUILD_SHA) - -echo "Running rake app_initializer:setup..." -bundle exec rake app_initializer:setup - -exec "$@" diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 47f9267c2..f762fb0ba 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -7,8 +7,18 @@ if [ -f tmp/pids/server.pid ]; then fi export RELEASE_FOOTPRINT=$(date -u +'%Y-%m-%dT%H:%M:%SZ') -export FOREM_BUILD_DATE=$(cat FOREM_BUILD_DATE) -export FOREM_BUILD_SHA=$(cat FOREM_BUILD_SHA) + +if [[ "${RAILS_ENV}" = "development" || "${RAILS_ENV}" = "test" ]]; then + + export FOREM_BUILD_DATE=RELEASE_FOOTPRINT + export FOREM_BUILD_SHA=$(git rev-parse --short HEAD) + +else + + export FOREM_BUILD_DATE=$(cat FOREM_BUILD_DATE) + export FOREM_BUILD_SHA=$(cat FOREM_BUILD_SHA) + +fi case "$@" in