Fix container composes w/r/t multistage container builds (#11531)

* Use premade development image.

* Use prebuilt container for podman-compose too!

* Fix bundle bundle_finished path.

* Fix docker-compose to work with the Multistage Containerfile

Add root user for Forem containers, fix bundle_finished path and make
webpacker wait for the bundle to finish.

Switch Redis to redis:6.0.9-alpine

* Remove build steps and fix bundle path.

* Merge dev entrypoint into the main one.

* Use merged entrypoint.sh, remove old entrypoint directives, and fix a dockerize bug.

* Make sure unset BUNDLE_WITHOUT and remove without in bundle config.

* Unset BUNDLE_WITHOUT="development:test" as a global ENV.

Also ensure that we remove the without directives for development and testing.
This will make sure that the right gems are installed for each environment.

Removed some unneeded COPYs and remove a commented out RUN.

* Fix ENTRYPOINT on development and testing containers.

* Fix entrypoint and command on rails container.

* Ignore bundle_finished file.

* Fix REDIS_ env vars and adjust podman-compose file.

* Change bundle_finished path and update Yarn docker-compose command.

.gitignore

@@ -82,5 +82,8 @@ db_data
 es_data
 gem_cache
 
+# Compose generated files
+bundle_finished
+
 # Auto-generated Storybook stories
 app/javascript/generated_stories/

Containerfile

@@ -31,9 +31,8 @@ COPY ./.ruby-version "${APP_HOME}"/
 COPY ./Gemfile ./Gemfile.lock "${APP_HOME}"/
 COPY ./vendor/cache "${APP_HOME}"/vendor/cache
 
-ENV BUNDLE_WITHOUT="development:test"
 RUN bundle config --local build.sassc --disable-march-tune-native && \
-    bundle install --deployment --jobs 4 --retry 5 && \
+    BUNDLE_WITHOUT="development:test" bundle install --deployment --jobs 4 --retry 5 && \
     find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
     find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
 
@@ -90,7 +89,6 @@ RUN dnf install --setopt install_weak_deps=false -y \
     yum clean all && \
     rm -rf /var/cache/yum
 
-COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets
 COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
 COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
 
@@ -99,20 +97,18 @@ RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}"
 USER "${APP_USER}"
 
 RUN bundle config --local build.sassc --disable-march-tune-native && \
+    bundle config --delete without && \
     bundle install --deployment --jobs 4 --retry 5 && \
     find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
     find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
 
-#RUN yarn install --frozen-lockfile && RAILS_ENV=test NODE_ENV=test bundle exec rails webpacker:compile
+ENTRYPOINT ["./scripts/entrypoint.sh"]
-
-ENTRYPOINT ["./scripts/entrypoint-dev.sh"]
 
 CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
 
 ## Development
 FROM builder AS development
 
-COPY --chown="${APP_USER}":"${APP_USER}" ./app/assets "${APP_HOME}"/app/assets
 COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
 COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
 
@@ -121,11 +117,12 @@ RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}"
 USER "${APP_USER}"
 
 RUN bundle config --local build.sassc --disable-march-tune-native && \
+    bundle config --delete without && \
     bundle install --deployment --jobs 4 --retry 5 && \
     find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
     find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
 
-ENTRYPOINT ["./scripts/entrypoint-dev.sh"]
+ENTRYPOINT ["./scripts/entrypoint.sh"]
 
 CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
 

bin/container-setup

@@ -64,25 +64,12 @@ else
 
 fi
 
-echo "Building the Forem container image!"
-
-if [ "${CONTAINER_RUNTIME}" = "docker" ]; then
-  echo "This will take a while if you are building the container for the first time."
-
-  docker-compose build
-
-elif [ "${CONTAINER_RUNTIME}" = "podman" ]; then
-
-  podman-compose -f container-compose.yml build
-
-fi
-
 echo "Starting the Forem container stack..."
 
 if [ "${CONTAINER_RUNTIME}" = "docker" ]; then
 
-  if [[ ! -d .gems/ || ! "$(ls -A .gems)" ]]; then
+  if [[ ! -d vendor/bundle || ! "$(ls -A vendor/bundle)" ]]; then
-    echo "The .gems directory is empty or does not exist."
+    echo "The vendor/bundle directory is empty or does not exist."
     echo "This will cause bundle install to run which takes a long time with ${CONTAINER_RUNTIME}."
     echo "Depending on your hardware specs, the initial setup can take"
     echo "30 to 45 minutes. Please stand by..."

container-compose.yml

@@ -1,9 +1,7 @@
 version: "3"
 services:
   rails:
-    build: .
+    image: quay.io/forem/forem:development
-      target: development
-    image: forem-rails:latest
     container_name: forem_rails
     ports:
       - "3000:3000"
@@ -17,52 +15,57 @@ services:
       RAILS_ENV: development
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
-      REDIS_URL: redis://redis:6379
       REDIS_SESSIONS_URL: redis://redis:6379
+      REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       RACK_TIMEOUT_WAIT_TIMEOUT: 10000
       RACK_TIMEOUT_SERVICE_TIMEOUT: 10000
       STATEMENT_TIMEOUT: 10000
       APP_DOMAIN: rails
     volumes:
       - .:/opt/apps/forem:z
-    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "300s", "./scripts/entrypoint-dev.sh"]
+    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/forem/vendor/bundle/.bundle_finished", "-timeout", "2700s"]
-    command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"]
+    command: [ "bash", "-c", "./scripts/entrypoint.sh bootstrap && bundle exec rails server -b 0.0.0.0 -p 3000"]
 
   bundle:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_bundle
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:z
-    entrypoint: ["bash"]
     command: ["./scripts/bundle.sh"]
 
   yarn:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_yarn
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:z
-    entrypoint: ["bash", "-c"]
+    command: [ "bash", "-c", "yarn install --dev"]
-    command: ["yarn install --dev"]
 
   webpacker:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_webpacker
     depends_on:
       - rails
       - yarn
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
@@ -71,7 +74,7 @@ services:
     command: ["./bin/webpack-dev-server"]
 
   seed:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_seed
     depends_on:
       - rails
@@ -80,9 +83,9 @@ services:
       - elasticsearch
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
       REDIS_URL: redis://redis:6379
-      REDIS_SESSIONS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
@@ -91,7 +94,7 @@ services:
     command: ["bundle", "exec", "rake","db:seed"]
 
   sidekiq:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_sidekiq
     depends_on:
       - rails
@@ -100,7 +103,9 @@ services:
       - elasticsearch
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
@@ -121,7 +126,7 @@ services:
       - db_data:/var/lib/postgresql/data:Z
 
   redis:
-    image: "redis"
+    image: redis:6.0.9-alpine
     container_name: forem_redis
     ports:
       - "6379:6379"

docker-compose.yml

@@ -1,10 +1,9 @@
 version: "3"
 services:
   rails:
-    build: .
+    image: quay.io/forem/forem:development
-      target: development
-    image: forem-rails:latest
     container_name: forem_rails
+    user: root
     ports:
       - "3000:3000"
     depends_on:
@@ -17,62 +16,72 @@ services:
       RAILS_ENV: development
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
-      REDIS_URL: redis://redis:6379
       REDIS_SESSIONS_URL: redis://redis:6379
+      REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       RACK_TIMEOUT_WAIT_TIMEOUT: 10000
       RACK_TIMEOUT_SERVICE_TIMEOUT: 10000
       STATEMENT_TIMEOUT: 10000
       APP_DOMAIN: rails
     volumes:
       - .:/opt/apps/forem:delegated
-    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/bundle/bundle_finished", "-timeout", "2700s", "./scripts/entrypoint-dev.sh"]
+    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/forem/bundle_finished", "-timeout", "2700s"]
-    command: ["bundle", "exec", "rails","server","-b","0.0.0.0","-p","3000"]
+    command: [ "bash", "-c", "./scripts/entrypoint.sh bootstrap && bundle exec rails server -b 0.0.0.0 -p 3000"]
 
   bundle:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_bundle
+    user: root
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:delegated
-    entrypoint: ["bash"]
     command: ["./scripts/bundle.sh"]
 
   yarn:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_yarn
+    user: root
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:delegated
-    entrypoint: ["bash", "-c"]
     command: ["yarn install --dev"]
 
   webpacker:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_webpacker
+    user: root
     depends_on:
+      - bundle
       - rails
       - yarn
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:delegated
-    entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-timeout", "2700s"]
+    entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-wait", "file:///opt/apps/forem/bundle_finished", "-timeout", "2700s"]
     command: ["./bin/webpack-dev-server"]
 
   seed:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_seed
+    user: root
     depends_on:
       - rails
       - redis
@@ -80,20 +89,20 @@ services:
       - elasticsearch
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
       REDIS_URL: redis://redis:6379
-      REDIS_SESSIONS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:delegated
-
     entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"]
     command: ["bundle", "exec", "rake","db:seed"]
 
   sidekiq:
-    image: forem-rails:latest
+    image: quay.io/forem/forem:development
     container_name: forem_sidekiq
+    user: root
     depends_on:
       - rails
       - redis
@@ -101,12 +110,13 @@ services:
       - elasticsearch
     environment:
       RAILS_ENV: development
+      REDIS_SESSIONS_URL: redis://redis:6379
       REDIS_SIDEKIQ_URL: redis://redis:6379
+      REDIS_URL: redis://redis:6379
       DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
       ELASTICSEARCH_URL: http://elasticsearch:9200
     volumes:
       - .:/opt/apps/forem:delegated
-
     entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "http://elasticsearch:9200", "-wait", "tcp://redis:6379", "-wait", "http://rails:3000", "-timeout", "2700s"]
     command: ["bundle", "exec", "sidekiq","-c","2"]
 
@@ -123,7 +133,7 @@ services:
       - db_data:/var/lib/postgresql/data:delegated
 
   redis:
-    image: "redis"
+    image: redis:6.0.9-alpine
     container_name: forem_redis
     ports:
       - "6379:6379"

scripts/bundle.sh

@@ -2,9 +2,11 @@
 
 set -e
 
-if [ -f /opt/apps/bundle/bundle_finished ]; then
+if [ -f /opt/apps/forem/vendor/bundle/.bundle_finished ]; then
-  rm -f /opt/apps/bundle/bundle_finished
+  rm -f /opt/apps/forem/vendor/bundle/.bundle_finished
 fi
 
-bundle install --local --jobs 20 --retry 5
+unset BUNDLE_WITHOUT
-echo $(date --utc +%FT%T%Z) > /opt/apps/bundle/bundle_finished
+bundle config --delete without
+bundle install --local --jobs 20 --retry 5 --path vendor/bundle
+echo $(date --utc +%FT%T%Z) > /opt/apps/forem/vendor/bundle/.bundle_finished

scripts/entrypoint-dev.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [ -f tmp/pids/server.pid ]; then
-  rm -f tmp/pids/server.pid
-fi
-
-export RELEASE_FOOTPRINT=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-export FOREM_BUILD_DATE=$(cat FOREM_BUILD_DATE)
-export FOREM_BUILD_SHA=$(cat FOREM_BUILD_SHA)
-
-echo "Running rake app_initializer:setup..."
-bundle exec rake app_initializer:setup
-
-exec "$@"

scripts/entrypoint.sh

@@ -7,8 +7,18 @@ if [ -f tmp/pids/server.pid ]; then
 fi
 
 export RELEASE_FOOTPRINT=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-export FOREM_BUILD_DATE=$(cat FOREM_BUILD_DATE)
+
-export FOREM_BUILD_SHA=$(cat FOREM_BUILD_SHA)
+if [[ "${RAILS_ENV}" = "development" || "${RAILS_ENV}" = "test" ]]; then
+
+  export FOREM_BUILD_DATE=RELEASE_FOOTPRINT
+  export FOREM_BUILD_SHA=$(git rev-parse --short HEAD)
+
+else
+
+  export FOREM_BUILD_DATE=$(cat FOREM_BUILD_DATE)
+  export FOREM_BUILD_SHA=$(cat FOREM_BUILD_SHA)
+
+fi
 
 case "$@" in