Improve docker local development (#19996)

.dockerdev/.psqlrc

@@ -0,0 +1,26 @@
+-- Don't display the "helpful" message on startup.
+\set QUIET 1
+
+-- Allow specifying the path to history file via `PSQL_HISTFILE` env variable
+-- (and fallback to the default $HOME/.psql_history otherwise)
+\set HISTFILE `[ -z $PSQL_HISTFILE ] && echo $HOME/.psql_history || echo $PSQL_HISTFILE`
+
+-- Show how long each query takes to execute
+\timing
+
+-- Use best available output format
+\x auto
+
+-- Verbose error reports
+\set VERBOSITY verbose
+
+-- If a command is run more than once in a row,
+-- only store it once in the history
+\set HISTCONTROL ignoredups
+\set COMP_KEYWORD_CASE upper
+
+-- By default, NULL displays as an empty space. Is it actually an empty
+-- string, or is it null? This makes that distinction visible
+\pset null '[NULL]'
+
+\unset QUIET

.env_sample

@@ -22,17 +22,17 @@ HEROKU_SLUG_COMMIT=""
 # Redis caching storage
 REDIS_URL="redis://localhost:6379"
 
-# Redis sessions storage
+# Redis sessions storage, rely on REDIS_URL if not set
-REDIS_SESSIONS_URL="redis://localhost:6379"
+# REDIS_SESSIONS_URL=
 SESSION_KEY="_Dev_Community_Session"
 # two weeks in seconds
 SESSION_EXPIRY_SECONDS=1209600
 
-# Redis Sidekiq storage
+# Redis Sidekiq storage, rely on REDIS_URL if not set
-REDIS_SIDEKIQ_URL="redis://localhost:6379"
+# REDIS_SIDEKIQ_URL=
 
-# Redis Devices/Rpush storage
+# Redis Devices/Rpush storage, rely on REDIS_URL if not set
-REDIS_RPUSH_URL="redis://localhost:6379"
+# REDIS_RPUSH_URL=
 
 # OpenResty
 OPENRESTY_URL=""

Containerfile

@@ -147,19 +147,71 @@ ENTRYPOINT ["./scripts/entrypoint.sh"]
 CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
 
 ## Development
-FROM builder AS development
+FROM base AS development
 
-USER "${APP_USER}"
+# Common dependencies
+# Using --mount to speed up build with caching, see https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mount
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  --mount=type=tmpfs,target=/var/log \
+  rm -f /etc/apt/apt.conf.d/docker-clean; \
+  echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \
+  apt-get update -qq && \
+  DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+    build-essential \
+    gnupg2 \
+    curl \
+    less \
+    git
 
-COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
+ARG PG_MAJOR
-COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
+RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgres-archive-keyring.gpg \
+  && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt/" \
+    $(lsb_release -cs)-pgdg main $PG_MAJOR | tee /etc/apt/sources.list.d/postgres.list > /dev/null
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  --mount=type=tmpfs,target=/var/log \
+  apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+    libpq-dev \
+    postgresql-client
 
-RUN bundle config --local build.sassc --disable-march-tune-native && \
+ARG NODE_MAJOR
-    bundle config --delete without && \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    BUNDLE_FROZEN=true bundle install --deployment --jobs 4 --retry 5 && \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
+  --mount=type=tmpfs,target=/var/log \
-    find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
+  curl -sL https://deb.nodesource.com/setup_$NODE_MAJOR.x | bash - && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+    nodejs
 
-ENTRYPOINT ["./scripts/entrypoint.sh"]
+# Application dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    --mount=type=tmpfs,target=/var/log \
+    DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+      imagemagick
 
-CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
+# Configure bundler
+ENV LANG=C.UTF-8 \
+  BUNDLE_JOBS=4 \
+  BUNDLE_RETRY=3
+
+# Store Bundler settings in the project's root
+ENV BUNDLE_APP_CONFIG=.bundle
+
+# Uncomment this line if you want to run binstubs without prefixing with `bin/` or `bundle exec`
+# ENV PATH /app/bin:$PATH
+
+# Upgrade RubyGems and install the latest Bundler version
+RUN gem update --system && \
+    gem install bundler
+
+# Create a directory for the app code
+RUN mkdir -p /app
+WORKDIR /app
+
+# Document that we're going to expose port 3000
+EXPOSE 3000
+# Use Bash as the default command
+CMD ["/usr/bin/bash"]

dip.yml

@@ -0,0 +1,78 @@
+version: '7.1'
+
+# Define default environment variables to pass
+# to Docker Compose
+environment:
+  RAILS_ENV: development
+
+compose:
+  files:
+    - docker-compose.yml
+  project_name: forem
+
+interaction:
+  # This command spins up a Rails container with the required dependencies (such as databases),
+  # and opens a terminal within it.
+  runner:
+    description: Open a Bash shell within a Rails container (with dependencies up)
+    service: rails
+    command: /bin/bash
+
+  # Run a Rails container without any dependent services (useful for non-Rails scripts)
+  bash:
+    description: Run an arbitrary script within a container (or open a shell without deps)
+    service: rails
+    command: /bin/bash
+    compose_run_options: [ no-deps ]
+
+  # A shortcut to run Bundler commands
+  bundle:
+    description: Run Bundler commands
+    service: rails
+    command: bundle
+    compose_run_options: [ no-deps ]
+
+  # A shortcut to run RSpec (which overrides the RAILS_ENV)
+  rspec:
+    description: Run RSpec commands
+    service: rails
+    environment:
+      RAILS_ENV: test
+    command: bundle exec rspec
+
+  rails:
+    description: Run Rails commands
+    service: rails
+    command: bundle exec rails
+    subcommands:
+      s:
+        description: Run Rails server at http://localhost:3000
+        service: web
+        compose:
+          run_options: [ service-ports, use-aliases ]
+
+  yarn:
+    description: Run Yarn commands
+    service: rails
+    command: yarn
+    compose_run_options: [ no-deps ]
+
+  psql:
+    description: Run Postgres psql console
+    service: postgres
+    default_args: practicaldeveloper_development
+    command: psql -h postgres -U postgres
+
+  'redis-cli':
+    description: Run Redis console
+    service: redis
+    command: redis-cli -h redis
+
+provision:
+  # You may include this command if you would like to remove all volumes
+  # - dip compose down --volumes
+  #
+  - dip compose up -d postgres
+  - dip compose up -d redis
+  - dip bash -c bin/setup
+  - dip rails db:test:prepare RAILS_ENV=test

docker-compose.yml

@@ -1,150 +1,124 @@
-version: "3"
+x-app: &app
+  build:
+    target: development
+    context: .
+    args:
+      PG_MAJOR: '13'
+      NODE_MAJOR: '16'
+      YARN_VERSION: '1.22.18'
+  image: ghcr.io/forem/forem:development
+  environment: &env
+    NODE_ENV: ${NODE_ENV:-development}
+    RAILS_ENV: ${RAILS_ENV:-development}
+  tmpfs:
+    - /tmp
+    - /app/tmp/pids
+
+x-backend: &backend
+  <<: *app
+  stdin_open: true
+  tty: true
+  volumes:
+    - .:/app:cached
+    - bundle:/usr/local/bundle
+    - rails_cache:/app/tmp/cache
+    - assets:/app/public/assets
+    - node_modules:/app/node_modules
+    - packs:/app/public/packs
+    - packs-test:/app/public/packs-test
+    - history:/usr/local/hist
+    - ./.dockerdev/.psqlrc:/root/.psqlrc:ro
+  environment: &backend_environment
+    <<: *env
+    REDIS_URL: redis://redis:6379/
+    DATABASE_URL: postgres://postgres:postgres@postgres:5432
+    DATABASE_URL_TEST: postgres://postgres:postgres@postgres:5432
+    WEBPACKER_DEV_SERVER_HOST: webpacker
+    MALLOC_ARENA_MAX: 2
+    WEB_CONCURRENCY: ${WEB_CONCURRENCY:-1}
+    BOOTSNAP_CACHE_DIR: /usr/local/bundle/_bootsnap
+    XDG_DATA_HOME: /app/tmp/cache
+    YARN_CACHE_FOLDER: /app/node_modules/.yarn-cache
+    HISTFILE: /usr/local/hist/.bash_history
+    PSQL_HISTFILE: /usr/local/hist/.psql_history
+    IRB_HISTFILE: /usr/local/hist/.irb_history
+  depends_on:
+    postgres:
+      condition: service_healthy
+    redis:
+      condition: service_healthy
+
 services:
   rails:
-    image: quay.io/forem/forem:development
+    <<: *backend
-    container_name: forem_rails
+    command: bundle exec rails
-    user: root
+
+  web:
+    <<: *backend
+    command: bundle exec rails server -b 0.0.0.0
     ports:
-      - "3000:3000"
+      - '3000:3000'
     depends_on:
-      - bundle
+      webpacker:
-      - db
+        condition: service_started
-      - redis
+      sidekiq:
-      - yarn
+        condition: service_started
-      - testdb
-    environment:
-      RAILS_ENV: development
-      DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
-      DATABASE_URL_TEST: postgresql://forem:forem@testdb:5432/Forem_test
-      REDIS_SESSIONS_URL: redis://redis:6379
-      REDIS_SIDEKIQ_URL: redis://redis:6379
-      REDIS_URL: redis://redis:6379
-      RACK_TIMEOUT_WAIT_TIMEOUT: 10000
-      RACK_TIMEOUT_SERVICE_TIMEOUT: 10000
-      STATEMENT_TIMEOUT: 10000
-      APP_DOMAIN: localhost:3000
-    tmpfs:
-      - /tmp
-    volumes:
-      - .:/opt/apps/forem:delegated
-    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/forem/vendor/bundle/.bundle_finished", "-timeout", "2700s", "-wait-retry-interval", "10s"]
-    command: [ "bash", "-c", "./scripts/entrypoint.sh bootstrap && bundle exec rails server -b 0.0.0.0 -p 3000"]
-
-  bundle:
-    image: quay.io/forem/forem:development
-    container_name: forem_bundle
-    user: root
-    environment:
-      RAILS_ENV: development
-      REDIS_SESSIONS_URL: redis://redis:6379
-      REDIS_SIDEKIQ_URL: redis://redis:6379
-      REDIS_URL: redis://redis:6379
-      DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
-    volumes:
-      - .:/opt/apps/forem:delegated
-    command: ["./scripts/bundle.sh"]
-
-  yarn:
-    image: quay.io/forem/forem:development
-    container_name: forem_yarn
-    user: root
-    environment:
-      RAILS_ENV: development
-      REDIS_SESSIONS_URL: redis://redis:6379
-      REDIS_SIDEKIQ_URL: redis://redis:6379
-      REDIS_URL: redis://redis:6379
-      DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
-    volumes:
-      - .:/opt/apps/forem:delegated
-    command: yarn install --dev
-
-  webpacker:
-    image: quay.io/forem/forem:development
-    container_name: forem_webpacker
-    user: root
-    depends_on:
-      - bundle
-      - rails
-      - yarn
-    environment:
-      RAILS_ENV: development
-      REDIS_SESSIONS_URL: redis://redis:6379
-      REDIS_SIDEKIQ_URL: redis://redis:6379
-      REDIS_URL: redis://redis:6379
-      DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
-    volumes:
-      - .:/opt/apps/forem:delegated
-    entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-wait", "file:///opt/apps/forem/vendor/bundle/.bundle_finished", "-timeout", "2700s", "-wait-retry-interval", "10s"]
-    command: ["./bin/webpack-dev-server"]
-
-  seed:
-    image: quay.io/forem/forem:development
-    container_name: forem_seed
-    user: root
-    depends_on:
-      - rails
-      - redis
-      - db
-    environment:
-      RAILS_ENV: development
-      REDIS_SESSIONS_URL: redis://redis:6379
-      REDIS_SIDEKIQ_URL: redis://redis:6379
-      REDIS_URL: redis://redis:6379
-      DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
-    volumes:
-      - .:/opt/apps/forem:delegated
-    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "tcp://redis:6379", "-wait", "tcp://rails:3000", "-timeout", "2700s", "-wait-retry-interval", "20s"]
-    command: ["bundle", "exec", "rake","db:seed"]
 
   sidekiq:
-    image: quay.io/forem/forem:development
+    <<: *backend
-    container_name: forem_sidekiq
+    command: bundle exec sidekiq
-    user: root
-    depends_on:
-      - rails
-      - redis
-      - db
-    environment:
-      RAILS_ENV: development
-      REDIS_SESSIONS_URL: redis://redis:6379
-      REDIS_SIDEKIQ_URL: redis://redis:6379
-      REDIS_URL: redis://redis:6379
-      DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development
-    volumes:
-      - .:/opt/apps/forem:delegated
-    entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "tcp://redis:6379", "-wait", "tcp://rails:3000", "-timeout", "2700s", "-wait-retry-interval", "20s"]
-    command: ["bundle", "exec", "sidekiq","-c","2"]
 
-  db:
+  postgres:
-    image: postgres:13-alpine
+    image: postgres:13
-    container_name: forem_postgresql
+    volumes:
+      - ./.dockerdev/.psqlrc:/root/.psqlrc:ro
+      - postgres:/var/lib/postgresql/data
+      - history:/usr/local/hist
     environment:
-      POSTGRES_USER: forem
+      PSQL_HISTFILE: /usr/local/hist/.psql_history
-      POSTGRES_PASSWORD: forem
+      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: PracticalDeveloper_development
     ports:
-      - "5432:5432"
+      - 5432
-    volumes:
+    healthcheck:
-      - db_data:/var/lib/postgresql/data:delegated
+      test: pg_isready -U postgres -h 127.0.0.1
-
+      interval: 5s
-  testdb:
-    image: postgres:13-alpine
-    container_name: forem_postgresql_test
-    environment:
-      POSTGRES_USER: forem
-      POSTGRES_PASSWORD: forem
-      POSTGRES_DB: Forem_test
-    expose:
-      - "5432"
-    volumes:
-      - testdb_data:/var/lib/postgresql/data:delegated
 
   redis:
-    image: redis:6.0.9-alpine
+    image: redis:7.0-alpine
-    container_name: forem_redis
+    volumes:
+      - redis:/data
     ports:
-      - "6379:6379"
+      - 6379
+    healthcheck:
+      test: redis-cli ping
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  webpacker:
+    <<: *app
+    command: bundle exec ./bin/webpack-dev-server
+    ports:
+      - '3035:3035'
+    volumes:
+      - .:/app:cached
+      - bundle:/usr/local/bundle
+      - node_modules:/app/node_modules
+      - packs:/app/public/packs
+      - packs-test:/app/public/packs-test
+    environment:
+      <<: *env
+      WEBPACKER_DEV_SERVER_HOST: 0.0.0.0
+      YARN_CACHE_FOLDER: /app/node_modules/.yarn-cache
 
 volumes:
-  db_data:
+  bundle:
-  testdb_data:
+  node_modules:
+  history:
+  rails_cache:
+  postgres:
+  redis:
+  assets:
+  packs:
+  packs-test:
+

jest.config.js

@@ -3,6 +3,7 @@
 
 /* eslint-env node */
 
+process.env.NODE_ENV = 'test';
 process.env.TZ = 'UTC';
 
 module.exports = {