From ca6e6355493ddb49a31f697dd64baa2904356df6 Mon Sep 17 00:00:00 2001 From: Mac Siri Date: Fri, 1 Sep 2023 14:13:40 -0400 Subject: [PATCH] Improve docker local development (#19996) --- .dockerdev/.psqlrc | 26 +++++ .env_sample | 12 +-- Containerfile | 74 ++++++++++++-- dip.yml | 78 ++++++++++++++ docker-compose.yml | 246 ++++++++++++++++++++------------------------- jest.config.js | 1 + 6 files changed, 284 insertions(+), 153 deletions(-) create mode 100644 .dockerdev/.psqlrc create mode 100644 dip.yml diff --git a/.dockerdev/.psqlrc b/.dockerdev/.psqlrc new file mode 100644 index 000000000..05c0dcb22 --- /dev/null +++ b/.dockerdev/.psqlrc @@ -0,0 +1,26 @@ +-- Don't display the "helpful" message on startup. +\set QUIET 1 + +-- Allow specifying the path to history file via `PSQL_HISTFILE` env variable +-- (and fallback to the default $HOME/.psql_history otherwise) +\set HISTFILE `[ -z $PSQL_HISTFILE ] && echo $HOME/.psql_history || echo $PSQL_HISTFILE` + +-- Show how long each query takes to execute +\timing + +-- Use best available output format +\x auto + +-- Verbose error reports +\set VERBOSITY verbose + +-- If a command is run more than once in a row, +-- only store it once in the history +\set HISTCONTROL ignoredups +\set COMP_KEYWORD_CASE upper + +-- By default, NULL displays as an empty space. Is it actually an empty +-- string, or is it null? This makes that distinction visible +\pset null '[NULL]' + +\unset QUIET diff --git a/.env_sample b/.env_sample index 628d06270..e662d812d 100644 --- a/.env_sample +++ b/.env_sample @@ -22,17 +22,17 @@ HEROKU_SLUG_COMMIT="" # Redis caching storage REDIS_URL="redis://localhost:6379" -# Redis sessions storage -REDIS_SESSIONS_URL="redis://localhost:6379" +# Redis sessions storage, rely on REDIS_URL if not set +# REDIS_SESSIONS_URL= SESSION_KEY="_Dev_Community_Session" # two weeks in seconds SESSION_EXPIRY_SECONDS=1209600 -# Redis Sidekiq storage -REDIS_SIDEKIQ_URL="redis://localhost:6379" +# Redis Sidekiq storage, rely on REDIS_URL if not set +# REDIS_SIDEKIQ_URL= -# Redis Devices/Rpush storage -REDIS_RPUSH_URL="redis://localhost:6379" +# Redis Devices/Rpush storage, rely on REDIS_URL if not set +# REDIS_RPUSH_URL= # OpenResty OPENRESTY_URL="" diff --git a/Containerfile b/Containerfile index 1cd2fd9db..120b908f1 100644 --- a/Containerfile +++ b/Containerfile @@ -147,19 +147,71 @@ ENTRYPOINT ["./scripts/entrypoint.sh"] CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] ## Development -FROM builder AS development +FROM base AS development -USER "${APP_USER}" +# Common dependencies +# Using --mount to speed up build with caching, see https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mount +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=tmpfs,target=/var/log \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ + apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + build-essential \ + gnupg2 \ + curl \ + less \ + git -COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec -COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize +ARG PG_MAJOR +RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgres-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt/" \ + $(lsb_release -cs)-pgdg main $PG_MAJOR | tee /etc/apt/sources.list.d/postgres.list > /dev/null +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=tmpfs,target=/var/log \ + apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + libpq-dev \ + postgresql-client -RUN bundle config --local build.sassc --disable-march-tune-native && \ - bundle config --delete without && \ - BUNDLE_FROZEN=true bundle install --deployment --jobs 4 --retry 5 && \ - find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \ - find "${APP_HOME}"/vendor/bundle -name "*.o" -delete +ARG NODE_MAJOR +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=tmpfs,target=/var/log \ + curl -sL https://deb.nodesource.com/setup_$NODE_MAJOR.x | bash - && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + nodejs -ENTRYPOINT ["./scripts/entrypoint.sh"] +# Application dependencies +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=tmpfs,target=/var/log \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + imagemagick -CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"] +# Configure bundler +ENV LANG=C.UTF-8 \ + BUNDLE_JOBS=4 \ + BUNDLE_RETRY=3 + +# Store Bundler settings in the project's root +ENV BUNDLE_APP_CONFIG=.bundle + +# Uncomment this line if you want to run binstubs without prefixing with `bin/` or `bundle exec` +# ENV PATH /app/bin:$PATH + +# Upgrade RubyGems and install the latest Bundler version +RUN gem update --system && \ + gem install bundler + +# Create a directory for the app code +RUN mkdir -p /app +WORKDIR /app + +# Document that we're going to expose port 3000 +EXPOSE 3000 +# Use Bash as the default command +CMD ["/usr/bin/bash"] diff --git a/dip.yml b/dip.yml new file mode 100644 index 000000000..79375888d --- /dev/null +++ b/dip.yml @@ -0,0 +1,78 @@ +version: '7.1' + +# Define default environment variables to pass +# to Docker Compose +environment: + RAILS_ENV: development + +compose: + files: + - docker-compose.yml + project_name: forem + +interaction: + # This command spins up a Rails container with the required dependencies (such as databases), + # and opens a terminal within it. + runner: + description: Open a Bash shell within a Rails container (with dependencies up) + service: rails + command: /bin/bash + + # Run a Rails container without any dependent services (useful for non-Rails scripts) + bash: + description: Run an arbitrary script within a container (or open a shell without deps) + service: rails + command: /bin/bash + compose_run_options: [ no-deps ] + + # A shortcut to run Bundler commands + bundle: + description: Run Bundler commands + service: rails + command: bundle + compose_run_options: [ no-deps ] + + # A shortcut to run RSpec (which overrides the RAILS_ENV) + rspec: + description: Run RSpec commands + service: rails + environment: + RAILS_ENV: test + command: bundle exec rspec + + rails: + description: Run Rails commands + service: rails + command: bundle exec rails + subcommands: + s: + description: Run Rails server at http://localhost:3000 + service: web + compose: + run_options: [ service-ports, use-aliases ] + + yarn: + description: Run Yarn commands + service: rails + command: yarn + compose_run_options: [ no-deps ] + + psql: + description: Run Postgres psql console + service: postgres + default_args: practicaldeveloper_development + command: psql -h postgres -U postgres + + 'redis-cli': + description: Run Redis console + service: redis + command: redis-cli -h redis + +provision: + # You may include this command if you would like to remove all volumes + # - dip compose down --volumes + # + - dip compose up -d postgres + - dip compose up -d redis + - dip bash -c bin/setup + - dip rails db:test:prepare RAILS_ENV=test diff --git a/docker-compose.yml b/docker-compose.yml index 58be68b15..1e0bf595f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,150 +1,124 @@ -version: "3" +x-app: &app + build: + target: development + context: . + args: + PG_MAJOR: '13' + NODE_MAJOR: '16' + YARN_VERSION: '1.22.18' + image: ghcr.io/forem/forem:development + environment: &env + NODE_ENV: ${NODE_ENV:-development} + RAILS_ENV: ${RAILS_ENV:-development} + tmpfs: + - /tmp + - /app/tmp/pids + +x-backend: &backend + <<: *app + stdin_open: true + tty: true + volumes: + - .:/app:cached + - bundle:/usr/local/bundle + - rails_cache:/app/tmp/cache + - assets:/app/public/assets + - node_modules:/app/node_modules + - packs:/app/public/packs + - packs-test:/app/public/packs-test + - history:/usr/local/hist + - ./.dockerdev/.psqlrc:/root/.psqlrc:ro + environment: &backend_environment + <<: *env + REDIS_URL: redis://redis:6379/ + DATABASE_URL: postgres://postgres:postgres@postgres:5432 + DATABASE_URL_TEST: postgres://postgres:postgres@postgres:5432 + WEBPACKER_DEV_SERVER_HOST: webpacker + MALLOC_ARENA_MAX: 2 + WEB_CONCURRENCY: ${WEB_CONCURRENCY:-1} + BOOTSNAP_CACHE_DIR: /usr/local/bundle/_bootsnap + XDG_DATA_HOME: /app/tmp/cache + YARN_CACHE_FOLDER: /app/node_modules/.yarn-cache + HISTFILE: /usr/local/hist/.bash_history + PSQL_HISTFILE: /usr/local/hist/.psql_history + IRB_HISTFILE: /usr/local/hist/.irb_history + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + services: rails: - image: quay.io/forem/forem:development - container_name: forem_rails - user: root + <<: *backend + command: bundle exec rails + + web: + <<: *backend + command: bundle exec rails server -b 0.0.0.0 ports: - - "3000:3000" + - '3000:3000' depends_on: - - bundle - - db - - redis - - yarn - - testdb - environment: - RAILS_ENV: development - DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development - DATABASE_URL_TEST: postgresql://forem:forem@testdb:5432/Forem_test - REDIS_SESSIONS_URL: redis://redis:6379 - REDIS_SIDEKIQ_URL: redis://redis:6379 - REDIS_URL: redis://redis:6379 - RACK_TIMEOUT_WAIT_TIMEOUT: 10000 - RACK_TIMEOUT_SERVICE_TIMEOUT: 10000 - STATEMENT_TIMEOUT: 10000 - APP_DOMAIN: localhost:3000 - tmpfs: - - /tmp - volumes: - - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "tcp://redis:6379", "-wait", "file:///opt/apps/forem/vendor/bundle/.bundle_finished", "-timeout", "2700s", "-wait-retry-interval", "10s"] - command: [ "bash", "-c", "./scripts/entrypoint.sh bootstrap && bundle exec rails server -b 0.0.0.0 -p 3000"] - - bundle: - image: quay.io/forem/forem:development - container_name: forem_bundle - user: root - environment: - RAILS_ENV: development - REDIS_SESSIONS_URL: redis://redis:6379 - REDIS_SIDEKIQ_URL: redis://redis:6379 - REDIS_URL: redis://redis:6379 - DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development - volumes: - - .:/opt/apps/forem:delegated - command: ["./scripts/bundle.sh"] - - yarn: - image: quay.io/forem/forem:development - container_name: forem_yarn - user: root - environment: - RAILS_ENV: development - REDIS_SESSIONS_URL: redis://redis:6379 - REDIS_SIDEKIQ_URL: redis://redis:6379 - REDIS_URL: redis://redis:6379 - DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development - volumes: - - .:/opt/apps/forem:delegated - command: yarn install --dev - - webpacker: - image: quay.io/forem/forem:development - container_name: forem_webpacker - user: root - depends_on: - - bundle - - rails - - yarn - environment: - RAILS_ENV: development - REDIS_SESSIONS_URL: redis://redis:6379 - REDIS_SIDEKIQ_URL: redis://redis:6379 - REDIS_URL: redis://redis:6379 - DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development - volumes: - - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "file:///opt/apps/forem/node_modules/.bin/webpack-dev-server", "-wait", "file:///opt/apps/forem/vendor/bundle/.bundle_finished", "-timeout", "2700s", "-wait-retry-interval", "10s"] - command: ["./bin/webpack-dev-server"] - - seed: - image: quay.io/forem/forem:development - container_name: forem_seed - user: root - depends_on: - - rails - - redis - - db - environment: - RAILS_ENV: development - REDIS_SESSIONS_URL: redis://redis:6379 - REDIS_SIDEKIQ_URL: redis://redis:6379 - REDIS_URL: redis://redis:6379 - DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development - volumes: - - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "tcp://redis:6379", "-wait", "tcp://rails:3000", "-timeout", "2700s", "-wait-retry-interval", "20s"] - command: ["bundle", "exec", "rake","db:seed"] + webpacker: + condition: service_started + sidekiq: + condition: service_started sidekiq: - image: quay.io/forem/forem:development - container_name: forem_sidekiq - user: root - depends_on: - - rails - - redis - - db - environment: - RAILS_ENV: development - REDIS_SESSIONS_URL: redis://redis:6379 - REDIS_SIDEKIQ_URL: redis://redis:6379 - REDIS_URL: redis://redis:6379 - DATABASE_URL: postgresql://forem:forem@db:5432/PracticalDeveloper_development - volumes: - - .:/opt/apps/forem:delegated - entrypoint: ["dockerize", "-wait", "tcp://db:5432", "-wait", "tcp://redis:6379", "-wait", "tcp://rails:3000", "-timeout", "2700s", "-wait-retry-interval", "20s"] - command: ["bundle", "exec", "sidekiq","-c","2"] + <<: *backend + command: bundle exec sidekiq - db: - image: postgres:13-alpine - container_name: forem_postgresql + postgres: + image: postgres:13 + volumes: + - ./.dockerdev/.psqlrc:/root/.psqlrc:ro + - postgres:/var/lib/postgresql/data + - history:/usr/local/hist environment: - POSTGRES_USER: forem - POSTGRES_PASSWORD: forem - POSTGRES_DB: PracticalDeveloper_development + PSQL_HISTFILE: /usr/local/hist/.psql_history + POSTGRES_PASSWORD: postgres ports: - - "5432:5432" - volumes: - - db_data:/var/lib/postgresql/data:delegated - - testdb: - image: postgres:13-alpine - container_name: forem_postgresql_test - environment: - POSTGRES_USER: forem - POSTGRES_PASSWORD: forem - POSTGRES_DB: Forem_test - expose: - - "5432" - volumes: - - testdb_data:/var/lib/postgresql/data:delegated + - 5432 + healthcheck: + test: pg_isready -U postgres -h 127.0.0.1 + interval: 5s redis: - image: redis:6.0.9-alpine - container_name: forem_redis + image: redis:7.0-alpine + volumes: + - redis:/data ports: - - "6379:6379" + - 6379 + healthcheck: + test: redis-cli ping + interval: 1s + timeout: 3s + retries: 30 + + webpacker: + <<: *app + command: bundle exec ./bin/webpack-dev-server + ports: + - '3035:3035' + volumes: + - .:/app:cached + - bundle:/usr/local/bundle + - node_modules:/app/node_modules + - packs:/app/public/packs + - packs-test:/app/public/packs-test + environment: + <<: *env + WEBPACKER_DEV_SERVER_HOST: 0.0.0.0 + YARN_CACHE_FOLDER: /app/node_modules/.yarn-cache volumes: - db_data: - testdb_data: + bundle: + node_modules: + history: + rails_cache: + postgres: + redis: + assets: + packs: + packs-test: + diff --git a/jest.config.js b/jest.config.js index 1fd82d500..321f7d183 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,6 +3,7 @@ /* eslint-env node */ +process.env.NODE_ENV = 'test'; process.env.TZ = 'UTC'; module.exports = {