From 0409722899987ff6fbe043cfae8e22ec6e8e9b9b Mon Sep 17 00:00:00 2001 From: Molly Struve Date: Mon, 3 Feb 2020 09:12:05 -0500 Subject: [PATCH] Introduce Elasticsearch to DEV env (#5762) [deploy] --- Envfile | 3 + Gemfile | 2 + Gemfile.lock | 14 +++++ config/initializers/elasticsearch.rb | 9 +++ docs/installation/linux.md | 65 +++++++++++++--------- docs/installation/mac.md | 82 +++++++++++++++++++++------- docs/installation/windows.md | 47 +++++++++++----- 7 files changed, 160 insertions(+), 62 deletions(-) create mode 100644 config/initializers/elasticsearch.rb diff --git a/Envfile b/Envfile index 0dac44ec7..f90cf3d77 100644 --- a/Envfile +++ b/Envfile @@ -30,6 +30,9 @@ variable :SESSION_EXPIRY_SECONDS, :Integer, default: 1209600 # two weeks in seco # Redis Sidekiq storage variable :REDIS_SIDEKIQ_URL, :String, default: "redis://localhost:6379" +# Elasticsearch +variable :ELASTICSEARCH_URL, :String, default: "http://localhost:9200" + ################################################ ############## 3rd Party Services ############## ################################################ diff --git a/Gemfile b/Gemfile index 32ac01f9d..08d0c7c26 100644 --- a/Gemfile +++ b/Gemfile @@ -36,6 +36,7 @@ gem "dogstatsd-ruby", "~> 4.6" # A client for DogStatsD, an extension of the Sta gem "doorkeeper", "~> 5.2" # Oauth 2 provider gem "draper", "~> 3.1" # Draper adds an object-oriented layer of presentation logic to your Rails apps gem "dry-struct", "~> 1.2" # Typed structs and value objects +gem "elasticsearch", "~> 7.4" # Powers DEVs core search functionality gem "email_validator", "~> 2.0" # Email validator for Rails and ActiveModel gem "emoji_regex", "~> 2.0" # A pair of Ruby regular expressions for matching Unicode Emoji symbols gem "envied", "~> 0.9" # Ensure presence and type of your app's ENV-variables @@ -97,6 +98,7 @@ gem "timber", "~> 3.0" # Great Ruby logging made easy gem "timber-rails", "~> 1.0" # Timber integration for Rails gem "twilio-ruby", "~> 5.31" # The official library for communicating with the Twilio REST API gem "twitter", "~> 6.2" # A Ruby interface to the Twitter API +gem "typhoeus", "~> 1.3.1" # Used with Elasticsearch to support http keep-alive connections gem "uglifier", "~> 4.2" # Uglifier minifies JavaScript files gem "ulid", "~> 1.1" # Universally Unique Lexicographically Sortable Identifier implementation for Ruby gem "validate_url", "~> 1.0" # Library for validating urls in Rails diff --git a/Gemfile.lock b/Gemfile.lock index 6c2fb99a7..bd50b7c52 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -283,6 +283,14 @@ GEM dry-equalizer (~> 0.3) dry-inflector (~> 0.1, >= 0.1.2) dry-logic (~> 1.0, >= 1.0.2) + elasticsearch (7.4.0) + elasticsearch-api (= 7.4.0) + elasticsearch-transport (= 7.4.0) + elasticsearch-api (7.4.0) + multi_json + elasticsearch-transport (7.4.0) + faraday + multi_json em-websocket (0.5.1) eventmachine (>= 0.12.9) http_parser.rb (~> 0.6.0) @@ -302,6 +310,8 @@ GEM smart_properties errbase (0.1.1) erubi (1.9.0) + ethon (0.12.0) + ffi (>= 1.3.0) eventmachine (1.2.5) excon (0.71.0) execjs (2.7.0) @@ -808,6 +818,8 @@ GEM multipart-post (~> 2.0) naught (~> 1.0) simple_oauth (~> 0.3.0) + typhoeus (1.3.1) + ethon (>= 0.9.0) tzinfo (1.2.6) thread_safe (~> 0.1) uber (0.1.0) @@ -900,6 +912,7 @@ DEPENDENCIES doorkeeper (~> 5.2) draper (~> 3.1) dry-struct (~> 1.2) + elasticsearch (~> 7.4) email_validator (~> 2.0) emoji_regex (~> 2.0) envied (~> 0.9) @@ -996,6 +1009,7 @@ DEPENDENCIES timecop (~> 0.9) twilio-ruby (~> 5.31) twitter (~> 6.2) + typhoeus (~> 1.3.1) uglifier (~> 4.2) ulid (~> 1.1) validate_url (~> 1.0) diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb new file mode 100644 index 000000000..272e80b82 --- /dev/null +++ b/config/initializers/elasticsearch.rb @@ -0,0 +1,9 @@ +require "elasticsearch" + +SearchClient = Elasticsearch::Client.new( + url: ApplicationConfig["ELASTICSEARCH_URL"], + retry_on_failure: 5, + request_timeout: 30, + adapter: :typhoeus, + log: Rails.env.development?, +) diff --git a/docs/installation/linux.md b/docs/installation/linux.md index f0c2bd77e..019777666 100644 --- a/docs/installation/linux.md +++ b/docs/installation/linux.md @@ -12,35 +12,36 @@ but they have only been tested on Ubuntu 18.04._ ### Ruby 1. If you don't already have a Ruby version manager, we highly recommend - [rbenv](https://github.com/rbenv/rbenv). Please follow their [installation - guide](https://github.com/rbenv/rbenv#installation). + [rbenv](https://github.com/rbenv/rbenv). Please follow their + [installation guide](https://github.com/rbenv/rbenv#installation). 1. With the Ruby version manager, install the Ruby version listed on our badge. (ie with rbenv: `rbenv install 2.6.5`) For very detailed rbenv installation directions on several distros, please visit -[DigitalOcean's -guide](https://www.digitalocean.com/community/tutorials/how-to-install-ruby-on-rails-with-rbenv-on-ubuntu-18-04). +[DigitalOcean's guide](https://www.digitalocean.com/community/tutorials/how-to-install-ruby-on-rails-with-rbenv-on-ubuntu-18-04). ### Yarn There are two ways to install Yarn. -- Yarn's official [installation - guide](https://yarnpkg.com/en/docs/install#debian-stable) (recommended). -- [DigitalOcean's detailed - tutorial](https://www.digitalocean.com/community/tutorials/how-to-install-node-js-on-ubuntu-18-04) - describes how to install [Node version - Manager](https://github.com/creationix/nvm). By installing NVM you can select - a Node version (we recommend either LTS or current); the guide will also - explain how to install NPM. This way you'll have Node, NPM, and then you can - run `npm install -g yarn` to install Yarn. +- Yarn's official + [installation guide](https://yarnpkg.com/en/docs/install#debian-stable) + (recommended). +- [DigitalOcean's detailed tutorial](https://www.digitalocean.com/community/tutorials/how-to-install-node-js-on-ubuntu-18-04) + describes how to install + [Node version Manager](https://github.com/creationix/nvm). By installing NVM + you can select a Node version (we recommend either LTS or current); the guide + will also explain how to install NPM. This way you'll have Node, NPM, and then + you can run `npm install -g yarn` to install Yarn. ### PostgreSQL -1. Run `sudo apt update && sudo apt install postgresql postgresql-contrib libpq-dev`. +1. Run + `sudo apt update && sudo apt install postgresql postgresql-contrib libpq-dev`. 1. To test the installation you can run `sudo -u postgres psql` which should - open a PostgreSQL prompt. Exit the prompt by running `\q` then run `sudo -u postgres createuser -s $YOUR_USERNAME` where `$YOUR_USERNAME` is the username - you are currently logged in as. + open a PostgreSQL prompt. Exit the prompt by running `\q` then run + `sudo -u postgres createuser -s $YOUR_USERNAME` where `$YOUR_USERNAME` is the + username you are currently logged in as. There are more than one ways to setup PostgreSQL. For additional configuration, check out our [PostgreSQL setup guide](/installation/postgresql) or the official @@ -58,15 +59,23 @@ it. DEV requires Redis version 4.0 or higher. -We recommend following Digital Ocean's extensive [How To Install and Configure -Redis on Ubuntu -16.04](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-redis-on-ubuntu-16-04) +We recommend following Digital Ocean's extensive +[How To Install and Configure Redis on Ubuntu 16.04](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-redis-on-ubuntu-16-04) (available for other Linux distributions as well) to setup Redis. +### Elasticsearch + +DEV requires Elasticsearch version 7 or higher. + +We recommend following +[Elasticsearch's guide for installing on Linux](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/targz.html#install-linux). +NOTE: Make sure to download the OSS version, `elasticsearch-oss`. + ## Installing DEV 1. Fork DEV's repository, e.g. -1. Clone your forked repository, e.g. `git clone https://github.com//dev.to.git` +1. Clone your forked repository, e.g. + `git clone https://github.com//dev.to.git` 1. Install bundler with `gem install bundler` 1. Set up your environment variables/secrets @@ -79,8 +88,9 @@ Redis on Ubuntu - For any key that you wish to enter/replace: 1. Create `config/application.yml` by copying from the provided template - (i.e. with bash: `cp config/sample_application.yml config/application.yml`). This is a personal file that is ignored in - git. + (i.e. with bash: + `cp config/sample_application.yml config/application.yml`). This is a + personal file that is ignored in git. 1. Obtain the development variable and apply the key you wish to enter/replace. i.e.: @@ -91,7 +101,8 @@ Redis on Ubuntu - If you are missing `ENV` variables on bootup, the [envied](https://rubygems.org/gems/envied) gem will alert you with messages - similar to `'error_on_missing_variables!': The following environment variables should be set: A_MISSING_KEY.`. + similar to + `'error_on_missing_variables!': The following environment variables should be set: A_MISSING_KEY.`. - You do not need "real" keys for basic development. Some features require certain keys, so you may be able to add them as you go. @@ -104,9 +115,11 @@ PostgreSQL libraries. If so, please run `sudo apt-get install libpq-dev` before retrying. While installing, you might run into an error due to the `sass-rails` gem -requiring `sassc`, which requires the `g++` compiler. If so, please run `sudo apt-get install g++` before retrying. +requiring `sassc`, which requires the `g++` compiler. If so, please run +`sudo apt-get install g++` before retrying. While installing, if you didn't install `node` or `nvm` manually, you might run into an error due to an older system node version being present, which can cause -issues while `yarn` is installing packages. If so, you'll need to [install -`nvm`](https://github.com/nvm-sh/nvm#installation-and-update) and then run `nvm install node` to get the most recent node version before retrying. +issues while `yarn` is installing packages. If so, you'll need to +[install `nvm`](https://github.com/nvm-sh/nvm#installation-and-update) and then +run `nvm install node` to get the most recent node version before retrying. diff --git a/docs/installation/mac.md b/docs/installation/mac.md index d4437311c..f360038e7 100644 --- a/docs/installation/mac.md +++ b/docs/installation/mac.md @@ -9,8 +9,8 @@ title: macOS ### Ruby 1. If you don't already have a Ruby version manager, we highly recommend - [rbenv](https://github.com/rbenv/rbenv). Please follow their [installation - guide](https://github.com/rbenv/rbenv#installation). + [rbenv](https://github.com/rbenv/rbenv). Please follow their + [installation guide](https://github.com/rbenv/rbenv#installation). 2. With the Ruby version manager, install the Ruby version listed on our badge. (i.e. with rbenv: `rbenv install 2.6.5`) @@ -25,8 +25,8 @@ to use [Postgres.app](https://postgresapp.com/). Alternatively, check out the official [PostgreSQL](https://www.postgresql.org/) site for more installation options. -For additional configuration options, check our [PostgreSQL setup -guide](/installation/postgresql). +For additional configuration options, check our +[PostgreSQL setup guide](/installation/postgresql). ### ImageMagick @@ -44,7 +44,8 @@ We recommend using [Homebrew](https://brew.sh): brew install redis ``` -you can follow the post installation instructions, we recommend using `brew services` to start Redis in the background: +you can follow the post installation instructions, we recommend using +`brew services` to start Redis in the background: ```shell brew services start redis @@ -56,10 +57,40 @@ You can test if it's up and running by issuing the following command: redis-cli ping ``` +### Elasticsearch + +DEV requires Elasticsearch version 7 or higher. + +We recommend installing from archive on Mac. The following directions were +[taken from the Elasticsearch docs themselves](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/targz.html#install-macos), +so check those out if you run into any issues or want further information. NOTE: +Make sure to download the OSS version, `elasticsearch-oss`. + +```shell +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.5.2-darwin-x86_64.tar.gz +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.5.2-darwin-x86_64.tar.gz +shasum -a 512 -c elasticsearch-oss-7.5.2-darwin-x86_64.tar.gz.sha512 +tar -xzf elasticsearch-oss-7.5.2-darwin-x86_64.tar.gz +cd elasticsearch-7.5.2/ +``` + +To start elasticsearch: + +```shell +./bin/elasticsearch +``` + +To start elasticsearch as a daemonized process: + +```shell +./bin/elasticsearch -d +``` + ## Installing DEV 1. Fork DEV's repository, e.g. -1. Clone your forked repository, e.g. `git clone https://github.com//dev.to.git` +1. Clone your forked repository, e.g. + `git clone https://github.com//dev.to.git` 1. Install bundler with `gem install bundler` 1. Set up your environment variables/secrets @@ -68,12 +99,14 @@ redis-cli ping - The [backend guide](/backend) will show you how to get free API keys for additional services that may be required to run certain parts of the app. - For any key that you wish to enter/replace, follow the steps below. At a - minimum, you'll need to get your own free [Algolia - credentials](/backend/algolia) to get your development environment running. + minimum, you'll need to get your own free + [Algolia credentials](/backend/algolia) to get your development environment + running. 1. Create `config/application.yml` by copying from the provided template - (i.e. with bash: `cp config/sample_application.yml config/application.yml`). This is a personal file that is ignored in - git. + (i.e. with bash: + `cp config/sample_application.yml config/application.yml`). This is a + personal file that is ignored in git. 1. Obtain the development variable and apply the key you wish to enter/replace. i.e.: @@ -84,7 +117,8 @@ redis-cli ping - If you are missing `ENV` variables on bootup, the [envied](https://rubygems.org/gems/envied) gem will alert you with messages - similar to `'error_on_missing_variables!': The following environment variables should be set: A_MISSING_KEY.`. + similar to + `'error_on_missing_variables!': The following environment variables should be set: A_MISSING_KEY.`. - You do not need "real" keys for basic development. Some features require certain keys, so you may be able to add them as you go. @@ -92,7 +126,8 @@ redis-cli ping ### Possible error messages -**Error:** `__NSPlaceholderDate initialize] may have been in progress in another thread when fork() was called` +**Error:** +`__NSPlaceholderDate initialize] may have been in progress in another thread when fork() was called` **_Solution:_** Run the command `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` (or `set -x OBJC_DISABLE_INITIALIZE_FORK_SAFETY YES` in fish shell) @@ -101,12 +136,13 @@ redis-cli ping **Error:** `User does not have CONNECT privilege.` -**_Solution:_** Complete the steps outlined in the [PostgreSQL setup -guide](/installation/postgresql). +**_Solution:_** Complete the steps outlined in the +[PostgreSQL setup guide](/installation/postgresql). --- -**Error:** `rbenv: version '' is not installed (set by /Path/To/Local/Repository/.ruby-version)` +**Error:** +`rbenv: version '' is not installed (set by /Path/To/Local/Repository/.ruby-version)` **_Solution:_** Run the command `rbenv install ` @@ -122,8 +158,9 @@ See all available versions with `rbenv install --list`. If the version you need is missing, try upgrading ruby-build: ``` -**_Solution:_** Run the following to update `ruby-build`, `brew update && brew upgrade ruby-build`. After that, rerun `rbenv install ` and that -version will get installed. +**_Solution:_** Run the following to update `ruby-build`, +`brew update && brew upgrade ruby-build`. After that, rerun +`rbenv install ` and that version will get installed. --- @@ -138,9 +175,11 @@ rails aborted! LoadError: dlopen(/Users//.rbenv/versions/2.6.5/lib/ruby/2.6.0/x86_64-darwin18/readline.bundle, 9): Library not loaded: /usr/local/opt/readline/lib/libreadline..dylib ``` -**_Solution:_** Run `ln -s /usr/local/opt/readline/lib/libreadline.dylib /usr/local/opt/readline/lib/libreadline..dylib` from the -command line then run `bin/setup` again. You may have a different version of -libreadline, so replace `` with the version that errored. +**_Solution:_** Run +`ln -s /usr/local/opt/readline/lib/libreadline.dylib /usr/local/opt/readline/lib/libreadline..dylib` +from the command line then run `bin/setup` again. You may have a different +version of libreadline, so replace `` with the version that +errored. --- @@ -174,7 +213,8 @@ Can't find the 'libpq-fe.h header **_Solution:_** You may encounter this when installing PostgreSQL with the Postgres.app. Try restarting the app and reinitializing the database. If that -doesn't work, install PostgreSQL with Homebrew instead: `brew install postgresql` +doesn't work, install PostgreSQL with Homebrew instead: +`brew install postgresql` --- diff --git a/docs/installation/windows.md b/docs/installation/windows.md index 4e71766ea..da38a7ac5 100644 --- a/docs/installation/windows.md +++ b/docs/installation/windows.md @@ -17,13 +17,14 @@ triggered errors when installing on Windows, so using WSL allows you to work on the software and not having to fix gem incompatibilities. First, let's enable Windows Subsystem for Linux in your machine. You can do this -by opening `Control Panel`, going to `Programs`, and then clicking `Turn Windows Features On or Off`. Look for the `Windows Subsystem for Linux` option and check -the box next to it. Windows will ask for a reboot. +by opening `Control Panel`, going to `Programs`, and then clicking +`Turn Windows Features On or Off`. Look for the `Windows Subsystem for Linux` +option and check the box next to it. Windows will ask for a reboot. ![Enable WSL on Windows](/wsl-feature.png 'Enable WSL on Windows') -Once you've got this installed and after rebooting, [install Ubuntu 18.04 on -Windows](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q). +Once you've got this installed and after rebooting, +[install Ubuntu 18.04 on Windows](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q). On your first run, the system will ask for username and password. Take note of both since it will be used for `sudo` commands. @@ -120,8 +121,8 @@ Pay attention to the username and password you setup during installation of PostgreSQL as you will use this to configure your Rails applications to login to the database later. -For additional configuration options, check our [PostgreSQL setup -guide](/installation/postgresql). +For additional configuration options, check our +[PostgreSQL setup guide](/installation/postgresql). ### ImageMagick @@ -135,13 +136,24 @@ it. DEV requires Redis version 4.0 or higher. -We recommend to follow [this -guide](https://redislabs.com/blog/redis-on-windows-10/) to run Redis under WSL. +We recommend to follow +[this guide](https://redislabs.com/blog/redis-on-windows-10/) to run Redis under +WSL. + +### Elasticsearch + +DEV requires Elasticsearch version 7 or higher. + +We recommend following the install guide +[in Elasticsearch's docs](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/zip-windows.html) +for installing on Windows machines. NOTE: Make sure to download the OSS version, +`elasticsearch-oss`. ## Installing DEV 1. Fork DEV's repository, eg. -1. Clone your forked repository, eg. `git clone https://github.com//dev.to.git` +1. Clone your forked repository, eg. + `git clone https://github.com//dev.to.git` 1. Install bundler with `gem install bundler` 1. Set up your environment variables/secrets @@ -154,8 +166,9 @@ guide](https://redislabs.com/blog/redis-on-windows-10/) to run Redis under WSL. - For any key that you wish to enter/replace: 1. Create `config/application.yml` by copying from the provided template - (ie. with bash: `cp config/sample_application.yml config/application.yml`). This is a personal file that is ignored in - git. + (ie. with bash: + `cp config/sample_application.yml config/application.yml`). This is a + personal file that is ignored in git. 1. Obtain the development variable and apply the key you wish to enter/replace. ie: @@ -166,7 +179,8 @@ guide](https://redislabs.com/blog/redis-on-windows-10/) to run Redis under WSL. - If you are missing `ENV` variables on bootup, the [envied](https://rubygems.org/gems/envied) gem will alert you with messages - similar to `'error_on_missing_variables!': The following environment variables should be set: A_MISSING_KEY.`. + similar to + `'error_on_missing_variables!': The following environment variables should be set: A_MISSING_KEY.`. - You do not need "real" keys for basic development. Some features require certain keys, so you may be able to add them as you go. @@ -178,15 +192,18 @@ guide](https://redislabs.com/blog/redis-on-windows-10/) to run Redis under WSL. seeding the database for the first time. Please increase the value of `statement_timeout` to `9999999` in `config/database.yml`. -2. If the installation process failed with the following error `ERROR: Error installing pg`. Please consider installing the following package `libpq-dev` - : +2. If the installation process failed with the following error + `ERROR: Error installing pg`. Please consider installing the following + package `libpq-dev` : ```bash sudo apt-get install libpq-dev ``` 3. If the command `bin/setup` fails at installing `cld-0.8.0` with the warnings - `'aclocal-1.10' is missing on your system` and `'automake-1.10' is missing on your system`. Please install `automake-1.10` using the commands below. + `'aclocal-1.10' is missing on your system` and + `'automake-1.10' is missing on your system`. Please install `automake-1.10` + using the commands below. ```shell cd