From e871a01c01cfe9fd0a734adb3e3b2a54ee2bd350 Mon Sep 17 00:00:00 2001 From: Molly Struve Date: Thu, 5 Dec 2019 13:19:01 -0600 Subject: [PATCH] Setup dual Redis store for rolling over fragment cacheing to Redis (#4991) [deploy] --- config/environments/development.rb | 6 +- config/environments/production.rb | 11 +- lib/active_support/cache/dual_rails_store.rb | 94 ++++++++++++++ .../cache/dual_rails_store_spec.rb | 120 ++++++++++++++++++ 4 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 lib/active_support/cache/dual_rails_store.rb create mode 100644 spec/lib/active_support/cache/dual_rails_store_spec.rb diff --git a/config/environments/development.rb b/config/environments/development.rb index 9768123ed..e68e56044 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -22,8 +22,12 @@ Rails.application.configure do # Enable/disable caching. By default caching is disabled. if Rails.root.join("tmp", "caching-dev.txt").exist? config.action_controller.perform_caching = true + REDIS_DEFAULT_EXPIRATION = 24.hours.to_i.freeze - config.cache_store = :memory_store + config.cache_store = :dual_rails_store, { + default_store: [ :memory_store ], + redis: [ :redis_store, url: ENV["REDIS_URL"], expires_in: REDIS_DEFAULT_EXPIRATION] + } config.public_file_server.headers = { "Cache-Control" => "public, max-age=#{2.days.to_i}" } diff --git a/config/environments/production.rb b/config/environments/production.rb index 1b5e9fcf5..af9ebfac3 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -99,13 +99,18 @@ Rails.application.configure do # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - config.cache_store = :dalli_store, - (ENV["MEMCACHIER_SERVERS"] || "").split(","), + REDIS_DEFAULT_EXPIRATION = 24.hours.to_i.freeze + config.cache_store = :dual_rails_store, { + default_store: [ :dalli_store, (ENV["MEMCACHIER_SERVERS"] || "").split(","), { username: ENV["MEMCACHIER_USERNAME"], password: ENV["MEMCACHIER_PASSWORD"], failover: true, socket_timeout: 1.5, - socket_failure_delay: 0.2 } + socket_failure_delay: 0.2, + expires_in: REDIS_DEFAULT_EXPIRATION, + dalli_store: true } ], + redis: [ :redis_store, url: ENV["REDIS_URL"], expires_in: REDIS_DEFAULT_EXPIRATION ] + } config.app_domain = ENV["APP_DOMAIN"] diff --git a/lib/active_support/cache/dual_rails_store.rb b/lib/active_support/cache/dual_rails_store.rb new file mode 100644 index 000000000..08b34597f --- /dev/null +++ b/lib/active_support/cache/dual_rails_store.rb @@ -0,0 +1,94 @@ +require "active_support/cache" +require "active_support/version" + +# This class is modeled off of https://github.com/mezis/level2 and is temporary +# +# Any cache statement as-is will use :default_store (Memcache) +# Any cache statement with the option only: :redis will only use Redis +# Any cache statement with the option all: true will write to Redis and Memcache BUT will read/return the value read from Memcache. + +module ActiveSupport + module Cache + class DualRailsStore < Store + attr_reader :stores + + def initialize(store_options) + @stores = store_options.each_with_object({}) do |(name, options), h| + h[name] = ActiveSupport::Cache.lookup_store(options) + end + @options = {} + end + + def cleanup(*args) + raise "Do not clear production caches" if Rails.env.production? + + @stores.each_value { |s| s.cleanup(*args) } + end + + def clear(*args) + raise "Do not clear production caches" if Rails.env.production? + + @stores.each_value { |s| s.clear(*args) } + end + + def read_multi(*names) + result = {} + @stores.each do |_name, store| + data = store.read_multi(*names) + result.merge! data + names -= data.keys + end + result + end + + protected + + def read_entry(key, options) + stores = selected_stores(options) + stores.each do |store| + entry = if store.options[:dalli_store] + store.instance_variable_get(:@data).get(key, options) + else + store.send :read_entry, key, options + end + + return entry if entry.present? + end + + nil + end + + def write_entry(key, entry, options) + stores = selected_stores(options) + stores.each do |store| + # Ensure default expiration gets set for keys without + write_options = options.reverse_merge(expires_in: store.options[:expires_in]) + + # Add connection to options hash for dalli_store + write_options = write_options.merge(connection: store.instance_variable_get(:@data)) if store.options[:dalli_store] + + result = store.send(:write_entry, key, entry, write_options) + return false unless result + end + + "OK" + end + + def delete_entry(key, options) + stores = selected_stores(options) + stores.map { |store| store.send(:delete_entry, key, options) }.all? + end + + private + + def selected_stores(options) + return @stores.values if options[:all] + + only = options[:only] || :default_store + raise "Store not found!" unless @stores[only] + + Array.wrap(@stores[only]) + end + end + end +end diff --git a/spec/lib/active_support/cache/dual_rails_store_spec.rb b/spec/lib/active_support/cache/dual_rails_store_spec.rb new file mode 100644 index 000000000..0e592bf3b --- /dev/null +++ b/spec/lib/active_support/cache/dual_rails_store_spec.rb @@ -0,0 +1,120 @@ +require "rails_helper" + +RSpec.describe ActiveSupport::Cache::DualRailsStore, type: :lib do + subject do + ActiveSupport::Cache.lookup_store :dual_rails_store, + default_store: [ + :memory_store, + { size: 10.megabytes }, + ], + second_store: [ + :redis_store, + { size: 5.megabytes, expires_in: 200 }, + ] + end + + let(:dual_store) { subject } + let(:default_store) { subject.stores.values.first } + let(:second_store) { subject.stores.values.last } + + describe "#initialized" do + it "does not raise an error" do + expect { dual_store }.not_to raise_error + end + + it "sets stores instance variable" do + expect(dual_store.stores.count).to eq(2) + end + end + + describe "cache behavior" do + it "can #write" do + expect(dual_store.write("foo", "bar")).to be_truthy + end + + it "can #read" do + dual_store.write("foo", "bar") + expect(dual_store.read("foo")).to eq("bar") + end + + it "can #delete" do + dual_store.write("foo", "bar") + dual_store.delete("foo") + expect(dual_store.read("foo")).to be_nil + end + + it "can #fetch" do + expect(dual_store.fetch("foo") { "bar" }).to eq("bar") + expect(dual_store.fetch("foo") { "qux" }).to eq("bar") + end + + it "honors expiration" do + dual_store.write("foo", "bar", expires_in: 0) + expect(dual_store.read("foo")).to be_nil + end + + it "honors default expiration" do + dual_store.write("foo", "bar", all: true) + ttl = second_store.data.ttl("foo") + expect(ttl).to be < 201 + expect(ttl).to be > 0 + end + + it "can #clear" do + dual_store.write("foo", "bar") + dual_store.clear + expect(dual_store.read("foo")).to be_nil + end + + it "raises an error for an unknown store" do + expect { dual_store.write("foo", "bar", only: :foo) }.to raise_error("Store not found!") + end + end + + describe "storage" do + it "writes to all stores with all option" do + dual_store.write("foo", "bar", all: true) + expect(default_store.read("foo")).to eq("bar") + expect(second_store.read("foo")).to eq("bar") + end + + it "reads from the default store" do + default_store.write("foo", "bar1") + second_store.write("foo", "bar2") + expect(dual_store.read("foo")).to eq("bar1") + end + + it "can read from the bottom store" do + second_store.write("foo", "bar2") + expect(dual_store.read("foo", only: :second_store)).to eq("bar2") + end + end + + describe ":only restrictions" do + it "only writes to the selected store" do + dual_store.write("foo", "bar", only: :second_store) + expect(default_store.read("foo")).to be_nil + expect(second_store.read("foo")).to eq("bar") + end + + it "only reads the selected store" do + default_store.write("foo", "bar1", only: :default_store) + second_store.write("foo", "bar2", only: :second_store) + expect(dual_store.read("foo", only: :default_store)).to eq("bar1") + expect(dual_store.read("foo", only: :second_store)).to eq("bar2") + end + end + + describe "#read_multi" do + before do + dual_store.write("a", "a1", only: :default_store) + dual_store.write("a", "a2", only: :second_store) + dual_store.write("b", "b1", only: :default_store) + dual_store.write("c", "c2", only: :second_store) + end + + it "mass-reads data from each store" do + expect(dual_store.read_multi("a", "b", "c", "d")).to eq("a" => "a1", "b" => "b1", "c" => "c2") + end + end +end