From 9d464476970176fa4bdcd448910c52f5d387d188 Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Fri, 27 May 2016 00:42:21 +0200 Subject: [PATCH 01/10] Create hybrid hash/redis storage --- .travis.yml | 19 ++-- lib/lit.rb | 3 + lib/lit/adapters/hybrid_storage.rb | 153 +++++++++++++++++++++++++++++ lib/lit/i18n_backend.rb | 8 +- lit.gemspec | 1 + test/dummy/config/database.yml | 4 +- test/support/clear_snapshots.rb | 7 ++ test/test_helper.rb | 1 + 8 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 lib/lit/adapters/hybrid_storage.rb create mode 100644 test/support/clear_snapshots.rb diff --git a/.travis.yml b/.travis.yml index 97f26977..f8bceee0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,22 @@ language: ruby sudo: false rvm: - - 2.0.0 - - 2.1.2 - - 2.2.5 - - 2.3.1 +# - 2.0.0 +# - 2.1.2 +# - 2.2.5 + - 2.3.0 gemfile: - - gemfiles/rails_4.gemfile - - gemfiles/rails_4.1.gemfile +# - gemfiles/rails_4.gemfile +# - gemfiles/rails_4.1.gemfile - gemfiles/rails_4.2.gemfile services: - redis-server env: - - LIT_STORAGE=hash - - LIT_STORAGE=redis + # - LIT_STORAGE=hash + # - LIT_STORAGE=redis + - LIT_STORAGE=hybrid before_script: - cp test/dummy/config/database.yml.travis test/dummy/config/database.yml - psql -c 'create database lit_test;' -U postgres - RAILS_ENV=test bundle exec rake db:migrate -script: "bundle exec rake test" +script: "bundle exec rake test" # TEST=test/unit/lit_behaviour_test.rb" diff --git a/lib/lit.rb b/lib/lit.rb index e10d6448..b79feea7 100644 --- a/lib/lit.rb +++ b/lib/lit.rb @@ -46,6 +46,9 @@ def self.get_key_value_engine when 'redis' require 'lit/adapters/redis_storage' return RedisStorage.new + when 'hybrid' + require 'lit/adapters/hybrid_storage' + return HybridStorage.new else require 'lit/adapters/hash_storage' return HashStorage.new diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb new file mode 100644 index 00000000..f49d26a1 --- /dev/null +++ b/lib/lit/adapters/hybrid_storage.rb @@ -0,0 +1,153 @@ +require 'redis' +require 'concurrent' + +module Lit + extend self + def redis + $redis = Redis.new(url: determine_redis_provider) unless $redis + $redis + end + + def determine_redis_provider + ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] + end + + def _hash + $_hash ||= ::Concurrent::Hash.new + end + + def reset_hash + $_hash = nil + end + + def hash_dirty? + # Hash is considered dirty if hash snapshot is older + # than Redis snapshot. + Lit.hash_snapshot < Lit.redis_snapshot + end + + def hash_snapshot + $_hash_snapshot ||= DateTime.new + end + + def hash_snapshot= (timestamp) + $_hash_snapshot = timestamp + end + + def redis_snapshot + timestamp = Lit.redis.get('lit:_snapshot') + if timestamp.nil? + timestamp = Time.current.to_s + Lit.redis_snapshot = timestamp + end + DateTime.parse(timestamp) + end + + def redis_snapshot= (timestamp) + Lit.redis.set('lit:_snapshot', timestamp) + end + + def determine_redis_provider + ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] + end + + class HybridStorage + def initialize + Lit.redis + Lit._hash + end + + def [](key) + if Lit.hash_dirty? + Lit.hash_snapshot = DateTime.current + Lit._hash.clear + end + if Lit._hash.key? key + return Lit._hash[key] + else + redis_val = get_from_redis(key) + Lit._hash[key] = redis_val + end + end + + def get_from_redis(key) + if Lit.redis.exists(_prefixed_key_for_array(key)) + Lit.redis.lrange(_prefixed_key(key), 0, -1) + elsif Lit.redis.exists(_prefixed_key_for_nil(key)) + nil + else + Lit.redis.get(_prefixed_key(key)) + end + end + + def []=(k, v) + delete(k) + Lit._hash[k] = v + if v.is_a?(Array) + Lit.redis.set(_prefixed_key_for_array(k), '1') + v.each do |ve| + Lit.redis.rpush(_prefixed_key(k), ve.to_s) + end + elsif v.nil? + Lit.redis.set(_prefixed_key_for_nil(k), '1') + Lit.redis.set(_prefixed_key(k), '') + else + Lit.redis.set(_prefixed_key(k), v) + end + end + + def delete(k) + Lit.redis_snapshot = Time.current + Lit._hash.delete(k) + Lit.redis.del(_prefixed_key_for_array(k)) + Lit.redis.del(_prefixed_key_for_nil(k)) + Lit.redis.del(_prefixed_key(k)) + end + + def clear + Lit.redis_snapshot = Time.current + Lit._hash.clear + Lit.redis.del(keys) if keys.length > 0 + end + + def keys + Lit.redis.keys(_prefixed_key + '*') + end + + def has_key?(key) + Lit._hash.has_key?(key) || Lit.redis.exists(_prefixed_key(key)) # This is a derp + end + + def incr(key) + Lit.redis.incr(_prefixed_key(key)) + end + + def sort + Lit.redis.keys.sort.map do |k| + [k, self.[](k)] + end + end + + private + + def _prefix + prefix = 'lit:' + if Lit.storage_options.is_a?(Hash) + prefix += "#{Lit.storage_options[:prefix]}:" if Lit.storage_options.key?(:prefix) + end + prefix + end + + def _prefixed_key(key = '') + _prefix + key.to_s + end + + def _prefixed_key_for_array(key = '') + _prefix + 'array_flags:' + key.to_s + end + + def _prefixed_key_for_nil(key = '') + _prefix + 'nil_flags:' + key.to_s + end + end +end diff --git a/lib/lit/i18n_backend.rb b/lib/lit/i18n_backend.rb index a6f48a43..641a4c67 100644 --- a/lib/lit/i18n_backend.rb +++ b/lib/lit/i18n_backend.rb @@ -59,6 +59,7 @@ def lookup(locale, key, scope = [], options = {}) return content if parts.size <= 1 if should_cache?(key_with_locale) + puts "SHOULD CACHE #{key_with_locale}" new_content = @cache.init_key_with_value(key_with_locale, content) content = new_content if content.nil? # Content can change when Lit.humanize is true for example @@ -141,12 +142,17 @@ def is_ignored_key(key_without_locale) end def should_cache?(key_with_locale) - return false if @cache.has_key?(key_with_locale) + return false if @cache[key_with_locale] != nil _, key_without_locale = ::Lit::Cache.split_key(key_with_locale) return false if is_ignored_key(key_without_locale) true end + + def extract_non_symbol_default!(options) + defaults = [options[:default]].flatten + defaults.detect{|default| !default.is_a?(Symbol)} + end end end diff --git a/lit.gemspec b/lit.gemspec index 6abde74a..0e2fcef9 100644 --- a/lit.gemspec +++ b/lit.gemspec @@ -20,6 +20,7 @@ Gem::Specification.new do |s| s.add_dependency 'rails', '> 3.1.0' s.add_dependency 'i18n', '~> 0.7.0' s.add_dependency 'jquery-rails' + s.add_dependency 'concurrent-ruby' s.add_development_dependency 'pg' s.add_development_dependency 'devise' diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 47455e7d..98a2ab84 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -15,8 +15,8 @@ development: test: adapter: postgresql database: lit_test - username: lit - password: lit + username: ebin + password: ebin host: localhost pool: 5 timeout: 5000 diff --git a/test/support/clear_snapshots.rb b/test/support/clear_snapshots.rb new file mode 100644 index 00000000..30d0f6cf --- /dev/null +++ b/test/support/clear_snapshots.rb @@ -0,0 +1,7 @@ +require 'lit/adapters/hybrid_storage' +def clear_snapshots + trace_var :$_hash, proc { |h| print 'hash is now', v } + Lit.reset_hash if defined?($_hash) + Lit.hash_snapshot = nil if defined?($_hash_snapshot) + Lit.redis.del('lit:_snapshot') if defined?($redis) +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 970c014e..47ce3c29 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -28,6 +28,7 @@ class ActiveSupport::TestCase self.use_transactional_fixtures = true setup do + clear_snapshots clear_redis Lit.init.cache.reset end From 29ff09cff8cba5fb9a9ba179fb435dac9f8b164d Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Fri, 27 May 2016 12:03:22 +0200 Subject: [PATCH 02/10] Do not overwrite UI-changed texts with ones from .yml --- lib/lit/cache.rb | 11 +++++++---- lib/lit/i18n_backend.rb | 18 ++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/lit/cache.rb b/lib/lit/cache.rb index aecb554b..622e25c8 100644 --- a/lib/lit/cache.rb +++ b/lib/lit/cache.rb @@ -49,11 +49,12 @@ def keys localizations.keys end - def update_locale(key, value, force_array = false) + def update_locale(key, value, force_array = false, unless_is_changed = false) key = key.to_s locale_key, key_without_locale = split_key(key) locale = find_locale(locale_key) localization = find_localization(locale, key_without_locale, value, force_array, true) + return localization.get_value if unless_is_changed && localization.is_changed? localizations[key] = localization.get_value if localization end @@ -62,11 +63,12 @@ def update_cache(key, value) localizations[key] = value end - def delete_locale(key) + def delete_locale(key, unless_is_changed = false) + binding.pry if key.match /abbr_month/ key = key.to_s locale_key, key_without_locale = split_key(key) locale = find_locale(locale_key) - delete_localization(locale, key_without_locale) + delete_localization(locale, key_without_locale, unless_is_changed) end def load_all_translations @@ -227,8 +229,9 @@ def find_localization_for_delete(locale, key_without_locale) where(localization_key_id: localization_key.id).first end - def delete_localization(locale, key_without_locale) + def delete_localization(locale, key_without_locale, unless_is_changed = false) localization = find_localization_for_delete(locale, key_without_locale) + return if unless_is_changed && localization.try(:is_changed?) if localization localizations.delete("#{locale.locale}.#{key_without_locale}") localization_keys.delete(key_without_locale) diff --git a/lib/lit/i18n_backend.rb b/lib/lit/i18n_backend.rb index 641a4c67..0701e716 100644 --- a/lib/lit/i18n_backend.rb +++ b/lib/lit/i18n_backend.rb @@ -59,7 +59,6 @@ def lookup(locale, key, scope = [], options = {}) return content if parts.size <= 1 if should_cache?(key_with_locale) - puts "SHOULD CACHE #{key_with_locale}" new_content = @cache.init_key_with_value(key_with_locale, content) content = new_content if content.nil? # Content can change when Lit.humanize is true for example @@ -84,26 +83,25 @@ def lookup(locale, key, scope = [], options = {}) content end - def store_item(locale, data, scope = []) + def store_item(locale, data, scope = [], unless_is_changed = false) if data.respond_to?(:to_hash) - # ActiveRecord::Base.transaction do - data.to_hash.each do |key, value| - store_item(locale, value, scope + [key]) - end - # end + data.to_hash.each do |key, value| + store_item(locale, value, scope + [key], unless_is_changed) + end elsif data.respond_to?(:to_str) key = ([locale] + scope).join('.') - @cache[key] ||= data + @cache.update_locale(key, data, false, unless_is_changed) elsif data.nil? key = ([locale] + scope).join('.') - @cache.delete_locale(key) + @cache.delete_locale(key, unless_is_changed) end end def load_translations_to_cache ActiveRecord::Base.transaction do (@translations || {}).each do |locale, data| - store_item(locale, data) if valid_locale?(locale) + # TODO maybe don't store if marked as changed in db? :) + store_item(locale, data, [], true) if valid_locale?(locale) end end end From 1a6cacbde51a8d108b03c5caac4bed350fab4179 Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Fri, 27 May 2016 12:17:46 +0200 Subject: [PATCH 03/10] Properly prefix redis snapshot key --- lib/lit/adapters/hybrid_storage.rb | 18 +++++++++++------- test/support/clear_snapshots.rb | 3 +-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb index f49d26a1..45153742 100644 --- a/lib/lit/adapters/hybrid_storage.rb +++ b/lib/lit/adapters/hybrid_storage.rb @@ -35,7 +35,7 @@ def hash_snapshot= (timestamp) end def redis_snapshot - timestamp = Lit.redis.get('lit:_snapshot') + timestamp = Lit.redis.get(Lit.prefix + '_snapshot') if timestamp.nil? timestamp = Time.current.to_s Lit.redis_snapshot = timestamp @@ -44,13 +44,21 @@ def redis_snapshot end def redis_snapshot= (timestamp) - Lit.redis.set('lit:_snapshot', timestamp) + Lit.redis.set(Lit.prefix + '_snapshot', timestamp) end def determine_redis_provider ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] end + def prefix + pfx = 'lit:' + if Lit.storage_options.is_a?(Hash) + pfx += "#{Lit.storage_options[:prefix]}:" if Lit.storage_options.key?(:prefix) + end + pfx + end + class HybridStorage def initialize Lit.redis @@ -131,11 +139,7 @@ def sort private def _prefix - prefix = 'lit:' - if Lit.storage_options.is_a?(Hash) - prefix += "#{Lit.storage_options[:prefix]}:" if Lit.storage_options.key?(:prefix) - end - prefix + Lit.prefix end def _prefixed_key(key = '') diff --git a/test/support/clear_snapshots.rb b/test/support/clear_snapshots.rb index 30d0f6cf..26428371 100644 --- a/test/support/clear_snapshots.rb +++ b/test/support/clear_snapshots.rb @@ -1,7 +1,6 @@ require 'lit/adapters/hybrid_storage' def clear_snapshots - trace_var :$_hash, proc { |h| print 'hash is now', v } Lit.reset_hash if defined?($_hash) Lit.hash_snapshot = nil if defined?($_hash_snapshot) - Lit.redis.del('lit:_snapshot') if defined?($redis) + Lit.redis.del(Lit.prefix + '_snapshot') if defined?($redis) end From e4edfe77cab89575f6b51840109929b9f54fad31 Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Fri, 27 May 2016 13:02:30 +0200 Subject: [PATCH 04/10] Update travis.yml for hybrid storage --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8bceee0..8adde4d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,22 @@ language: ruby sudo: false rvm: -# - 2.0.0 -# - 2.1.2 -# - 2.2.5 - - 2.3.0 + - 2.0.0 + - 2.1.2 + - 2.2.5 + - 2.3.0 gemfile: -# - gemfiles/rails_4.gemfile -# - gemfiles/rails_4.1.gemfile + - gemfiles/rails_4.gemfile + - gemfiles/rails_4.1.gemfile - gemfiles/rails_4.2.gemfile services: - redis-server env: - # - LIT_STORAGE=hash - # - LIT_STORAGE=redis + - LIT_STORAGE=hash + - LIT_STORAGE=redis - LIT_STORAGE=hybrid before_script: - cp test/dummy/config/database.yml.travis test/dummy/config/database.yml - psql -c 'create database lit_test;' -U postgres - RAILS_ENV=test bundle exec rake db:migrate -script: "bundle exec rake test" # TEST=test/unit/lit_behaviour_test.rb" +script: "bundle exec rake test" From a8d82d7dde803f642c453cd46ec23f057597f77b Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Mon, 30 May 2016 09:53:58 +0200 Subject: [PATCH 05/10] Ensure consistency of time types --- lib/lit/adapters/hybrid_storage.rb | 6 +++--- lib/lit/i18n_backend.rb | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb index 45153742..61557832 100644 --- a/lib/lit/adapters/hybrid_storage.rb +++ b/lib/lit/adapters/hybrid_storage.rb @@ -37,7 +37,7 @@ def hash_snapshot= (timestamp) def redis_snapshot timestamp = Lit.redis.get(Lit.prefix + '_snapshot') if timestamp.nil? - timestamp = Time.current.to_s + timestamp = DateTime.now.to_s Lit.redis_snapshot = timestamp end DateTime.parse(timestamp) @@ -105,7 +105,7 @@ def []=(k, v) end def delete(k) - Lit.redis_snapshot = Time.current + Lit.redis_snapshot = DateTime.now Lit._hash.delete(k) Lit.redis.del(_prefixed_key_for_array(k)) Lit.redis.del(_prefixed_key_for_nil(k)) @@ -113,7 +113,7 @@ def delete(k) end def clear - Lit.redis_snapshot = Time.current + Lit.redis_snapshot = DateTime.now Lit._hash.clear Lit.redis.del(keys) if keys.length > 0 end diff --git a/lib/lit/i18n_backend.rb b/lib/lit/i18n_backend.rb index 0701e716..2d79b6b3 100644 --- a/lib/lit/i18n_backend.rb +++ b/lib/lit/i18n_backend.rb @@ -100,7 +100,6 @@ def store_item(locale, data, scope = [], unless_is_changed = false) def load_translations_to_cache ActiveRecord::Base.transaction do (@translations || {}).each do |locale, data| - # TODO maybe don't store if marked as changed in db? :) store_item(locale, data, [], true) if valid_locale?(locale) end end From 6e01a99b58640ee541593c8066516cf56074bc75 Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Mon, 30 May 2016 10:30:14 +0200 Subject: [PATCH 06/10] Improve timestamp precision --- lib/lit/adapters/hybrid_storage.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb index 61557832..0b6fa952 100644 --- a/lib/lit/adapters/hybrid_storage.rb +++ b/lib/lit/adapters/hybrid_storage.rb @@ -27,7 +27,7 @@ def hash_dirty? end def hash_snapshot - $_hash_snapshot ||= DateTime.new + $_hash_snapshot ||= DateTime.new.to_f.to_d end def hash_snapshot= (timestamp) @@ -37,16 +37,20 @@ def hash_snapshot= (timestamp) def redis_snapshot timestamp = Lit.redis.get(Lit.prefix + '_snapshot') if timestamp.nil? - timestamp = DateTime.now.to_s + timestamp = DateTime.now.to_f.to_s Lit.redis_snapshot = timestamp end - DateTime.parse(timestamp) + DateTime.strptime(timestamp, '%s').to_f.to_d end def redis_snapshot= (timestamp) Lit.redis.set(Lit.prefix + '_snapshot', timestamp) end + def now_timestamp + DateTime.now.to_f.to_d + end + def determine_redis_provider ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] end @@ -67,7 +71,7 @@ def initialize def [](key) if Lit.hash_dirty? - Lit.hash_snapshot = DateTime.current + Lit.hash_snapshot = Lit.now_timestamp Lit._hash.clear end if Lit._hash.key? key @@ -105,7 +109,7 @@ def []=(k, v) end def delete(k) - Lit.redis_snapshot = DateTime.now + Lit.redis_snapshot = Lit.now_timestamp Lit._hash.delete(k) Lit.redis.del(_prefixed_key_for_array(k)) Lit.redis.del(_prefixed_key_for_nil(k)) @@ -113,7 +117,7 @@ def delete(k) end def clear - Lit.redis_snapshot = DateTime.now + Lit.redis_snapshot = Lit.now_timestamp Lit._hash.clear Lit.redis.del(keys) if keys.length > 0 end From 3d6d814764d4e98d1bdbbc83219e3e3961db8e41 Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Mon, 30 May 2016 14:40:51 +0200 Subject: [PATCH 07/10] Preserve redis snapshot until end of request --- lib/lit/adapters/hybrid_storage.rb | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb index 0b6fa952..3fafd3c3 100644 --- a/lib/lit/adapters/hybrid_storage.rb +++ b/lib/lit/adapters/hybrid_storage.rb @@ -1,6 +1,14 @@ require 'redis' require 'concurrent' +ActionController::Base.class_eval do + after_action :clear_saved_redis_snapshot + + def clear_saved_redis_snapshot + Lit.saved_redis_snapshot = nil + end +end + module Lit extend self def redis @@ -35,18 +43,29 @@ def hash_snapshot= (timestamp) end def redis_snapshot + return Lit.saved_redis_snapshot unless Lit.saved_redis_snapshot.nil? timestamp = Lit.redis.get(Lit.prefix + '_snapshot') if timestamp.nil? timestamp = DateTime.now.to_f.to_s Lit.redis_snapshot = timestamp end - DateTime.strptime(timestamp, '%s').to_f.to_d + Lit.saved_redis_snapshot = DateTime.strptime(timestamp, '%s').to_f.to_d end def redis_snapshot= (timestamp) Lit.redis.set(Lit.prefix + '_snapshot', timestamp) end + def saved_redis_snapshot + $saved_redis_snapshot ||= Concurrent::MVar.new(nil) + $saved_redis_snapshot.value + end + + def saved_redis_snapshot= (snap) + $saved_redis_snapshot ||= Concurrent::MVar.new(nil) + $saved_redis_snapshot.set!(snap) + end + def now_timestamp DateTime.now.to_f.to_d end From bf3dc997ee5a7784b185c58baf868f2b1ee4ee1a Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Tue, 31 May 2016 17:19:10 +0200 Subject: [PATCH 08/10] Revert to previously used item preloading --- lib/lit/cache.rb | 11 ++++------- lib/lit/i18n_backend.rb | 10 +++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/lit/cache.rb b/lib/lit/cache.rb index 622e25c8..aecb554b 100644 --- a/lib/lit/cache.rb +++ b/lib/lit/cache.rb @@ -49,12 +49,11 @@ def keys localizations.keys end - def update_locale(key, value, force_array = false, unless_is_changed = false) + def update_locale(key, value, force_array = false) key = key.to_s locale_key, key_without_locale = split_key(key) locale = find_locale(locale_key) localization = find_localization(locale, key_without_locale, value, force_array, true) - return localization.get_value if unless_is_changed && localization.is_changed? localizations[key] = localization.get_value if localization end @@ -63,12 +62,11 @@ def update_cache(key, value) localizations[key] = value end - def delete_locale(key, unless_is_changed = false) - binding.pry if key.match /abbr_month/ + def delete_locale(key) key = key.to_s locale_key, key_without_locale = split_key(key) locale = find_locale(locale_key) - delete_localization(locale, key_without_locale, unless_is_changed) + delete_localization(locale, key_without_locale) end def load_all_translations @@ -229,9 +227,8 @@ def find_localization_for_delete(locale, key_without_locale) where(localization_key_id: localization_key.id).first end - def delete_localization(locale, key_without_locale, unless_is_changed = false) + def delete_localization(locale, key_without_locale) localization = find_localization_for_delete(locale, key_without_locale) - return if unless_is_changed && localization.try(:is_changed?) if localization localizations.delete("#{locale.locale}.#{key_without_locale}") localization_keys.delete(key_without_locale) diff --git a/lib/lit/i18n_backend.rb b/lib/lit/i18n_backend.rb index 2d79b6b3..ad1a09c6 100644 --- a/lib/lit/i18n_backend.rb +++ b/lib/lit/i18n_backend.rb @@ -83,24 +83,24 @@ def lookup(locale, key, scope = [], options = {}) content end - def store_item(locale, data, scope = [], unless_is_changed = false) + def store_item(locale, data, scope = []) if data.respond_to?(:to_hash) data.to_hash.each do |key, value| - store_item(locale, value, scope + [key], unless_is_changed) + store_item(locale, value, scope + [key]) end elsif data.respond_to?(:to_str) key = ([locale] + scope).join('.') - @cache.update_locale(key, data, false, unless_is_changed) + @cache[key] ||= data elsif data.nil? key = ([locale] + scope).join('.') - @cache.delete_locale(key, unless_is_changed) + @cache.delete_locale(key) end end def load_translations_to_cache ActiveRecord::Base.transaction do (@translations || {}).each do |locale, data| - store_item(locale, data, [], true) if valid_locale?(locale) + store_item(locale, data) if valid_locale?(locale) end end end From 97dbf5f4b90f0fd203b2e50f7e27fb6750c7e0f7 Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Fri, 22 Jul 2016 14:05:22 +0200 Subject: [PATCH 09/10] Add tests for hybrid storage --- lib/lit/adapters/hybrid_storage.rb | 2 +- test/unit/hybrid_storage_test.rb | 89 ++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 test/unit/hybrid_storage_test.rb diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb index 3fafd3c3..c2a4b90f 100644 --- a/lib/lit/adapters/hybrid_storage.rb +++ b/lib/lit/adapters/hybrid_storage.rb @@ -49,7 +49,7 @@ def redis_snapshot timestamp = DateTime.now.to_f.to_s Lit.redis_snapshot = timestamp end - Lit.saved_redis_snapshot = DateTime.strptime(timestamp, '%s').to_f.to_d + Lit.saved_redis_snapshot = timestamp.to_f end def redis_snapshot= (timestamp) diff --git a/test/unit/hybrid_storage_test.rb b/test/unit/hybrid_storage_test.rb new file mode 100644 index 00000000..1b17f298 --- /dev/null +++ b/test/unit/hybrid_storage_test.rb @@ -0,0 +1,89 @@ +require 'test_helper' +require 'pry' + +# Applicable only for LIT_STORAGE=hybrid +class HybridStorageTest < ActiveSupport::TestCase + if ENV['LIT_STORAGE'] == 'hybrid' + class Backend < Lit::I18nBackend + end + + fixtures :all + + def setup + Lit.init + Lit::Localization.delete_all + Lit::LocalizationKey.delete_all + Lit::LocalizationVersion.delete_all + @old_humanize_key = Lit.humanize_key + Lit.humanize_key = false + @old_load_path = I18n.load_path + Lit.reset_hash + I18n.backend.cache.clear + @locale = Lit::Locale.find_by_locale(I18n.locale) + super + end + + def teardown + Lit.loader = @old_loader + Lit.humanize_key = @old_humanize_key + I18n.backend = @old_backend + I18n.load_path = @old_load_path + super + end + + test 'it should update translation both in hash and in redis' do + # assertions to ensure that storage has been properly cleared + assert_nil Lit._hash['en.fizz'] + assert_nil Lit.redis.get(Lit.prefix + 'en.fizz') + I18n.t('fizz', default: 'buzz') + assert_equal 'buzz', Lit._hash['en.fizz'] + assert_equal 'buzz', Lit.redis.get(Lit.prefix + 'en.fizz') + end + + test 'it should clear hash when loading from redis something not yet in hash' do + # let's do something that creates a hash snapshot timestamp + assert_nil Lit._hash['en.fizz'] + old_hash_snapshot = Lit.hash_snapshot + I18n.t('fizz', default: 'buzz') + assert_operator Lit.hash_snapshot, :>, old_hash_snapshot + + # in the meantime let's create some new translation + # simulate as if it were created and redis snapshot has been updated + lk = Lit::LocalizationKey.create(localization_key: 'abcd') + l = lk.localizations.create!(locale: @locale, default_value: 'efgh') + + Lit.redis.set(Lit.prefix + 'en.abcd', 'efgh') + Lit.saved_redis_snapshot = Lit.now_timestamp + Lit.redis_snapshot = Lit.saved_redis_snapshot + # TODO: consider if this is not too implementation-specific + + # assert that the newly created localization has been fetched into hash + assert_equal 'efgh', I18n.t('abcd') + assert_equal 'efgh', Lit._hash['en.abcd'] + assert_equal 'efgh', Lit.redis.get(Lit.prefix + 'en.abcd') + + # assert that hash cache has been cleared + assert_nil Lit._hash['en.fizz'] + I18n.t('fizz') + + # assert that the value then gets loaded into hash again + assert_equal 'buzz', Lit._hash['en.fizz'] + end + + test 'local cache is used even when redis is cleared' do + # define a translation by specifying default value + assert_nil Lit._hash['en.fizz'] + I18n.t('fizz', default: 'buzz') + assert_equal 'buzz', Lit._hash['en.fizz'] + + # clear redis + I18n.backend.cache.clear + + # modify local cache and then see if it's used for loading translation + Lit._hash['en.fizz'] = 'fizzbuzz' + assert_equal 'fizzbuzz', I18n.t('fizz') + end + else + puts 'Skipping hybrid storage test' + end +end From f8f4da25186e3b2cb9ba87e1f2469cdd7bc1bdfa Mon Sep 17 00:00:00 2001 From: Michal Buszkiewicz Date: Fri, 22 Jul 2016 16:04:56 +0200 Subject: [PATCH 10/10] Cleanup --- test/unit/hybrid_storage_test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/hybrid_storage_test.rb b/test/unit/hybrid_storage_test.rb index 1b17f298..3077d3b0 100644 --- a/test/unit/hybrid_storage_test.rb +++ b/test/unit/hybrid_storage_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'pry' # Applicable only for LIT_STORAGE=hybrid class HybridStorageTest < ActiveSupport::TestCase