feat: uploading key access log

lib/copy_tuner_client/client.rb

@@ -85,6 +85,14 @@ def deploy
       end
     end
 
+    def upload_key_acess_log(data)
+      connect(host) do |http|
+        response = http.post(uri('blurb_access_logs'), data.to_json, 'Content-Type' => 'application/json', 'User-Agent' => USER_AGENT)
+        check response
+        log 'Uploaded key access log'
+      end
+    end
+
     private
 
     attr_reader :host, :port, :api_key, :http_read_timeout,

lib/copy_tuner_client/configuration.rb

@@ -2,6 +2,7 @@
 require 'copy_tuner_client/i18n_backend'
 require 'copy_tuner_client/client'
 require 'copy_tuner_client/cache'
+require 'copy_tuner_client/key_access_log'
 require 'copy_tuner_client/process_guard'
 require 'copy_tuner_client/poller'
 require 'copy_tuner_client/prefixed_logger'
@@ -107,6 +108,9 @@ class Configuration
 
     # @return [Cache] instance used internally to synchronize changes.
     attr_accessor :cache
+
+    # @return [KeyAccessLog] instance used key access log.
+    attr_accessor :key_access_log
 
     # @return [Client] instance used to communicate with a CopyTuner Server.
     attr_accessor :client
@@ -131,6 +135,9 @@ class Configuration
     # @return [Boolean] Html escape
     attr_accessor :html_escape
 
+    # @return [Boolean] To enable uploading key access log to server
+    attr_accessor :enable_key_access_log
+
     alias_method :secure?, :secure
 
     # Instantiated from {CopyTunerClient.configure}. Sets defaults.
@@ -152,6 +159,7 @@ def initialize
       self.s3_host = 'copy-tuner-data-prod.s3.amazonaws.com'
       self.disable_copyray_comment_injection = false
       self.html_escape = false
+      self.enable_key_access_log = false
 
       @applied = false
     end
@@ -233,13 +241,14 @@ def apply
 
       self.client ||= Client.new(to_hash)
       self.cache ||= Cache.new(client, to_hash)
-      @poller = Poller.new(cache, to_hash)
+      self.key_access_log = enable_key_access_log ? KeyAccessLog.new(client) : KeyAccessLog::NullLog.new
+      @poller = Poller.new(cache, key_access_log, to_hash)
       process_guard = ProcessGuard.new(cache, @poller, to_hash)
-      I18n.backend = I18nBackend.new(cache)
+      I18n.backend = I18nBackend.new(cache, key_access_log)
 
       if enable_middleware?
         logger.info "Using copytuner sync middleware"
-        request_sync_options = {:poller => @poller, :cache => cache, :interval => sync_interval, :ignore_regex => sync_ignore_path_regex}
+        request_sync_options = {:poller => @poller, :cache => cache, :key_access_log => key_access_log, :interval => sync_interval, :ignore_regex => sync_ignore_path_regex}
         if middleware_position.is_a?(Hash) && middleware_position[:before]
           middleware.insert_before middleware_position[:before], RequestSync, request_sync_options
           middleware.insert_before middleware_position[:before], CopyTunerClient::CopyrayMiddleware

lib/copy_tuner_client/i18n_backend.rb

@@ -16,8 +16,9 @@ class I18nBackend
 
     # Usually instantiated when {Configuration#apply} is invoked.
     # @param cache [Cache] must act like a hash, returning and accept blurbs by key.
-    def initialize(cache)
+    def initialize(cache, key_access_log)
       @cache = cache
+      @key_access_log = key_access_log
     end
 
     # Translates the given local and key. See the I18n API documentation for details.
@@ -65,7 +66,10 @@ def lookup(locale, key, scope = [], options = {})
 
       parts = I18n.normalize_keys(locale, key, scope, options[:separator])
       key_with_locale = parts.join('.')
-      content = cache[key_with_locale] || super
+      if (cached_content = cache[key_with_locale])
+        key_access_log.add(key_with_locale)
+      end
+      content = cached_content || super
       cache[key_with_locale] = nil if content.nil?
       content
     end
@@ -101,6 +105,6 @@ def default(locale, object, subject, options = {})
       content
     end
 
-    attr_reader :cache
+    attr_reader :cache, :key_access_log
   end
 end

lib/copy_tuner_client/key_access_log.rb

@@ -0,0 +1,48 @@
+module CopyTunerClient
+  class KeyAccessLog
+    class NullLog
+      def add(key)
+      end
+
+      def flush
+      end
+    end
+
+    def initialize(client)
+      @client = client
+      @entries = {}
+      @mutex = Mutex.new
+    end
+
+    def add(key_with_locale)
+      return if key_with_locale.nil? || key_with_locale.empty?
+
+      key = key_with_locale.split('.', 2)[1]
+      return if key.nil? || key.empty?
+
+      lock do
+        @entries[key] = Time.now.to_i
+      end
+    end
+
+    def flush
+      to_be_uploaded = nil
+      lock do
+        to_be_uploaded = @entries
+        @entries = {}
+      end
+
+      return if to_be_uploaded.empty?
+
+      client.upload_key_acess_log(to_be_uploaded)
+    end
+
+    private
+
+    attr_reader :client
+
+    def lock(&block)
+      @mutex.synchronize &block
+    end
+  end
+end

lib/copy_tuner_client/poller.rb

@@ -9,8 +9,9 @@ class Poller
     # @param options [Hash]
     # @option options [Logger] :logger where errors should be logged
     # @option options [Fixnum] :polling_delay how long to wait in between requests
-    def initialize(cache, options)
+    def initialize(cache, key_access_log, options)
       @cache         = cache
+      @key_access_log = key_access_log
       @polling_delay = options[:polling_delay]
       @logger        = options[:logger]
       @command_queue = CopyTunerClient::QueueWithTimeout.new
@@ -45,11 +46,12 @@ def wait_for_download
 
     private
 
-    attr_reader :cache, :logger, :polling_delay
+    attr_reader :cache, :key_access_log, :logger, :polling_delay
 
     def poll
       loop do
         cache.sync
+        key_access_log.flush
         logger.flush if logger.respond_to?(:flush)
         begin
           command = @command_queue.pop_with_timeout(polling_delay)

lib/copy_tuner_client/request_sync.rb

@@ -18,6 +18,7 @@ def initialize(app, options)
       @app = app
       @poller = options[:poller]
       @cache = options[:cache]
+      @key_access_log = options[:key_access_log]
       @interval = options[:interval]
       @ignore_regex = options[:ignore_regex]
       @last_synced = options[:last_synced]
@@ -71,6 +72,7 @@ def index
 
     def sync
       @cache.sync
+      @key_access_log.flush
       ::Rack::Response.new{|r| r.redirect('/copytuner/')}.finish
     end
 

spec/copy_tuner_client/client_spec.rb

@@ -93,6 +93,11 @@ def build_client_with_project(config = {})
       expect { client.upload({}) }.to raise_error(CopyTunerClient::ConnectionError)
     end
 
+    it 'handles 500 errors from key access log uploads with ConnectionError' do
+      client = build_client(:api_key => 'raise_error')
+      expect { client.upload_key_acess_log({}) }.to raise_error(CopyTunerClient::ConnectionError)
+    end
+
     it 'handles 404 errors from downloads with ConnectionError' do
       client = build_client(:api_key => 'bogus')
       expect { client.download { |ignore| } }.
@@ -103,7 +108,12 @@ def build_client_with_project(config = {})
       client = build_client(:api_key => 'bogus')
       expect { client.upload({}) }.to raise_error(CopyTunerClient::InvalidApiKey)
     end
-  end
+
+    it 'handles 404 errors from key access log uploads with ConnectionError' do
+      client = build_client(:api_key => 'bogus')
+      expect { client.upload_key_acess_log({}) }.to raise_error(CopyTunerClient::InvalidApiKey)
+    end
+end
 
   it 'downloads published blurbs for an existing project' do
     project = add_project
@@ -195,7 +205,21 @@ def build_client_with_project(config = {})
     client.upload({})
     expect(logger).to have_entry(:info, "Uploaded missing translations")
   end
+
+  it "uploads key access logs in an existing project" do
+    project = add_project
+
+    logs = {
+      'key.one' => Time.now.to_i,
+      'key.two' => Time.now.to_i,
+    }
 
+    client = build_client(:api_key => project.api_key, :public => true)
+    client.upload_key_acess_log(logs)
+
+    expect(project.reload.key_access_logs).to eq(logs)
+  end
+
   it "deploys from the top-level constant" do
     client = build_client
     allow(client).to receive(:download)

spec/copy_tuner_client/configuration_spec.rb

@@ -195,6 +195,8 @@
   subject { CopyTunerClient::Configuration.new }
   let(:backend) { double('i18n-backend') }
   let(:cache) { double('cache', download: "download") }
+  let(:key_access_log) { double('key_access_log') }
+  let(:null_key_access_log) { double('null_key_access_log') }
   let(:client) { double('client') }
   let(:logger) { FakeLogger.new }
   let(:poller) { double('poller') }
@@ -204,6 +206,8 @@
     allow(CopyTunerClient::I18nBackend).to receive(:new).and_return(backend)
     allow(CopyTunerClient::Client).to receive(:new).and_return(client)
     allow(CopyTunerClient::Cache).to receive(:new).and_return(cache)
+    allow(CopyTunerClient::KeyAccessLog).to receive(:new).and_return(key_access_log)
+    allow(CopyTunerClient::KeyAccessLog::NullLog).to receive(:new).and_return(null_key_access_log)
     allow(CopyTunerClient::Poller).to receive(:new).and_return(poller)
     allow(CopyTunerClient::ProcessGuard).to receive(:new).and_return(process_guard)
     subject.logger = logger
@@ -217,12 +221,12 @@
   it { is_expected.to be_applied }
 
   it 'builds and assigns an I18n backend' do
-    expect(CopyTunerClient::I18nBackend).to have_received(:new).with(cache)
+    expect(CopyTunerClient::I18nBackend).to have_received(:new).with(cache, null_key_access_log)
     expect(I18n.backend).to eq(backend)
   end
 
   it 'builds and assigns a poller' do
-    expect(CopyTunerClient::Poller).to have_received(:new).with(cache, subject.to_hash)
+    expect(CopyTunerClient::Poller).to have_received(:new).with(cache, null_key_access_log, subject.to_hash)
   end
 
   it 'builds a process guard' do
@@ -332,6 +336,32 @@ def apply
   end
 end
 
+describe CopyTunerClient::Configuration, 'applied when enable_key_access_log is true' do
+  include_context 'stubbed configuration'
+
+  def apply
+    subject.enable_key_access_log = true
+    subject.apply
+  end
+
+  it 'instanciate CopyTunerClient::KeyAccessLog' do
+    expect(subject.key_access_log).to eq key_access_log
+  end
+end
+
+describe CopyTunerClient::Configuration, 'applied when enable_key_access_log is false' do
+  include_context 'stubbed configuration'
+
+  def apply
+    subject.enable_key_access_log = false
+    subject.apply
+  end
+
+  it 'instanciate CopyTunerClient::KeyAccessLog::NullLog' do
+    expect(subject.key_access_log).to eq null_key_access_log
+  end
+end
+
 describe CopyTunerClient::Configuration, 'applied with Rails i18n config' do
   let!(:rails_defined) { Object.const_defined?(:Rails) }
 

spec/copy_tuner_client/i18n_backend_spec.rb

@@ -2,9 +2,10 @@
 
 describe CopyTunerClient::I18nBackend do
   let(:cache) { {} }
+  let(:key_access_log) { double('key_access_log', add: nil, flush: nil) }
 
   def build_backend
-    backend = CopyTunerClient::I18nBackend.new(cache)
+    backend = CopyTunerClient::I18nBackend.new(cache, key_access_log)
     I18n.backend = backend
     backend
   end
@@ -35,6 +36,7 @@ def build_backend
 
     backend = build_backend
 
+    expect(key_access_log).to receive(:add)
     expect(backend.translate('en', 'test.key', :scope => 'prefix')).to eq(value)
   end
 
@@ -53,6 +55,7 @@ def build_backend
 
     expect(subject.translate('en', 'test.key', :default => default)).to eq(default)
 
+    expect(key_access_log).not_to receive(:add)
     expect(cache['en.test.key']).to eq(default)
   end
 

spec/copy_tuner_client/poller_spec.rb

@@ -5,12 +5,13 @@
 
   let(:client) { FakeClient.new }
   let(:cache) { CopyTunerClient::Cache.new(client, :logger => FakeLogger.new) }
+  let(:key_access_log) { CopyTunerClient::KeyAccessLog.new(client) }
 
   def build_poller(config = {})
     config[:logger] ||= FakeLogger.new
     config[:polling_delay] = POLLING_DELAY
     default_config = CopyTunerClient::Configuration.new.to_hash
-    poller = CopyTunerClient::Poller.new(cache, default_config.update(config))
+    poller = CopyTunerClient::Poller.new(cache, key_access_log, default_config.update(config))
     @pollers << poller
     poller
   end
@@ -28,22 +29,28 @@ def wait_for_next_sync
   end
 
   it "it polls after being started" do
+    key_access_log.add('some.key')
+
     poller = build_poller
     poller.start
 
     client['test.key'] = 'value'
     wait_for_next_sync
 
     expect(cache['test.key']).to eq('value')
+    expect(client.uploaded_key_access_logs.keys).to eq %w[key]
   end
 
   it "it doesn't poll before being started" do
+    key_access_log.add('some.key')
+
     poller = build_poller
     client['test.key'] = 'value'
 
     wait_for_next_sync
 
     expect(cache['test.key']).to be_nil
+    expect(client.uploaded_key_access_logs.keys).to be_empty
   end
 
   it "stops polling when stopped" do