Merge branch 'configure-randomize-metrics-sample-interval' into 'master'

app/controllers/admin/application_settings_controller.rb

@@ -73,6 +73,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :metrics_pool_size,
       :metrics_timeout,
       :metrics_method_call_threshold,
+      :metrics_sample_interval,
       :recaptcha_enabled,
       :recaptcha_site_key,
       :recaptcha_private_key,

app/views/admin/application_settings/_form.html.haml

@@ -202,6 +202,13 @@
         .help-block
           A method call is only tracked when it takes longer to complete than
           the given amount of milliseconds.
+    .form-group
+      = f.label :metrics_sample_interval, 'Sampler Interval (sec)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :metrics_sample_interval, class: 'form-control'
+        .help-block
+          The sampling interval in seconds. Sampled data includes memory usage,
+          retained Ruby objects, file descriptors and so on.
 
   %fieldset
     %legend Spam and Anti-bot Protection

db/migrate/20160113111034_add_metrics_sample_interval.rb

+class AddMetricsSampleInterval < ActiveRecord::Migration
+  def change
+    add_column :application_settings, :metrics_sample_interval, :integer,
+      default: 15
+  end
+end

lib/gitlab/metrics.rb

@@ -13,7 +13,8 @@ module Gitlab
         timeout:               current_application_settings[:metrics_timeout],
         method_call_threshold: current_application_settings[:metrics_method_call_threshold],
         host:                  current_application_settings[:metrics_host],
-        port:                  current_application_settings[:metrics_port]
+        port:                  current_application_settings[:metrics_port],
+        sample_interval:       current_application_settings[:metrics_sample_interval] || 15
       }
     end
 

lib/gitlab/metrics/sampler.rb

@@ -7,9 +7,14 @@ module Gitlab
     # statistics, etc.
     class Sampler
       # interval - The sampling interval in seconds.
-      def initialize(interval = 15)
-        @interval = interval
-        @metrics  = []
+      def initialize(interval = Metrics.settings[:sample_interval])
+        interval_half = interval.to_f / 2
+
+        @interval       = interval
+        @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+        @last_step      = nil
+
+        @metrics = []
 
         @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
         @last_major_gc = Delta.new(GC.stat[:major_gc_count])
@@ -26,7 +31,7 @@ module Gitlab
           Thread.current.abort_on_exception = true
 
           loop do
-            sleep(@interval)
+            sleep(sleep_interval)
 
             sample
           end
@@ -102,6 +107,23 @@ module Gitlab
       def sidekiq?
         Sidekiq.server?
       end
+
+      # Returns the sleep interval with a random adjustment.
+      #
+      # The random adjustment is put in place to ensure we:
+      #
+      # 1. Don't generate samples at the exact same interval every time (thus
+      #    potentially missing anything that happens in between samples).
+      # 2. Don't sample data at the same interval two times in a row.
+      def sleep_interval
+        while step = @interval_steps.sample
+          if step != @last_step
+            @last_step = step
+
+            return @interval + @last_step
+          end
+        end
+      end
     end
   end
 end

spec/lib/gitlab/metrics/sampler_spec.rb

@@ -9,7 +9,7 @@ describe Gitlab::Metrics::Sampler do
 
   describe '#start' do
     it 'gathers a sample at a given interval' do
-      expect(sampler).to receive(:sleep).with(5)
+      expect(sampler).to receive(:sleep).with(a_kind_of(Numeric))
       expect(sampler).to receive(:sample)
       expect(sampler).to receive(:loop).and_yield
 
@@ -116,4 +116,24 @@ describe Gitlab::Metrics::Sampler do
       sampler.add_metric('cats', value: 10)
     end
   end
+
+  describe '#sleep_interval' do
+    it 'returns a Numeric' do
+      expect(sampler.sleep_interval).to be_a_kind_of(Numeric)
+    end
+
+    # Testing random behaviour is very hard, so treat this test as a basic smoke
+    # test instead of a very accurate behaviour/unit test.
+    it 'does not return the same interval twice in a row' do
+      last = nil
+
+      100.times do
+        interval = sampler.sleep_interval
+
+        expect(interval).to_not eq(last)
+
+        last = interval
+      end
+    end
+  end
 end