Add sidekiq metrics endpoint and add http server to sidekiq

changelogs/unreleased/pawel-add-sidekiq-metrics-endpoint-32145.yml

+---
+title: Add Prometheus metrics exporter to Sidekiq
+merge_request: 13082
+author:

config/gitlab.yml.example

@@ -590,6 +590,12 @@ production: &base
     ip_whitelist:
       - 127.0.0.0/8
 
+    # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics
+    sidekiq_exporter:
+    #  enabled: true
+    #  address: localhost
+    #  port: 3807
+
   #
   # 5. Extra customization
   # ==========================

config/initializers/1_settings.rb

@@ -524,6 +524,10 @@ Settings.webpack.dev_server['port']    ||= 3808
 Settings['monitoring'] ||= Settingslogic.new({})
 Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8']
 Settings.monitoring['unicorn_sampler_interval'] ||= 10
+Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
+Settings.monitoring.sidekiq_exporter['enabled'] ||= false
+Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
+Settings.monitoring.sidekiq_exporter['port'] ||= 3807
 
 #
 # Testing settings

config/initializers/7_prometheus_metrics.rb

@@ -10,3 +10,9 @@ Prometheus::Client.configure do |config|
     config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
   end
 end
+
+Sidekiq.configure_server do |config|
+  config.on(:startup) do
+    Gitlab::Metrics::SidekiqMetricsExporter.instance.start
+  end
+end

lib/gitlab/daemon.rb

+module Gitlab
+  class Daemon
+    def self.initialize_instance(*args)
+      raise "#{name} singleton instance already initialized" if @instance
+      @instance = new(*args)
+      Kernel.at_exit(&@instance.method(:stop))
+      @instance
+    end
+
+    def self.instance
+      @instance ||= initialize_instance
+    end
+
+    attr_reader :thread
+
+    def thread?
+      !thread.nil?
+    end
+
+    def initialize
+      @mutex = Mutex.new
+    end
+
+    def enabled?
+      true
+    end
+
+    def start
+      return unless enabled?
+
+      @mutex.synchronize do
+        return thread if thread?
+
+        @thread = Thread.new { start_working }
+      end
+    end
+
+    def stop
+      @mutex.synchronize do
+        return unless thread?
+
+        stop_working
+
+        if thread
+          thread.wakeup if thread.alive?
+          thread.join
+          @thread = nil
+        end
+      end
+    end
+
+    private
+
+    def start_working
+      raise NotImplementedError
+    end
+
+    def stop_working
+      # no-ops
+    end
+  end
+end

lib/gitlab/metrics/base_sampler.rb

 require 'logger'
 module Gitlab
   module Metrics
-    class BaseSampler
-      def self.initialize_instance(*args)
-        raise "#{name} singleton instance already initialized" if @instance
-        @instance = new(*args)
-        at_exit(&@instance.method(:stop))
-        @instance
-      end
-
-      def self.instance
-        @instance
-      end
-
-      attr_reader :running
-
+    class BaseSampler < Daemon
       # interval - The sampling interval in seconds.
       def initialize(interval)
         interval_half = interval.to_f / 2
@@ -22,44 +9,7 @@ module Gitlab
         @interval = interval
         @interval_steps = (-interval_half..interval_half).step(0.1).to_a
 
-        @mutex = Mutex.new
-      end
-
-      def enabled?
-        true
-      end
-
-      def start
-        return unless enabled?
-
-        @mutex.synchronize do
-          return if running
-          @running = true
-
-          @thread = Thread.new do
-            sleep(sleep_interval)
-
-            while running
-              safe_sample
-
-              sleep(sleep_interval)
-            end
-          end
-        end
-      end
-
-      def stop
-        @mutex.synchronize do
-          return unless running
-
-          @running = false
-
-          if @thread
-            @thread.wakeup if @thread.alive?
-            @thread.join
-            @thread = nil
-          end
-        end
+        super()
       end
 
       def safe_sample
@@ -81,7 +31,7 @@ module Gitlab
       #    potentially missing anything that happens in between samples).
       # 2. Don't sample data at the same interval two times in a row.
       def sleep_interval
-        while step = @interval_steps.sample
+        while (step = @interval_steps.sample)
           if step != @last_step
             @last_step = step
 
@@ -89,6 +39,25 @@ module Gitlab
           end
         end
       end
+
+      private
+
+      attr_reader :running
+
+      def start_working
+        @running = true
+        sleep(sleep_interval)
+
+        while running
+          safe_sample
+
+          sleep(sleep_interval)
+        end
+      end
+
+      def stop_working
+        @running = false
+      end
     end
   end
 end

lib/gitlab/metrics/sidekiq_metrics_exporter.rb

+require 'webrick'
+require 'prometheus/client/rack/exporter'
+
+module Gitlab
+  module Metrics
+    class SidekiqMetricsExporter < Daemon
+      def enabled?
+        Gitlab::Metrics.metrics_folder_present? && settings.enabled
+      end
+
+      def settings
+        Settings.monitoring.sidekiq_exporter
+      end
+
+      private
+
+      attr_reader :server
+
+      def start_working
+        @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address)
+        server.mount "/", Rack::Handler::WEBrick, rack_app
+        server.start
+      end
+
+      def stop_working
+        server.shutdown
+        @server = nil
+      end
+
+      def rack_app
+        Rack::Builder.app do
+          use Rack::Deflater
+          use ::Prometheus::Client::Rack::Exporter
+          run -> (env) { [404, {}, ['']] }
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/daemon_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Daemon do
+  subject { described_class.new }
+
+  before do
+    allow(subject).to receive(:start_working)
+    allow(subject).to receive(:stop_working)
+  end
+
+  describe '.instance' do
+    before do
+      allow(Kernel).to receive(:at_exit)
+    end
+
+    after(:each) do
+      described_class.instance_variable_set(:@instance, nil)
+    end
+
+    it 'provides instance of Daemon' do
+      expect(described_class.instance).to be_instance_of(described_class)
+    end
+
+    it 'subsequent invocations provide the same instance' do
+      expect(described_class.instance).to eq(described_class.instance)
+    end
+
+    it 'creates at_exit hook when instance is created' do
+      expect(described_class.instance).not_to be_nil
+
+      expect(Kernel).to have_received(:at_exit)
+    end
+  end
+
+  describe 'when Daemon is enabled' do
+    before do
+      allow(subject).to receive(:enabled?).and_return(true)
+    end
+
+    describe 'when Daemon is stopped' do
+      describe '#start' do
+        it 'starts the Daemon' do
+          expect { subject.start.join }.to change { subject.thread? }.from(false).to(true)
+
+          expect(subject).to have_received(:start_working)
+        end
+      end
+
+      describe '#stop' do
+        it "doesn't shutdown stopped Daemon" do
+          expect { subject.stop }.not_to change { subject.thread? }
+
+          expect(subject).not_to have_received(:start_working)
+        end
+      end
+    end
+
+    describe 'when Daemon is running' do
+      before do
+        subject.start.join
+      end
+
+      describe '#start' do
+        it "doesn't start running Daemon" do
+          expect { subject.start.join }.not_to change { subject.thread? }
+
+          expect(subject).to have_received(:start_working).once
+        end
+      end
+
+      describe '#stop' do
+        it 'shutdowns Daemon' do
+          expect { subject.stop }.to change { subject.thread? }.from(true).to(false)
+
+          expect(subject).to have_received(:stop_working)
+        end
+      end
+    end
+  end
+
+  describe 'when Daemon is disabled' do
+    before do
+      allow(subject).to receive(:enabled?).and_return(false)
+    end
+
+    describe '#start' do
+      it "doesn't start working" do
+        expect(subject.start).to be_nil
+        expect { subject.start }.not_to change { subject.thread? }
+
+        expect(subject).not_to have_received(:start_working)
+      end
+    end
+
+    describe '#stop' do
+      it "doesn't stop working" do
+        expect { subject.stop }.not_to change { subject.thread? }
+
+        expect(subject).not_to have_received(:stop_working)
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/influx_sampler_spec.rb

@@ -11,7 +11,7 @@ describe Gitlab::Metrics::InfluxSampler do
     it 'runs once and gathers a sample at a given interval' do
       expect(sampler).to receive(:sleep).with(a_kind_of(Numeric)).twice
       expect(sampler).to receive(:sample).once
-      expect(sampler).to receive(:running).and_return(false, true, false)
+      expect(sampler).to receive(:running).and_return(true, false)
 
       sampler.start.join
     end

spec/lib/gitlab/metrics/sidekiq_metrics_exporter_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Metrics::SidekiqMetricsExporter do
+  let(:exporter) { described_class.new }
+  let(:server) { double('server') }
+
+  before do
+    allow(::WEBrick::HTTPServer).to receive(:new).and_return(server)
+    allow(server).to receive(:mount)
+    allow(server).to receive(:start)
+    allow(server).to receive(:shutdown)
+  end
+
+  describe 'when exporter is enabled' do
+    before do
+      allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(true)
+    end
+
+    describe 'when exporter is stopped' do
+      describe '#start' do
+        it 'starts the exporter' do
+          expect { exporter.start.join }.to change { exporter.thread? }.from(false).to(true)
+
+          expect(server).to have_received(:start)
+        end
+
+        describe 'with custom settings' do
+          let(:port) { 99999 }
+          let(:address) { 'sidekiq_exporter_address' }
+
+          before do
+            allow(Settings.monitoring.sidekiq_exporter).to receive(:port).and_return(port)
+            allow(Settings.monitoring.sidekiq_exporter).to receive(:address).and_return(address)
+          end
+
+          it 'starts server with port and address from settings' do
+            exporter.start.join
+
+            expect(::WEBrick::HTTPServer).to have_received(:new).with(
+              Port: port,
+              BindAddress: address
+            )
+          end
+        end
+      end
+
+      describe '#stop' do
+        it "doesn't shutdown stopped server" do
+          expect { exporter.stop }.not_to change { exporter.thread? }
+
+          expect(server).not_to have_received(:shutdown)
+        end
+      end
+    end
+
+    describe 'when exporter is running' do
+      before do
+        exporter.start.join
+      end
+
+      describe '#start' do
+        it "doesn't start running server" do
+          expect { exporter.start.join }.not_to change { exporter.thread? }
+
+          expect(server).to have_received(:start).once
+        end
+      end
+
+      describe '#stop' do
+        it 'shutdowns server' do
+          expect { exporter.stop }.to change { exporter.thread? }.from(true).to(false)
+
+          expect(server).to have_received(:shutdown)
+        end
+      end
+    end
+  end
+
+  describe 'when exporter is disabled' do
+    before do
+      allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(false)
+    end
+
+    describe '#start' do
+      it "doesn't start" do
+        expect(exporter.start).to be_nil
+        expect { exporter.start }.not_to change { exporter.thread? }
+
+        expect(server).not_to have_received(:start)
+      end
+    end
+
+    describe '#stop' do
+      it "doesn't shutdown" do
+        expect { exporter.stop }.not_to change { exporter.thread? }
+
+        expect(server).not_to have_received(:shutdown)
+      end
+    end
+  end
+end