Merge branch 'fair-usage-of-shared-runners' into 'master'

CHANGELOG

@@ -22,6 +22,7 @@ v 8.9.0 (unreleased)
   - Added descriptions to notification settings dropdown
   - Improve note validation to prevent errors when creating invalid note via API
   - Reduce number of fog gem dependencies
+  - Implement a fair usage of shared runners
   - Remove project notification settings associated with deleted projects
   - Fix 404 page when viewing TODOs that contain milestones or labels in different projects
   - Add a metric for the number of new Redis connections created by a transaction

app/services/ci/register_build_service.rb

@@ -7,15 +7,19 @@ module Ci
 
       builds =
         if current_runner.shared?
-          # don't run projects which have not enables shared runners
-          builds.joins(:project).where(projects: { builds_enabled: true, shared_runners_enabled: true })
+          builds.
+            # don't run projects which have not enabled shared runners
+            joins(:project).where(projects: { builds_enabled: true, shared_runners_enabled: true }).
+
+            # this returns builds that are ordered by number of running builds
+            # we prefer projects that don't use shared runners at all
+            joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.gl_project_id=project_builds.gl_project_id").
+            order('COALESCE(project_builds.running_builds, 0) ASC', 'ci_builds.id ASC')
         else
-          # do run projects which are only assigned to this runner
-          builds.where(project: current_runner.projects.where(builds_enabled: true))
+          # do run projects which are only assigned to this runner (FIFO)
+          builds.where(project: current_runner.projects.where(builds_enabled: true)).order('created_at ASC')
         end
 
-      builds = builds.order('created_at ASC')
-
       build = builds.find do |build|
         build.can_be_served?(current_runner)
       end
@@ -35,5 +39,12 @@ module Ci
     rescue StateMachines::InvalidTransition
       nil
     end
+
+    private
+
+    def running_builds_for_shared_runners
+      Ci::Build.running.where(runner: Ci::Runner.shared).
+        group(:gl_project_id).select(:gl_project_id, 'count(*) AS running_builds')
+    end
   end
 end

spec/services/ci/register_build_service_spec.rb

@@ -45,11 +45,73 @@ module Ci
         end
       end
 
+      context 'deleted projects' do
+        before do
+          project.update(pending_delete: true)
+        end
+
+        context 'for shared runners' do
+          before do
+            project.update(shared_runners_enabled: true)
+          end
+
+          it 'does not pick a build' do
+            expect(service.execute(shared_runner)).to be_nil
+          end
+        end
+
+        context 'for specific runner' do
+          it 'does not pick a build' do
+            expect(service.execute(specific_runner)).to be_nil
+          end
+        end
+      end
+
       context 'allow shared runners' do
         before do
           project.update(shared_runners_enabled: true)
         end
 
+        context 'for multiple builds' do
+          let!(:project2) { create :empty_project, shared_runners_enabled: true }
+          let!(:pipeline2) { create :ci_pipeline, project: project2 }
+          let!(:project3) { create :empty_project, shared_runners_enabled: true }
+          let!(:pipeline3) { create :ci_pipeline, project: project3 }
+          let!(:build1_project1) { pending_build }
+          let!(:build2_project1) { FactoryGirl.create :ci_build, pipeline: pipeline }
+          let!(:build3_project1) { FactoryGirl.create :ci_build, pipeline: pipeline }
+          let!(:build1_project2) { FactoryGirl.create :ci_build, pipeline: pipeline2 }
+          let!(:build2_project2) { FactoryGirl.create :ci_build, pipeline: pipeline2 }
+          let!(:build1_project3) { FactoryGirl.create :ci_build, pipeline: pipeline3 }
+
+          it 'prefers projects without builds first' do
+            # it gets for one build from each of the projects
+            expect(service.execute(shared_runner)).to eq(build1_project1)
+            expect(service.execute(shared_runner)).to eq(build1_project2)
+            expect(service.execute(shared_runner)).to eq(build1_project3)
+
+            # then it gets a second build from each of the projects
+            expect(service.execute(shared_runner)).to eq(build2_project1)
+            expect(service.execute(shared_runner)).to eq(build2_project2)
+
+            # in the end the third build
+            expect(service.execute(shared_runner)).to eq(build3_project1)
+          end
+
+          it 'equalises number of running builds' do
+            # after finishing the first build for project 1, get a second build from the same project
+            expect(service.execute(shared_runner)).to eq(build1_project1)
+            build1_project1.success
+            expect(service.execute(shared_runner)).to eq(build2_project1)
+
+            expect(service.execute(shared_runner)).to eq(build1_project2)
+            build1_project2.success
+            expect(service.execute(shared_runner)).to eq(build2_project2)
+            expect(service.execute(shared_runner)).to eq(build1_project3)
+            expect(service.execute(shared_runner)).to eq(build3_project1)
+          end
+        end
+
         context 'shared runner' do
           let(:build) { service.execute(shared_runner) }