Merge branch '10-8-stable-prepare-rc6' into '10-8-stable'

db/migrate/20180420010616_cleanup_build_stage_migration.rb

@@ -2,6 +2,7 @@ class CleanupBuildStageMigration < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
+  TMP_INDEX = 'tmp_id_stage_partial_null_index'.freeze
 
   disable_ddl_transaction!
 
@@ -13,16 +14,48 @@ class CleanupBuildStageMigration < ActiveRecord::Migration
   end
 
   def up
+    disable_statement_timeout
+
+    ##
+    # We steal from the background migrations queue to catch up with the
+    # scheduled migrations set.
+    #
     Gitlab::BackgroundMigration.steal('MigrateBuildStage')
 
+    ##
+    # We add temporary index, to make iteration over batches more performant.
+    # Conditional here is to avoid the need of doing that in a separate
+    # migration file to make this operation idempotent.
+    #
+    unless index_exists_by_name?(:ci_builds, TMP_INDEX)
+      add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', name: TMP_INDEX)
+    end
+
+    ##
+    # We check if there are remaining rows that should be migrated (for example
+    # if Sidekiq / Redis fails / is restarted, what could result in not all
+    # background migrations being executed correctly.
+    #
+    # We migrate remaining rows synchronously in a blocking way, to make sure
+    # that when this migration is done we are confident that all rows are
+    # already migrated.
+    #
     Build.where('stage_id IS NULL').each_batch(of: 50) do |batch|
       range = batch.pluck('MIN(id)', 'MAX(id)').first
 
       Gitlab::BackgroundMigration::MigrateBuildStage.new.perform(*range)
     end
+
+    ##
+    # We remove temporary index, because it is not required during standard
+    # operations and runtime.
+    #
+    remove_concurrent_index_by_name(:ci_builds, TMP_INDEX)
   end
 
   def down
-    # noop
+    if index_exists_by_name?(:ci_builds, TMP_INDEX)
+      remove_concurrent_index_by_name(:ci_builds, TMP_INDEX)
+    end
   end
 end

db/schema.rb

@@ -2148,6 +2148,7 @@ ActiveRecord::Schema.define(version: 20180508055821) do
   add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "projects", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
+  add_foreign_key "ci_builds", "ci_pipelines", column: "commit_id", name: "fk_d3130c9a7f", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
   add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
   add_foreign_key "ci_builds_metadata", "ci_builds", column: "build_id", on_delete: :cascade

doc/ci/yaml/README.md

@@ -738,10 +738,15 @@ cache:
 rspec:
   script: test
   cache:
+    key: rspec
     paths:
     - binaries/
 ```
 
+Note that since cache is shared between jobs, if you're using different
+paths for different jobs, you should also set a different **cache:key**
+otherwise cache content can be overwritten.
+
 ### `cache:key`
 
 > Introduced in GitLab Runner v1.0.0.
@@ -756,10 +761,9 @@ or any other way that fits your workflow. This way, you can fine tune caching,
 allowing you to cache data between different jobs or even different branches.
 
 The `cache:key` variable can use any of the
-[predefined variables](../variables/README.md), and the default key, if not set,
-is `$CI_JOB_NAME-$CI_COMMIT_REF_NAME` which translates as "per-job and
-per-branch". It is the default across the project, therefore everything is
-shared between pipelines and jobs running on the same branch by default.
+[predefined variables](../variables/README.md), and the default key, if not
+set, is just literal `default` which means everything is shared between each
+pipelines and jobs by default, starting from GitLab 9.0.
 
 NOTE: **Note:**
 The `cache:key` variable cannot contain the `/` character, or the equivalent
@@ -779,7 +783,7 @@ If you use **Windows Batch** to run your shell scripts you need to replace
 
 ```yaml
 cache:
-  key: "%CI_JOB_STAGE%-%CI_COMMIT_REF_SLUG%"
+  key: "%CI_COMMIT_REF_SLUG%"
   paths:
   - binaries/
 ```
@@ -789,7 +793,7 @@ If you use **Windows PowerShell** to run your shell scripts you need to replace
 
 ```yaml
 cache:
-  key: "$env:CI_JOB_STAGE-$env:CI_COMMIT_REF_SLUG"
+  key: "$env:CI_COMMIT_REF_SLUG"
   paths:
   - binaries/
 ```