Merge branch '10-8-stable-prepare-rc5' into '10-8-stable'

GITALY_SERVER_VERSION

-0.98.0
+0.99.0

app/assets/javascripts/ide/components/mr_file_icon.vue

@@ -16,8 +16,8 @@ export default {
   <icon
     name="git-merge"
     v-tooltip
-    title="__('Part of merge request changes')"
-    css-classes="ide-file-changed-icon"
+    :title="__('Part of merge request changes')"
+    css-classes="append-right-8"
     :size="12"
   />
 </template>

app/assets/javascripts/ide/lib/common/model.js

@@ -14,12 +14,12 @@ export default class Model {
       (this.originalModel = this.monaco.editor.createModel(
         head ? head.content : this.file.raw,
         undefined,
-        new this.monaco.Uri(null, null, `original/${this.file.key}`),
+        new this.monaco.Uri(null, null, `original/${this.path}`),
       )),
       (this.model = this.monaco.editor.createModel(
         this.content,
         undefined,
-        new this.monaco.Uri(null, null, this.file.key),
+        new this.monaco.Uri(null, null, this.path),
       )),
     );
     if (this.file.mrChange) {
@@ -27,7 +27,7 @@ export default class Model {
         (this.baseModel = this.monaco.editor.createModel(
           this.file.baseRaw,
           undefined,
-          new this.monaco.Uri(null, null, `target/${this.file.path}`),
+          new this.monaco.Uri(null, null, `target/${this.path}`),
         )),
       );
     }

app/assets/javascripts/ide/stores/actions/file.js

@@ -196,6 +196,8 @@ export const unstageChange = ({ commit }, path) => {
 };
 
 export const openPendingTab = ({ commit, getters, dispatch, state }, { file, keyPrefix }) => {
+  if (getters.activeFile && getters.activeFile.key === `${keyPrefix}-${file.key}`) return false;
+
   state.openFiles.forEach(f => eventHub.$emit(`editor.update.model.dispose.${f.key}`));
 
   commit(types.ADD_PENDING_TAB, { file, keyPrefix });

app/models/clusters/applications/runner.rb

@@ -43,12 +43,20 @@ module Clusters
 
       def create_and_assign_runner
         transaction do
-          project.runners.create!(name: 'kubernetes-cluster', tag_list: %w(kubernetes cluster)).tap do |runner|
+          project.runners.create!(runner_create_params).tap do |runner|
             update!(runner_id: runner.id)
           end
         end
       end
 
+      def runner_create_params
+        {
+          name: 'kubernetes-cluster',
+          runner_type: :project_type,
+          tag_list: %w(kubernetes cluster)
+        }
+      end
+
       def gitlab_url
         Gitlab::Routing.url_helpers.root_url(only_path: false)
       end

app/models/project.rb

@@ -661,9 +661,15 @@ class Project < ActiveRecord::Base
     return if !force && (self[:import_status] == 'none' || self[:import_status].nil?)
     return unless import_state.nil?
 
-    create_import_state(import_state_args)
+    if persisted?
+      create_import_state(import_state_args)
 
-    update_column(:import_status, 'none')
+      update_column(:import_status, 'none')
+    else
+      build_import_state(import_state_args)
+
+      self[:import_status] = 'none'
+    end
   end
 
   def import_schedule

app/services/users/migrate_to_ghost_user_service.rb

@@ -49,7 +49,7 @@ module Users
       migrate_merge_requests
       migrate_notes
       migrate_abuse_reports
-      migrate_award_emojis
+      migrate_award_emoji
     end
 
     def migrate_issues
@@ -70,7 +70,7 @@ module Users
       user.reported_abuse_reports.update_all(reporter_id: ghost_user.id)
     end
 
-    def migrate_award_emojis
+    def migrate_award_emoji
       user.award_emoji.update_all(user_id: ghost_user.id)
     end
   end

changelogs/unreleased/5750-backport-checksum-git-commanderror-exit-status-128.yml

+---
+title: Raise NoRepository error for non-valid repositories when calculating repository
+  checksum
+merge_request: 18594
+author:
+type: fixed

changelogs/unreleased/fix-gb-add-pipeline-builds-foreign-key.yml

+---
+title: Add database foreign key constraint between pipelines and build
+merge_request: 18822
+author:
+type: fixed

db/migrate/20180420010016_add_pipeline_build_foreign_key.rb

+class AddPipelineBuildForeignKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute <<~SQL
+      DELETE FROM ci_builds WHERE project_id IS NULL OR commit_id IS NULL
+    SQL
+
+    execute <<~SQL
+      DELETE FROM ci_builds WHERE NOT EXISTS
+        (SELECT true FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id)
+        AND stage_id IS NULL
+    SQL
+
+    add_concurrent_foreign_key(:ci_builds, :ci_pipelines, column: :commit_id)
+  end
+
+  def down
+    return unless foreign_key_exists?(:ci_builds, :ci_pipelines, column: :commit_id)
+
+    remove_foreign_key(:ci_builds, column: :commit_id)
+  end
+end

db/migrate/20180508055821_make_remote_mirrors_disabled_by_default.rb

+class MakeRemoteMirrorsDisabledByDefault < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    change_column_default :remote_mirrors, :enabled, false
+  end
+
+  def down
+    change_column_default :remote_mirrors, :enabled, true
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180503200320) do
+ActiveRecord::Schema.define(version: 20180508055821) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1712,7 +1712,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
   create_table "remote_mirrors", force: :cascade do |t|
     t.integer "project_id"
     t.string "url"
-    t.boolean "enabled", default: true
+    t.boolean "enabled", default: false
     t.string "update_status"
     t.datetime "last_update_at"
     t.datetime "last_successful_update_at"

doc/ci/examples/container_scanning.md

@@ -23,7 +23,7 @@ sast:container:
     - docker:stable-dind
   script:
     - docker run -d --name db arminc/clair-db:latest
-    - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
+    - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1
     - apk add -U wget ca-certificates
     - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
     - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64

doc/ci/runners/README.md

@@ -11,7 +11,7 @@ Ideally, the GitLab Runner should not be installed on the same machine as GitLab
 Read the [requirements documentation](../../install/requirements.md#gitlab-runner)
 for more information.
 
-## Shared vs specific Runners
+## Shared, specific and group Runners
 
 After [installing the Runner][install], you can either register it as shared or
 specific. You can only register a shared Runner if you have admin access to
@@ -32,6 +32,9 @@ are:
   Runners. For example, if you want to deploy a certain project, you can setup
   a specific Runner to have the right credentials for this. The [usage of tags](#using-tags)
   may be useful in this case. Specific Runners process jobs using a [FIFO] queue.
+- **Group Runners** are useful when you have multiple projects under one group
+  and would like all projects to have access to a set of Runners. Group Runners
+  process jobs using a [FIFO] queue.
 
 A Runner that is specific only runs for the specified project(s). A shared Runner
 can run jobs for every project that has enabled the option **Allow shared Runners**
@@ -66,7 +69,7 @@ Runners to disabled.
 
 ## Registering a specific Runner
 
-Registering a specific can be done in two ways:
+Registering a specific Runner can be done in two ways:
 
 1. Creating a Runner with the project registration token
 1. Converting a shared Runner into a specific Runner (one-way, admin only)
@@ -79,6 +82,14 @@ visit the project you want to make the Runner work for in GitLab:
 1. Go to **Settings > CI/CD** to obtain the token
 1. [Register the Runner][register]
 
+## Registering a group Runner
+
+Creating a group Runner requires Master permissions for the group. To create a
+group Runner visit the group you want to make the Runner work for in GitLab:
+
+1. Go to **Settings > CI/CD** to obtain the token
+1. [Register the Runner][register]
+
 ### Making an existing shared Runner specific
 
 If you are an admin on your GitLab instance, you can turn any shared Runner into
@@ -121,7 +132,7 @@ To enable/disable a Runner in your project:
 
 > **Note**:
 Consider that if you don't lock your specific Runner to a specific project, any
-user with Master role in you project can assign your runner to another arbitrary
+user with Master role in you project can assign your Runner to another arbitrary
 project without requiring your authorization, so use it with caution.
 
 An admin can enable/disable a specific Runner for projects:

lib/gitlab/git/repository.rb

@@ -30,6 +30,7 @@ module Gitlab
       EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze
 
       NoRepository = Class.new(StandardError)
+      InvalidRepository = Class.new(StandardError)
       InvalidBlobName = Class.new(StandardError)
       InvalidRef = Class.new(StandardError)
       GitError = Class.new(StandardError)
@@ -1584,7 +1585,7 @@ module Gitlab
 
       def checksum
         gitaly_migrate(:calculate_checksum,
-                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
+                       status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_repository_client.calculate_checksum
           else
@@ -2533,10 +2534,12 @@ module Gitlab
         output, status = run_git(args)
 
         if status.nil? || !status.zero?
-          # Empty repositories return with a non-zero status and an empty output.
-          return EMPTY_REPOSITORY_CHECKSUM if output&.empty?
+          # Non-valid git repositories return 128 as the status code and an error output
+          raise InvalidRepository if status == 128 && output.to_s.downcase =~ /not a git repository/
+          # Empty repositories returns with a non-zero status and an empty output.
+          raise ChecksumError, output unless output.blank?
 
-          raise ChecksumError, output
+          return EMPTY_REPOSITORY_CHECKSUM
         end
 
         refs = output.split("\n")

lib/gitlab/gitaly_client/repository_service.rb

@@ -292,6 +292,8 @@ module Gitlab
         request  = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo)
         response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request)
         response.checksum.presence
+      rescue GRPC::DataLoss => e
+        raise Gitlab::Git::Repository::InvalidRepository.new(e)
       end
 
       def raw_changes_between(from, to)

spec/helpers/projects_helper_spec.rb

@@ -283,7 +283,7 @@ describe ProjectsHelper do
     end
 
     it 'removes the repo path' do
-      repo = "#{storage_path}/namespace/test.git"
+      repo = File.join(storage_path, 'namespace/test.git')
       import_error = "Could not clone #{repo}\n"
 
       expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')

spec/javascripts/ide/lib/common/model_spec.js

@@ -28,6 +28,10 @@ describe('Multi-file editor library model', () => {
     expect(model.originalModel).not.toBeNull();
     expect(model.model).not.toBeNull();
     expect(model.baseModel).not.toBeNull();
+
+    expect(model.originalModel.uri.path).toBe('original/path--path');
+    expect(model.model.uri.path).toBe('path--path');
+    expect(model.baseModel.uri.path).toBe('target/path--path');
   });
 
   it('creates model with head file to compare against', () => {

spec/javascripts/ide/stores/actions/file_spec.js

@@ -569,6 +569,22 @@ describe('IDE store file actions', () => {
         .catch(done.fail);
     });
 
+    it('returns false when already opened', done => {
+      store.state.openFiles.push({
+        ...f,
+        active: true,
+        key: `pending-${f.key}`,
+      });
+
+      store
+        .dispatch('openPendingTab', { file: f, keyPrefix: 'pending' })
+        .then(added => {
+          expect(added).toBe(false);
+        })
+        .then(done)
+        .catch(done.fail);
+    });
+
     it('pushes router URL when added', done => {
       store.state.currentBranchId = 'master';
 

spec/lib/gitlab/git/repository_spec.rb

@@ -2275,7 +2275,22 @@ describe Gitlab::Git::Repository, seed_helper: true do
         expect(empty_repo.checksum).to eq '0000000000000000000000000000000000000000'
       end
 
-      it 'raises a no repository exception when there is no repo' do
+      it 'raises Gitlab::Git::Repository::InvalidRepository error for non-valid git repo' do
+        FileUtils.rm_rf(File.join(storage_path, 'non-valid.git'))
+
+        system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --bare #{TEST_REPO_PATH} non-valid.git),
+               chdir: SEED_STORAGE_PATH,
+               out: '/dev/null',
+               err: '/dev/null')
+
+        File.truncate(File.join(storage_path, 'non-valid.git/HEAD'), 0)
+
+        non_valid = described_class.new('default', 'non-valid.git', '')
+
+        expect { non_valid.checksum }.to raise_error(Gitlab::Git::Repository::InvalidRepository)
+      end
+
+      it 'raises Gitlab::Git::Repository::NoRepository error when there is no repo' do
         broken_repo = described_class.new('default', 'a/path.git', '')
 
         expect { broken_repo.checksum }.to raise_error(Gitlab::Git::Repository::NoRepository)

spec/migrations/add_pipeline_build_foreign_key_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20180420010016_add_pipeline_build_foreign_key.rb')
+
+describe AddPipelineBuildForeignKey, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:builds) { table(:ci_builds) }
+
+  before do
+    namespaces.create(id: 10, name: 'gitlab-org', path: 'gitlab-org')
+    projects.create!(id: 11, namespace_id: 10, name: 'gitlab', path: 'gitlab')
+    pipelines.create!(id: 12, project_id: 11, ref: 'master', sha: 'adf43c3a')
+
+    builds.create!(id: 101, commit_id: 12, project_id: 11)
+    builds.create!(id: 102, commit_id: 222, project_id: 11)
+    builds.create!(id: 103, commit_id: 333, project_id: 11)
+    builds.create!(id: 104, commit_id: 12, project_id: 11)
+    builds.create!(id: 106, commit_id: nil, project_id: 11)
+    builds.create!(id: 107, commit_id: 12, project_id: nil)
+  end
+
+  it 'adds foreign key after removing orphans' do
+    expect(builds.all.count).to eq 6
+    expect(foreign_key_exists?(:ci_builds, :ci_pipelines, column: :commit_id)).to be_falsey
+
+    migrate!
+
+    expect(builds.all.pluck(:id)).to eq [101, 104]
+    expect(foreign_key_exists?(:ci_builds, :ci_pipelines, column: :commit_id)).to be_truthy
+  end
+end

spec/models/clusters/applications/runner_spec.rb

@@ -74,9 +74,8 @@ describe Clusters::Applications::Runner do
 
       it 'assigns the new runner to runner' do
         subject
-        gitlab_runner.reload
 
-        expect(gitlab_runner.runner).not_to be_nil
+        expect(gitlab_runner.reload.runner).to be_project_type
       end
     end
 

spec/support/helpers/migrations_helpers.rb

@@ -24,6 +24,16 @@ module MigrationsHelpers
     end
   end
 
+  def foreign_key_exists?(source, target = nil, column: nil)
+    ActiveRecord::Base.connection.foreign_keys(source).any? do |key|
+      if column
+        key.options[:column].to_s == column.to_s
+      else
+        key.to_table.to_s == target.to_s
+      end
+    end
+  end
+
   def reset_column_in_all_models
     clear_schema_cache!
 

spec/support/services/migrate_to_ghost_user_service_shared_examples.rb

@@ -86,7 +86,7 @@ shared_examples "migrating a deleted user's associated records to the ghost user
       end
 
       it "blocks the user before #{record_class_name} migration begins" do
-        expect(service).to receive("migrate_#{record_class_name.parameterize('_')}s".to_sym) do
+        expect(service).to receive("migrate_#{record_class_name.parameterize('_').pluralize}".to_sym) do
           expect(user.reload).to be_blocked
         end
 

vendor/gitignore/Global/JetBrains.gitignore

@@ -4,7 +4,8 @@
 # User-specific stuff
 .idea/**/workspace.xml
 .idea/**/tasks.xml
-.idea/dictionaries
+.idea/**/dictionaries
+.idea/**/shelf
 
 # Sensitive or high-churn files
 .idea/**/dataSources/

vendor/gitignore/Global/Vim.gitignore

@@ -12,3 +12,5 @@ Session.vim
 *~
 # Auto-generated tag files
 tags
+# Persistent undo
+[._]*.un~

vendor/gitignore/Java.gitignore

@@ -13,6 +13,7 @@
 # Package Files #
 *.jar
 *.war
+*.nar
 *.ear
 *.zip
 *.tar.gz

vendor/gitignore/Objective-C.gitignore

@@ -52,7 +52,7 @@ Carthage/Build
 
 fastlane/report.xml
 fastlane/Preview.html
-fastlane/screenshots
+fastlane/screenshots/**/*.png
 fastlane/test_output
 
 # Code Injection

vendor/gitignore/Swift.gitignore

@@ -64,5 +64,5 @@ Carthage/Build
 
 fastlane/report.xml
 fastlane/Preview.html
-fastlane/screenshots
+fastlane/screenshots/**/*.png
 fastlane/test_output

vendor/gitignore/TeX.gitignore

@@ -198,6 +198,9 @@ pythontex-files-*/
 # easy-todo
 *.lod
 
+# xmpincl
+*.xmpi
+
 # xindy
 *.xdy
 
@@ -234,3 +237,6 @@ TSWLatexianTemp*
 
 # standalone packages
 *.sta
+
+# generated if using elsarticle.cls
+*.spl

vendor/gitignore/Unity.gitignore

@@ -6,7 +6,7 @@
 Assets/AssetStoreTools*
 
 # Visual Studio cache directory
-/.vs/
+.vs/
 
 # Autogenerated VS/MD/Consulo solution and project files
 ExportedObj/
@@ -22,6 +22,7 @@ ExportedObj/
 *.booproj
 *.svd
 *.pdb
+*.opendb
 
 # Unity3D generated meta files
 *.pidb.meta

vendor/gitignore/VisualStudio.gitignore

@@ -240,6 +240,7 @@ Backup*/
 UpgradeLog*.XML
 UpgradeLog*.htm
 ServiceFabricBackup/
+*.rptproj.bak
 
 # SQL Server files
 *.mdf

vendor/gitlab-ci-yml/Chef.gitlab-ci.yml

@@ -7,7 +7,7 @@
 
 image: "chef/chefdk"
 services:
-  - docker:stable-dind
+  - docker:dind
 
 variables:
   DOCKER_HOST: "tcp://docker:2375"

vendor/gitlab-ci-yml/Docker.gitlab-ci.yml

@@ -2,7 +2,7 @@
 image: docker:latest
 
 services:
-  - docker:stable-dind
+  - docker:dind
 
 before_script:
   - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

vendor/gitlab-ci-yml/Maven.gitlab-ci.yml

@@ -24,7 +24,7 @@ variables:
   MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
 
 # Cache downloaded dependencies and plugins between builds.
-# To keep cache across branches add 'key: "$CI_JOB_REF_NAME"'
+# To keep cache across branches add 'key: "$CI_JOB_NAME"'
 cache:
   paths:
     - .m2/repository

vendor/gitlab-ci-yml/Swift.gitlab-ci.yml

 # Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/
-# This file assumes an own GitLab CI runner, setup on an OS X system.
+# This file assumes an own GitLab CI runner, setup on an macOS system.
 stages:
   - build
   - archive
@@ -8,11 +8,11 @@ build_project:
   stage: build
   script:
     - xcodebuild clean -project ProjectName.xcodeproj -scheme SchemeName | xcpretty
-    - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 6s,OS=9.2' | xcpretty -s
+    - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 8,OS=11.3' | xcpretty -s
   tags:
-    - ios_9-2
-    - xcode_7-2
-    - osx_10-11
+    - ios_11-3
+    - xcode_9-3
+    - macos_10-13
 
 archive_project:
   stage: archive
@@ -25,6 +25,6 @@ archive_project:
     paths:
     - build/ProjectName.ipa
   tags:
-    - ios_9-2
-    - xcode_7-2
-    - osx_10-11
+    - ios_11-3
+    - xcode_9-3
+    - macos_10-13