Merge branch '10-4-stable-patch-2' into '10-4-stable'

app/assets/javascripts/behaviors/copy_as_gfm.js

@@ -299,6 +299,13 @@ const gfmRules = {
 
 export class CopyAsGFM {
   constructor() {
+    // iOS currently does not support clipboardData.setData(). This bug should
+    // be fixed in iOS 12, but for now we'll disable this for all iOS browsers
+    // ref: https://trac.webkit.org/changeset/222228/webkit
+    const userAgent = (typeof navigator !== 'undefined' && navigator.userAgent) || '';
+    const isIOS = /\b(iPad|iPhone|iPod)(?=;)/.test(userAgent);
+    if (isIOS) return;
+
     $(document).on('copy', '.md, .wiki', (e) => { CopyAsGFM.copyAsGFM(e, CopyAsGFM.transformGFMSelection); });
     $(document).on('copy', 'pre.code.highlight, .diff-content .line_content', (e) => { CopyAsGFM.copyAsGFM(e, CopyAsGFM.transformCodeSelection); });
     $(document).on('paste', '.js-gfm-input', CopyAsGFM.pasteGFM);

app/assets/javascripts/projects/permissions/components/settings_panel.vue

@@ -207,7 +207,7 @@ export default {
           </div>
         </div>
         <span class="help-block">{{ visibilityLevelDescription }}</span>
-        <label v-if="visibilityLevel !== visibilityOptions.PUBLIC" class="request-access">
+        <label v-if="visibilityLevel !== visibilityOptions.PRIVATE" class="request-access">
           <input
             type="hidden"
             name="project[request_access_enabled]"

app/helpers/webpack_helper.rb

@@ -2,7 +2,7 @@ require 'webpack/rails/manifest'
 
 module WebpackHelper
   def webpack_bundle_tag(bundle, force_same_domain: false)
-    javascript_include_tag(*gitlab_webpack_asset_paths(bundle, force_same_domain: true))
+    javascript_include_tag(*gitlab_webpack_asset_paths(bundle, force_same_domain: force_same_domain))
   end
 
   # override webpack-rails gem helper until changes can make it upstream

app/models/ci/build.rb

@@ -465,7 +465,7 @@ module Ci
 
       if cache && project.jobs_cache_index
         cache = cache.merge(
-          key: "#{cache[:key]}:#{project.jobs_cache_index}")
+          key: "#{cache[:key]}_#{project.jobs_cache_index}")
       end
 
       [cache]

app/models/project.rb

@@ -566,6 +566,9 @@ class Project < ActiveRecord::Base
         RepositoryForkWorker.perform_async(id,
                                            forked_from_project.repository_storage_path,
                                            forked_from_project.disk_path)
+      elsif gitlab_project_import?
+        # Do not retry on Import/Export until https://gitlab.com/gitlab-org/gitlab-ce/issues/26189 is solved.
+        RepositoryImportWorker.set(retry: false).perform_async(self.id)
       else
         RepositoryImportWorker.perform_async(self.id)
       end

app/models/project_services/emails_on_push_service.rb

@@ -2,7 +2,7 @@ class EmailsOnPushService < Service
   boolean_accessor :send_from_committer_email
   boolean_accessor :disable_diffs
   prop_accessor :recipients
-  validates :recipients, presence: true, if: :activated?
+  validates :recipients, presence: true, if: :valid_recipients?
 
   def title
     'Emails on push'

app/models/project_services/irker_service.rb

@@ -4,7 +4,7 @@ class IrkerService < Service
   prop_accessor :server_host, :server_port, :default_irc_uri
   prop_accessor :recipients, :channels
   boolean_accessor :colorize_messages
-  validates :recipients, presence: true, if: :activated?
+  validates :recipients, presence: true, if: :valid_recipients?
 
   before_validation :get_channels
 

app/models/project_services/pipelines_email_service.rb

 class PipelinesEmailService < Service
   prop_accessor :recipients
   boolean_accessor :notify_only_broken_pipelines
-  validates :recipients, presence: true, if: :activated?
+  validates :recipients, presence: true, if: :valid_recipients?
 
   def initialize_properties
     self.properties ||= { notify_only_broken_pipelines: true }

app/models/service.rb

@@ -2,6 +2,8 @@
 # and implement a set of methods
 class Service < ActiveRecord::Base
   include Sortable
+  include Importable
+
   serialize :properties, JSON # rubocop:disable Cop/ActiveRecordSerialize
 
   default_value_for :active, false
@@ -294,4 +296,8 @@ class Service < ActiveRecord::Base
       project.cache_has_external_wiki
     end
   end
+
+  def valid_recipients?
+    activated? && !importing?
+  end
 end

app/workers/repository_import_worker.rb

@@ -20,7 +20,11 @@ class RepositoryImportWorker
     # to those importers to mark the import process as complete.
     return if service.async?
 
-    raise result[:message] if result[:status] == :error
+    if result[:status] == :error
+      fail_import(project, result[:message]) if project.gitlab_project_import?
+
+      raise result[:message]
+    end
 
     project.after_import
   end
@@ -33,4 +37,8 @@ class RepositoryImportWorker
     Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while importing.")
     false
   end
+
+  def fail_import(project, message)
+    project.mark_import_as_failed(message)
+  end
 end

changelogs/unreleased/32546-cannot-copy-paste-on-ios.yml

+---
+title: Fix copy/paste on iOS devices due to a bug in webkit
+merge_request: 15804
+author:
+type: fixed

changelogs/unreleased/42022-allow-users-to-request-access-not-visible-when-project-visibility-is-public.yml

+---
+title: Fix missing "allow users to request access" option in public project permissions
+merge_request: 16485
+author:
+type: fixed

changelogs/unreleased/42161-gitaly-commitservice-encoding-undefinedconversionerror-u-c124-from-utf-8-to-ascii-8bit.yml

+---
+title: Fix encoding issue when counting commit count
+merge_request: 16637
+author:
+type: fixed

changelogs/unreleased/42327-import-from-gitlab-com-fails-destination-already-exists-and-is-not-an-empty-directory-error.yml

+---
+title: Fixes destination already exists, and some particular service errors on Import/Export
+  error
+merge_request: 16714
+author:
+type: fixed

changelogs/unreleased/fix-cache-clear-windows.yml

+---
+title: 'Fix cache clear bug withg using : on Windows'
+merge_request: 16740
+author:
+type: fixed

changelogs/unreleased/fix-postgresql-table-grant.yml

+---
+title: Use has_table_privilege for TRIGGER on PostgreSQL
+merge_request:
+author:
+type: fixed

db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb

@@ -6,6 +6,8 @@ class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
 
   # Set this constant to true if this migration requires downtime.
   DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
   disable_ddl_transaction!
 
   class DeploysKeyProject < ActiveRecord::Base
@@ -18,13 +20,22 @@ class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
     DeploysKeyProject.each_batch(of: 10_000) do |batch|
       start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
 
-      execute <<-EOF
-        UPDATE deploy_keys_projects
-        SET can_push = keys.can_push
-        FROM keys
-        WHERE deploy_key_id = keys.id
-        AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
-      EOF
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
     end
   end
 
@@ -32,13 +43,22 @@ class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
     DeploysKeyProject.each_batch(of: 10_000) do |batch|
       start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
 
-      execute <<-EOF
-        UPDATE keys
-        SET can_push = deploy_keys_projects.can_push
-        FROM deploy_keys_projects
-        WHERE deploy_keys_projects.deploy_key_id = keys.id
-        AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
-      EOF
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
     end
   end
 end

db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb

@@ -5,6 +5,8 @@ class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
   disable_ddl_transaction!
 
   class DeploysKeyProject < ActiveRecord::Base
@@ -17,13 +19,22 @@ class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
     DeploysKeyProject.each_batch(of: 10_000) do |batch|
       start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
 
-      execute <<-EOF
-        UPDATE deploy_keys_projects
-        SET can_push = keys.can_push
-        FROM keys
-        WHERE deploy_key_id = keys.id
-        AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
-      EOF
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
     end
   end
 
@@ -31,13 +42,22 @@ class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
     DeploysKeyProject.each_batch(of: 10_000) do |batch|
       start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
 
-      execute <<-EOF
-        UPDATE keys
-        SET can_push = deploy_keys_projects.can_push
-        FROM deploy_keys_projects
-        WHERE deploy_keys_projects.deploy_key_id = keys.id
-        AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
-      EOF
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
     end
   end
 end

lib/gitlab/database/grant.rb

@@ -12,30 +12,40 @@ module Gitlab
       # Returns true if the current user can create and execute triggers on the
       # given table.
       def self.create_and_execute_trigger?(table)
-        priv =
-          if Database.postgresql?
-            where(privilege_type: 'TRIGGER', table_name: table)
-              .where('grantee = user')
-          else
-            queries = [
-              Grant.select(1)
-                .from('information_schema.user_privileges')
-                .where("PRIVILEGE_TYPE = 'SUPER'")
-                .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"),
+        if Database.postgresql?
+          # We _must not_ use quote_table_name as this will produce double
+          # quotes on PostgreSQL and for "has_table_privilege" we need single
+          # quotes.
+          quoted_table = connection.quote(table)
 
-              Grant.select(1)
-                .from('information_schema.schema_privileges')
-                .where("PRIVILEGE_TYPE = 'TRIGGER'")
-                .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name)
-                .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')")
-            ]
+          begin
+            from(nil)
+              .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')")
+              .first
+          rescue ActiveRecord::StatementInvalid
+            # This error is raised when using a non-existing table name. In this
+            # case we just want to return false as a user technically can't
+            # create triggers for such a table.
+            false
+          end
+        else
+          queries = [
+            Grant.select(1)
+              .from('information_schema.user_privileges')
+              .where("PRIVILEGE_TYPE = 'SUPER'")
+              .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"),
 
-            union = SQL::Union.new(queries).to_sql
+            Grant.select(1)
+              .from('information_schema.schema_privileges')
+              .where("PRIVILEGE_TYPE = 'TRIGGER'")
+              .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name)
+              .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')")
+          ]
 
-            Grant.from("(#{union}) privs")
-          end
+          union = SQL::Union.new(queries).to_sql
 
-        priv.any?
+          Grant.from("(#{union}) privs").any?
+        end
       end
     end
   end

lib/gitlab/gitaly_client/commit_service.rb

@@ -125,11 +125,11 @@ module Gitlab
       def commit_count(ref, options = {})
         request = Gitaly::CountCommitsRequest.new(
           repository: @gitaly_repo,
-          revision: ref
+          revision: encode_binary(ref)
         )
         request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
         request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
-        request.path = options[:path] if options[:path].present?
+        request.path = encode_binary(options[:path]) if options[:path].present?
         request.max_count = options[:max_count] if options[:max_count].present?
 
         GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count

lib/gitlab/import_export/shared.rb

@@ -19,8 +19,13 @@ module Gitlab
       def error(error)
         error_out(error.message, caller[0].dup)
         @errors << error.message
+
         # Debug:
-        Rails.logger.error(error.backtrace.join("\n"))
+        if error.backtrace
+          Rails.logger.error("Import/Export backtrace: #{error.backtrace.join("\n")}")
+        else
+          Rails.logger.error("No backtrace found")
+        end
       end
 
       private

spec/lib/gitlab/gitaly_client/commit_service_spec.rb

@@ -131,6 +131,29 @@ describe Gitlab::GitalyClient::CommitService do
     end
   end
 
+  describe '#commit_count' do
+    before do
+      expect_any_instance_of(Gitaly::CommitService::Stub)
+        .to receive(:count_commits)
+        .with(gitaly_request_with_path(storage_name, relative_path),
+              kind_of(Hash))
+        .and_return([])
+    end
+
+    it 'sends a commit_count message' do
+      client.commit_count(revision)
+    end
+
+    context 'with UTF-8 params strings' do
+      let(:revision) { "branch\u011F" }
+      let(:path) { "foo/\u011F.txt" }
+
+      it 'handles string encodings correctly' do
+        client.commit_count(revision, path: path)
+      end
+    end
+  end
+
   describe '#find_commit' do
     let(:revision) { '4b825dc642cb6eb9a060e54bf8d69288fbee4904' }
     it 'sends an RPC request' do

spec/lib/gitlab/import_export/project.json

@@ -7096,7 +7096,7 @@
       "project_id": 5,
       "created_at": "2016-06-14T15:01:51.232Z",
       "updated_at": "2016-06-14T15:01:51.232Z",
-      "active": false,
+      "active": true,
       "properties": {
 
       },

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

@@ -164,6 +164,10 @@ describe Gitlab::ImportExport::ProjectTreeSaver do
         expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService')
       end
 
+      it 'saves the properties for a service' do
+        expect(saved_project_json['services'].first['properties']).to eq('one' => 'value')
+      end
+
       it 'has project feature' do
         project_feature = saved_project_json['project_feature']
         expect(project_feature).not_to be_empty
@@ -279,7 +283,7 @@ describe Gitlab::ImportExport::ProjectTreeSaver do
            commit_id: ci_build.pipeline.sha)
 
     create(:event, :created, target: milestone, project: project, author: user)
-    create(:service, project: project, type: 'CustomIssueTrackerService', category: 'issue_tracker')
+    create(:service, project: project, type: 'CustomIssueTrackerService', category: 'issue_tracker', properties: { one: 'value' })
 
     create(:project_custom_attribute, project: project)
     create(:project_custom_attribute, project: project)

spec/migrations/populate_can_push_from_deploy_keys_projects_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20171215113714_populate_can_push_from_deploy_keys_projects.rb')
+
+describe PopulateCanPushFromDeployKeysProjects, :migration do
+  let(:migration) { described_class.new }
+  let(:deploy_keys) { table(:keys) }
+  let(:deploy_keys_projects) { table(:deploy_keys_projects) }
+  let(:projects) { table(:projects) }
+
+  before do
+    deploy_keys.inheritance_column = nil
+
+    projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1')
+    (1..10).each do |index|
+      deploy_keys.create!(id: index, title: 'dummy', type: 'DeployKey', key: Spec::Support::Helpers::KeyGeneratorHelper.new(1024).generate + ' dummy@gitlab.com')
+      deploy_keys_projects.create!(id: index, deploy_key_id: index, project_id: 1)
+    end
+  end
+
+  describe '#up' do
+    it 'migrates can_push from deploy_keys to deploy_keys_projects' do
+      deploy_keys.limit(5).update_all(can_push: true)
+
+      expected = deploy_keys.order(:id).pluck(:id, :can_push)
+
+      migration.up
+
+      expect(deploy_keys_projects.order(:id).pluck(:deploy_key_id, :can_push)).to eq expected
+    end
+  end
+
+  describe '#down' do
+    it 'migrates can_push from deploy_keys_projects to deploy_keys' do
+      deploy_keys_projects.limit(5).update_all(can_push: true)
+
+      expected = deploy_keys_projects.order(:id).pluck(:deploy_key_id, :can_push)
+
+      migration.down
+
+      expect(deploy_keys.order(:id).pluck(:id, :can_push)).to eq expected
+    end
+  end
+end

spec/models/ci/build_spec.rb

@@ -270,7 +270,7 @@ describe Ci::Build do
           allow_any_instance_of(Project).to receive(:jobs_cache_index).and_return(1)
         end
 
-        it { is_expected.to be_an(Array).and all(include(key: "key:1")) }
+        it { is_expected.to be_an(Array).and all(include(key: "key_1")) }
       end
 
       context 'when project does not have jobs_cache_index' do

spec/workers/repository_import_worker_spec.rb

@@ -49,9 +49,22 @@ describe RepositoryImportWorker do
 
         expect do
           subject.perform(project.id)
-        end.to raise_error(StandardError, error)
+        end.to raise_error(RuntimeError, error)
         expect(project.reload.import_jid).not_to be_nil
       end
+
+      it 'updates the error on Import/Export' do
+        error = %q{remote: Not Found fatal: repository 'https://user:pass@test.com/root/repoC.git/' not found }
+
+        project.update_attributes(import_jid: '123', import_type: 'gitlab_project')
+        expect_any_instance_of(Projects::ImportService).to receive(:execute).and_return({ status: :error, message: error })
+
+        expect do
+          subject.perform(project.id)
+        end.to raise_error(RuntimeError, error)
+
+        expect(project.reload.import_error).not_to be_nil
+      end
     end
 
     context 'when using an asynchronous importer' do