Merge branch 'ce-gitaly-remote-mirror-prep' into 'master'

lib/gitlab/git/remote_mirror.rb

+module Gitlab
+  module Git
+    class RemoteMirror
+      def initialize(repository, ref_name)
+        @repository = repository
+        @ref_name = ref_name
+      end
+
+      def update(only_branches_matching: [], only_tags_matching: [])
+        local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching)
+        remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching)
+
+        updated_branches = changed_refs(local_branches, remote_branches)
+        push_branches(updated_branches.keys) if updated_branches.present?
+
+        delete_refs(local_branches, remote_branches)
+
+        local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching)
+        remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching)
+
+        updated_tags = changed_refs(local_tags, remote_tags)
+        @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present?
+
+        delete_refs(local_tags, remote_tags)
+      end
+
+      private
+
+      def refs_obj(refs, only_refs_matching: [])
+        refs.each_with_object({}) do |ref, refs|
+          next if only_refs_matching.present? && !only_refs_matching.include?(ref.name)
+
+          refs[ref.name] = ref
+        end
+      end
+
+      def changed_refs(local_refs, remote_refs)
+        local_refs.select do |ref_name, ref|
+          remote_ref = remote_refs[ref_name]
+
+          remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target
+        end
+      end
+
+      def push_branches(branches)
+        default_branch, branches = branches.partition do |branch|
+          @repository.root_ref == branch
+        end
+
+        # Push the default branch first so it works fine when remote mirror is empty.
+        branches.unshift(*default_branch)
+
+        @repository.push_remote_branches(@ref_name, branches)
+      end
+
+      def delete_refs(local_refs, remote_refs)
+        refs = refs_to_delete(local_refs, remote_refs)
+
+        @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present?
+      end
+
+      def refs_to_delete(local_refs, remote_refs)
+        default_branch_id = @repository.commit.id
+
+        remote_refs.select do |remote_ref_name, remote_ref|
+          next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo
+
+          remote_ref_id = remote_ref.dereferenced_target.try(:id)
+
+          remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -21,6 +21,7 @@ module Gitlab
       REBASE_WORKTREE_PREFIX = 'rebase'.freeze
       SQUASH_WORKTREE_PREFIX = 'squash'.freeze
       GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
+      GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout
 
       NoRepository = Class.new(StandardError)
       InvalidBlobName = Class.new(StandardError)
@@ -83,7 +84,7 @@ module Gitlab
       # Rugged repo object
       attr_reader :rugged
 
-      attr_reader :storage, :gl_repository, :relative_path
+      attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path
 
       # This initializer method is only used on the client side (gitlab-ce).
       # Gitaly-ruby uses a different initializer.
@@ -93,6 +94,12 @@ module Gitlab
         @gl_repository = gl_repository
 
         storage_path = Gitlab.config.repositories.storages[@storage]['path']
+        @gitlab_projects = Gitlab::Git::GitlabProjects.new(
+          storage_path,
+          relative_path,
+          global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
+          logger: Rails.logger
+        )
         @path = File.join(storage_path, @relative_path)
         @name = @relative_path.split("/").last
         @attributes = Gitlab::Git::Attributes.new(path)
@@ -1266,6 +1273,24 @@ module Gitlab
         fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
       end
 
+      def push_remote_branches(remote_name, branch_names, forced: true)
+        success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names)
+
+        success || gitlab_projects_error
+      end
+
+      def delete_remote_branches(remote_name, branch_names)
+        success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
+
+        success || gitlab_projects_error
+      end
+
+      def delete_remote_branches(remote_name, branch_names)
+        success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
+
+        success || gitlab_projects_error
+      end
+
       def gitaly_repository
         Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
       end
@@ -1944,6 +1969,10 @@ module Gitlab
       def fetch_remote(remote_name = 'origin', env: nil)
         run_git(['fetch', remote_name], env: env).last.zero?
       end
+
+      def gitlab_projects_error
+        raise CommandError, @gitlab_projects.output
+      end
     end
   end
 end

lib/gitlab/shell.rb

@@ -306,47 +306,6 @@ module Gitlab
       end
     end
 
-    # Push branch to remote repository
-    #
-    # storage - project's storage path
-    # project_name - project's disk path
-    # remote_name - remote name
-    # branch_names - remote branch names to push
-    # forced - should we use --force flag
-    #
-    # Ex.
-    #   push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature'])
-    #
-    def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true)
-      cmd = gitlab_projects(storage, "#{project_name}.git")
-
-      success = cmd.push_branches(remote_name, git_timeout, forced, branch_names)
-
-      raise Error, cmd.output unless success
-
-      success
-    end
-
-    # Delete branch from remote repository
-    #
-    # storage - project's storage path
-    # project_name - project's disk path
-    # remote_name - remote name
-    # branch_names - remote branch names
-    #
-    # Ex.
-    #   delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature'])
-    #
-    def delete_remote_branches(storage, project_name, remote_name, branch_names)
-      cmd = gitlab_projects(storage, "#{project_name}.git")
-
-      success = cmd.delete_remote_branches(remote_name, branch_names)
-
-      raise Error, cmd.output unless success
-
-      success
-    end
-
     protected
 
     def gitlab_shell_path

spec/lib/gitlab/git/repository_spec.rb

@@ -18,9 +18,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
   end
 
   let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
+  let(:storage_path) { TestEnv.repos_path }
 
   describe '.create_hooks' do
-    let(:repo_path) { File.join(TestEnv.repos_path, 'hook-test.git') }
+    let(:repo_path) { File.join(storage_path, 'hook-test.git') }
     let(:hooks_dir) { File.join(repo_path, 'hooks') }
     let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path }
     let(:existing_target) { File.join(repo_path, 'foobar') }
@@ -645,7 +646,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
 
     after do
-      Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project')
+      Gitlab::Shell.new.remove_repository(storage_path, 'my_project')
     end
 
     it 'fetches a repository as a mirror remote' do
@@ -1906,6 +1907,110 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+  describe '#gitlab_projects' do
+    subject { repository.gitlab_projects }
+
+    it { expect(subject.shard_path).to eq(storage_path) }
+    it { expect(subject.repository_relative_path).to eq(repository.relative_path) }
+  end
+
+  context 'gitlab_projects commands' do
+    let(:gitlab_projects) { repository.gitlab_projects }
+    let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }
+
+    describe '#push_remote_branches' do
+      subject do
+        repository.push_remote_branches('downstream-remote', ['master'])
+      end
+
+      it 'executes the command' do
+        expect(gitlab_projects).to receive(:push_branches)
+          .with('downstream-remote', timeout, true, ['master'])
+          .and_return(true)
+
+        is_expected.to be_truthy
+      end
+
+      it 'raises an error if the command fails' do
+        allow(gitlab_projects).to receive(:output) { 'error' }
+        expect(gitlab_projects).to receive(:push_branches)
+          .with('downstream-remote', timeout, true, ['master'])
+          .and_return(false)
+
+        expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
+      end
+    end
+
+    describe '#delete_remote_branches' do
+      subject do
+        repository.delete_remote_branches('downstream-remote', ['master'])
+      end
+
+      it 'executes the command' do
+        expect(gitlab_projects).to receive(:delete_remote_branches)
+          .with('downstream-remote', ['master'])
+          .and_return(true)
+
+        is_expected.to be_truthy
+      end
+
+      it 'raises an error if the command fails' do
+        allow(gitlab_projects).to receive(:output) { 'error' }
+        expect(gitlab_projects).to receive(:delete_remote_branches)
+          .with('downstream-remote', ['master'])
+          .and_return(false)
+
+        expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
+      end
+    end
+
+    describe '#delete_remote_branches' do
+      subject do
+        repository.delete_remote_branches('downstream-remote', ['master'])
+      end
+
+      it 'executes the command' do
+        expect(gitlab_projects).to receive(:delete_remote_branches)
+          .with('downstream-remote', ['master'])
+          .and_return(true)
+
+        is_expected.to be_truthy
+      end
+
+      it 'raises an error if the command fails' do
+        allow(gitlab_projects).to receive(:output) { 'error' }
+        expect(gitlab_projects).to receive(:delete_remote_branches)
+          .with('downstream-remote', ['master'])
+          .and_return(false)
+
+        expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
+      end
+    end
+
+    describe '#delete_remote_branches' do
+      subject do
+        repository.delete_remote_branches('downstream-remote', ['master'])
+      end
+
+      it 'executes the command' do
+        expect(gitlab_projects).to receive(:delete_remote_branches)
+          .with('downstream-remote', ['master'])
+          .and_return(true)
+
+        is_expected.to be_truthy
+      end
+
+      it 'raises an error if the command fails' do
+        allow(gitlab_projects).to receive(:output) { 'error' }
+        expect(gitlab_projects).to receive(:delete_remote_branches)
+          .with('downstream-remote', ['master'])
+          .and_return(false)
+
+        expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
+      end
+    end
+  end
+
   def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
     source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
     rugged = repository.rugged

spec/lib/gitlab/shell_spec.rb

@@ -347,62 +347,6 @@ describe Gitlab::Shell do
         end.to raise_error(Gitlab::Shell::Error, "error")
       end
     end
-
-    describe '#push_remote_branches' do
-      subject(:result) do
-        gitlab_shell.push_remote_branches(
-          project.repository_storage_path,
-          project.disk_path,
-          'downstream-remote',
-          ['master']
-        )
-      end
-
-      it 'executes the command' do
-        expect(gitlab_projects).to receive(:push_branches)
-          .with('downstream-remote', timeout, true, ['master'])
-          .and_return(true)
-
-        is_expected.to be_truthy
-      end
-
-      it 'fails to execute the command' do
-        allow(gitlab_projects).to receive(:output) { 'error' }
-        expect(gitlab_projects).to receive(:push_branches)
-          .with('downstream-remote', timeout, true, ['master'])
-          .and_return(false)
-
-        expect { result }.to raise_error(Gitlab::Shell::Error, 'error')
-      end
-    end
-
-    describe '#delete_remote_branches' do
-      subject(:result) do
-        gitlab_shell.delete_remote_branches(
-          project.repository_storage_path,
-          project.disk_path,
-          'downstream-remote',
-          ['master']
-        )
-      end
-
-      it 'executes the command' do
-        expect(gitlab_projects).to receive(:delete_remote_branches)
-          .with('downstream-remote', ['master'])
-          .and_return(true)
-
-        is_expected.to be_truthy
-      end
-
-      it 'fails to execute the command' do
-        allow(gitlab_projects).to receive(:output) { 'error' }
-        expect(gitlab_projects).to receive(:delete_remote_branches)
-          .with('downstream-remote', ['master'])
-          .and_return(false)
-
-        expect { result }.to raise_error(Gitlab::Shell::Error, 'error')
-      end
-    end
   end
 
   describe 'namespace actions' do