Merge branch 'migrate-restore-repo-to-gitaly' into 'master'

changelogs/unreleased/migrate-restore-repo-to-gitaly.yml

+---
+title: Support restoring repositories into gitaly
+merge_request:
+author:
+type: changed

lib/backup/repository.rb

@@ -73,6 +73,9 @@ module Backup
     end
 
     def prepare_directories
+      # TODO: Need to find a way to do this for gitaly
+      # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194
+
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         path = repository_storage.legacy_disk_path
         next unless File.exist?(path)
@@ -93,70 +96,65 @@ module Backup
       end
     end
 
+    def restore_custom_hooks(project)
+      # TODO: Need to find a way to do this for gitaly
+      # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
+      in_path(path_to_tars(project)) do |dir|
+        path_to_project_repo = path_to_repo(project)
+        cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
+
+        output, status = Gitlab::Popen.popen(cmd)
+        unless status.zero?
+          progress_warn(project, cmd.join(' '), output)
+        end
+      end
+    end
+
     def restore
       prepare_directories
+      gitlab_shell = Gitlab::Shell.new
       Project.find_each(batch_size: 1000) do |project|
-        progress.print " * #{display_repo_path(project)} ... "
-        path_to_project_repo = path_to_repo(project)
+        progress.print " * #{project.full_path} ... "
         path_to_project_bundle = path_to_bundle(project)
-
         project.ensure_storage_path_exists
 
-        cmd = if File.exist?(path_to_project_bundle)
-                %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo})
-              else
-                %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo})
-              end
+        restore_repo_success = nil
+        if File.exist?(path_to_project_bundle)
+          begin
+            gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists?
+            project.repository.create_from_bundle path_to_project_bundle
+            restore_repo_success = true
+          rescue => e
+            restore_repo_success = false
+            progress.puts "Error: #{e}".color(:red)
+          end
+        else
+          restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
+        end
 
-        output, status = Gitlab::Popen.popen(cmd)
-        if status.zero?
+        if restore_repo_success
           progress.puts "[DONE]".color(:green)
         else
-          progress_warn(project, cmd.join(' '), output)
+          progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
         end
 
-        in_path(path_to_tars(project)) do |dir|
-          cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-
-          output, status = Gitlab::Popen.popen(cmd)
-          unless status.zero?
-            progress_warn(project, cmd.join(' '), output)
-          end
-        end
+        restore_custom_hooks(project)
 
         wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = path_to_repo(wiki)
         path_to_wiki_bundle = path_to_bundle(wiki)
 
         if File.exist?(path_to_wiki_bundle)
-          progress.print " * #{display_repo_path(wiki)} ... "
-
-          # If a wiki bundle exists, first remove the empty repo
-          # that was initialized with ProjectWiki.new() and then
-          # try to restore with 'git clone --bare'.
-          FileUtils.rm_rf(path_to_wiki_repo)
-          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo})
-
-          output, status = Gitlab::Popen.popen(cmd)
-          if status.zero?
-            progress.puts " [DONE]".color(:green)
-          else
-            progress_warn(project, cmd.join(' '), output)
+          progress.print " * #{wiki.full_path} ... "
+          begin
+            gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists?
+            wiki.repository.create_from_bundle(path_to_wiki_bundle)
+            progress.puts "[DONE]".color(:green)
+          rescue => e
+            progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
+            progress.puts "Error #{e}".color(:red)
           end
         end
       end
-
-      progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
-      cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
-
-      output, status = Gitlab::Popen.popen(cmd)
-      if status.zero?
-        progress.puts " [DONE]".color(:green)
-      else
-        puts " [FAILED]".color(:red)
-        puts "failed: #{cmd}"
-        puts output
-      end
     end
     # rubocop:enable Metrics/AbcSize
 

spec/lib/backup/repository_spec.rb

@@ -48,14 +48,14 @@ describe Backup::Repository do
 
     describe 'command failure' do
       before do
-        allow(Gitlab::Popen).to receive(:popen).and_return(['error', 1])
+        allow_any_instance_of(Gitlab::Shell).to receive(:create_repository).and_return(false)
       end
 
       context 'hashed storage' do
         it 'shows the appropriate error' do
           subject.restore
 
-          expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} (#{project.disk_path}) - error")
+          expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository")
         end
       end
 
@@ -65,33 +65,10 @@ describe Backup::Repository do
         it 'shows the appropriate error' do
           subject.restore
 
-          expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} - error")
+          expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository")
         end
       end
     end
-
-    describe 'folders without permissions' do
-      before do
-        allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES)
-      end
-
-      it 'shows error message' do
-        expect(subject).to receive(:access_denied_error)
-        subject.restore
-      end
-    end
-
-    describe 'folder that is a mountpoint' do
-      before do
-        allow(FileUtils).to receive(:mv).and_raise(Errno::EBUSY)
-      end
-
-      it 'shows error message' do
-        expect(subject).to receive(:resource_busy_error).and_call_original
-
-        expect { subject.restore }.to raise_error(/is a mountpoint/)
-      end
-    end
   end
 
   describe '#empty_repo?' do