BigW Consortium Gitlab

gitlab_projects.rb 9.44 KB
Newer Older
1 2 3 4
module Gitlab
  module Git
    class GitlabProjects
      include Gitlab::Git::Popen
5 6 7
      include Gitlab::Utils::StrongMemoize

      ShardNameNotFoundError = Class.new(StandardError)
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46

      # Absolute path to directory where repositories are stored.
      # Example: /home/git/repositories
      attr_reader :shard_path

      # Relative path is a directory name for repository with .git at the end.
      # Example: gitlab-org/gitlab-test.git
      attr_reader :repository_relative_path

      # Absolute path to the repository.
      # Example: /home/git/repositorities/gitlab-org/gitlab-test.git
      attr_reader :repository_absolute_path

      # This is the path at which the gitlab-shell hooks directory can be found.
      # It's essential for integration between git and GitLab proper. All new
      # repositories should have their hooks directory symlinked here.
      attr_reader :global_hooks_path

      attr_reader :logger

      def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:)
        @shard_path = shard_path
        @repository_relative_path = repository_relative_path

        @logger = logger
        @global_hooks_path = global_hooks_path
        @repository_absolute_path = File.join(shard_path, repository_relative_path)
        @output = StringIO.new
      end

      def output
        io = @output.dup
        io.rewind
        io.read
      end

      # Import project via git clone --bare
      # URL must be publicly cloneable
      def import_project(source, timeout)
47 48 49 50 51 52
        Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
          if is_enabled
            gitaly_import_repository(source)
          else
            git_import_repository(source, timeout)
          end
53 54 55 56
        end
      end

      def fork_repository(new_shard_path, new_repository_relative_path)
57 58 59 60 61 62
        Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled|
          if is_enabled
            gitaly_fork_repository(new_shard_path, new_repository_relative_path)
          else
            git_fork_repository(new_shard_path, new_repository_relative_path)
          end
63 64 65
        end
      end

66
      def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true)
67 68 69
        tags_option = tags ? '--tags' : '--no-tags'

        logger.info "Fetching remote #{name} for repository #{repository_absolute_path}."
70
        cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet)
71
        cmd << '--prune' if prune
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
        cmd << '--force' if force
        cmd << tags_option

        setup_ssh_auth(ssh_key, known_hosts) do |env|
          success = run_with_timeout(cmd, timeout, repository_absolute_path, env)

          unless success
            logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed."
          end

          success
        end
      end

      def push_branches(remote_name, timeout, force, branch_names)
        logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
88
        cmd = %W(#{Gitlab.config.git.bin_path} push)
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
        cmd << '--force' if force
        cmd += %W(-- #{remote_name}).concat(branch_names)

        success = run_with_timeout(cmd, timeout, repository_absolute_path)

        unless success
          logger.error("Pushing branches to remote #{remote_name} failed.")
        end

        success
      end

      def delete_remote_branches(remote_name, branch_names)
        branches = branch_names.map { |branch_name| ":#{branch_name}" }

        logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
105
        cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches)
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145

        success = run(cmd, repository_absolute_path)

        unless success
          logger.error("Pushing deleted branches to remote #{remote_name} failed.")
        end

        success
      end

      protected

      def run(*args)
        output, exitstatus = popen(*args)
        @output << output

        exitstatus&.zero?
      end

      def run_with_timeout(*args)
        output, exitstatus = popen_with_timeout(*args)
        @output << output

        exitstatus&.zero?
      rescue Timeout::Error
        @output.puts('Timed out')

        false
      end

      def mask_password_in_url(url)
        result = URI(url)
        result.password = "*****" unless result.password.nil?
        result.user = "*****" unless result.user.nil? # it's needed for oauth access_token
        result
      rescue
        url
      end

      def remove_origin_in_repo
146
        cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin)
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
        run(cmd, repository_absolute_path)
      end

      # Builds a small shell script that can be used to execute SSH with a set of
      # custom options.
      #
      # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret
      # paths with spaces in them. We trust the user not to embed single or double
      # quotes in the key or value.
      def custom_ssh_script(options = {})
        args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ')

        [
          "#!/bin/sh",
          "exec ssh #{args} \"$@\""
        ].join("\n")
      end

      # Known hosts data and private keys can be passed to gitlab-shell in the
      # environment. If present, this method puts them into temporary files, writes
      # a script that can substitute as `ssh`, setting the options to respect those
      # files, and yields: { "GIT_SSH" => "/tmp/myScript" }
      def setup_ssh_auth(key, known_hosts)
        options = {}

        if key
          key_file = Tempfile.new('gitlab-shell-key-file')
          key_file.chmod(0o400)
          key_file.write(key)
          key_file.close

          options['IdentityFile'] = key_file.path
          options['IdentitiesOnly'] = 'yes'
        end

        if known_hosts
          known_hosts_file = Tempfile.new('gitlab-shell-known-hosts')
          known_hosts_file.chmod(0o400)
          known_hosts_file.write(known_hosts)
          known_hosts_file.close

          options['StrictHostKeyChecking'] = 'yes'
          options['UserKnownHostsFile'] = known_hosts_file.path
        end

        return yield({}) if options.empty?

        script = Tempfile.new('gitlab-shell-ssh-wrapper')
        script.chmod(0o755)
        script.write(custom_ssh_script(options))
        script.close

        yield('GIT_SSH' => script.path)
      ensure
        key_file&.close!
        known_hosts_file&.close!
        script&.close!
      end
205 206 207 208 209 210 211 212 213 214

      private

      def shard_name
        strong_memoize(:shard_name) do
          shard_name_from_shard_path(shard_path)
        end
      end

      def shard_name_from_shard_path(shard_path)
215
        Gitlab.config.repositories.storages.find { |_, info| info.legacy_disk_path == shard_path }&.first ||
216 217 218
          raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'")
      end

219 220 221 222 223 224 225
      def git_import_repository(source, timeout)
        # Skip import if repo already exists
        return false if File.exist?(repository_absolute_path)

        masked_source = mask_password_in_url(source)

        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
226
        cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path})
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254

        success = run_with_timeout(cmd, timeout, nil)

        unless success
          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
          FileUtils.rm_rf(repository_absolute_path)
          return false
        end

        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)

        # The project was imported successfully.
        # Remove the origin URL since it may contain password.
        remove_origin_in_repo

        true
      end

      def gitaly_import_repository(source)
        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)

        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
        true
      rescue GRPC::BadStatus => e
        @output << e.message
        false
      end

255 256 257 258 259 260 261 262 263 264 265 266 267 268
      def git_fork_repository(new_shard_path, new_repository_relative_path)
        from_path = repository_absolute_path
        to_path = File.join(new_shard_path, new_repository_relative_path)

        # The repository cannot already exist
        if File.exist?(to_path)
          logger.error "fork-repository failed: destination repository <#{to_path}> already exists."
          return false
        end

        # Ensure the namepsace / hashed storage directory exists
        FileUtils.mkdir_p(File.dirname(to_path), mode: 0770)

        logger.info "Forking repository from <#{from_path}> to <#{to_path}>."
269
        cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path})
270 271 272 273 274 275 276 277 278 279 280 281 282

        run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path)
      end

      def gitaly_fork_repository(new_shard_path, new_repository_relative_path)
        target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil)
        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)

        Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository)
      rescue GRPC::BadStatus => e
        logger.error "fork-repository failed: #{e.message}"
        false
      end
283 284 285
    end
  end
end