BigW Consortium Gitlab

shell.rb 15.1 KB
Newer Older
1 2 3
# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
# SSH key operations are not part of Gitaly so will never be migrated.

4 5
require 'securerandom'

6
module Gitlab
7
  class Shell
8 9
    GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze

10
    Error = Class.new(StandardError)
11

12
    KeyAdder = Struct.new(:io) do
13
      def add_key(id, key)
14 15 16 17 18 19
        key = Gitlab::Shell.strip_key(key)
        # Newline and tab are part of the 'protocol' used to transmit id+key to the other end
        if key.include?("\t") || key.include?("\n")
          raise Error.new("Invalid key: #{key.inspect}")
        end

20
        io.puts("#{id}\t#{key}")
21 22 23
      end
    end

24
    class << self
25 26 27 28 29 30 31 32 33 34 35 36
      def secret_token
        @secret_token ||= begin
          File.read(Gitlab.config.gitlab_shell.secret_file).chomp
        end
      end

      def ensure_secret_token!
        return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))

        generate_and_link_secret_token
      end

37
      def version_required
38 39
        @version_required ||= File.read(Rails.root
                                        .join('GITLAB_SHELL_VERSION')).strip
40
      end
41 42

      def strip_key(key)
43
        key.split(/[ ]+/)[0, 2].join(' ')
44
      end
45 46 47 48 49 50 51 52 53 54

      private

      # Create (if necessary) and link the secret token file
      def generate_and_link_secret_token
        secret_file = Gitlab.config.gitlab_shell.secret_file
        shell_path = Gitlab.config.gitlab_shell.path

        unless File.size?(secret_file)
          # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
55 56
          @secret_token = SecureRandom.hex(16)
          File.write(secret_file, @secret_token)
57 58 59 60 61 62 63
        end

        link_path = File.join(shell_path, '.gitlab_shell_secret')
        if File.exist?(shell_path) && !File.exist?(link_path)
          FileUtils.symlink(secret_file, link_path)
        end
      end
64 65
    end

66
    # Init new repository
67
    #
68
    # storage - project's storage name
69
    # name - project disk path
70 71
    #
    # Ex.
72
    #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
73
    #
74
    def add_repository(storage, name)
75 76 77 78 79 80 81 82 83 84 85 86 87
      relative_path = name.dup
      relative_path << '.git' unless relative_path.end_with?('.git')

      gitaly_migrate(:create_repository) do |is_enabled|
        if is_enabled
          repository = Gitlab::Git::Repository.new(storage, relative_path, '')
          repository.gitaly_repository_client.create_repository
          true
        else
          repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path)
          Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
        end
      end
88 89 90
    rescue => err
      Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}")
      false
91 92
    end

93 94
    # Import repository
    #
95
    # storage - project's storage path
96 97
    # name - project disk path
    # url - URL to import from
98 99
    #
    # Ex.
100
    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
101
    #
102
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
103
    def import_repository(storage, name, url)
Jacob Vosmaer committed
104
      if url.start_with?('.', '/')
Jacob Vosmaer committed
105 106 107
        raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
      end

108
      # The timeout ensures the subprocess won't hang forever
109 110 111 112 113 114
      cmd = gitlab_projects(storage, "#{name}.git")
      success = cmd.import_project(url, git_timeout)

      raise Error, cmd.output unless success

      success
115 116
    end

117 118
    # Fetch remote for repository
    #
119
    # repository - an instance of Git::Repository
120
    # remote - remote name
121
    # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication
122
    # forced - should we use --force flag?
123
    # no_tags - should we use --no-tags flag?
124 125
    #
    # Ex.
126
    #   fetch_remote(my_repo, "upstream")
127
    #
128 129 130
    def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false)
      gitaly_migrate(:fetch_remote) do |is_enabled|
        if is_enabled
131
          repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout)
132 133 134
        else
          storage_path = Gitlab.config.repositories.storages[repository.storage]["path"]
          local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
135 136
        end
      end
137 138
    end

139 140 141 142
    # Move repository reroutes to mv_directory which is an alias for
    # mv_namespace. Given the underlying implementation is a move action,
    # indescriminate of what the folders might be.
    #
143
    # storage - project's storage path
144 145
    # path - project disk path
    # new_path - new project disk path
146 147
    #
    # Ex.
148
    #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
149
    #
150
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
151
    def mv_repository(storage, path, new_path)
152 153 154
      return false if path.empty? || new_path.empty?

      !!mv_directory(storage, "#{path}.git", "#{new_path}.git")
155 156
    end

157
    # Fork repository to new path
158
    # forked_from_storage - forked-from project's storage path
159
    # forked_from_disk_path - project disk path
160
    # forked_to_storage - forked-to project's storage path
161
    # forked_to_disk_path - forked project disk path
162 163
    #
    # Ex.
164
    #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci")
165
    #
166
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817
167
    def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
168 169
      gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
        .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
170 171
    end

172 173 174
    # Removes a repository from file system, using rm_diretory which is an alias
    # for rm_namespace. Given the underlying implementation removes the name
    # passed as second argument on the passed storage.
175
    #
176
    # storage - project's storage path
177
    # name - project disk path
178 179
    #
    # Ex.
180
    #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
181
    #
182
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
183
    def remove_repository(storage, name)
184 185 186 187 188 189
      return false if name.empty?

      !!rm_directory(storage, "#{name}.git")
    rescue ArgumentError => e
      Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git")
      false
190 191
    end

192
    # Add new key to gitlab-shell
193
    #
194
    # Ex.
195
    #   add_key("key-42", "sha-rsa ...")
196
    #
197
    def add_key(key_id, key_content)
198 199
      return unless self.authorized_keys_enabled?

200 201
      gitlab_shell_fast_execute([gitlab_shell_keys_path,
                                 'add-key', key_id, self.class.strip_key(key_content)])
202 203
    end

204 205 206 207 208
    # Batch-add keys to authorized_keys
    #
    # Ex.
    #   batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") }
    def batch_add_keys(&block)
209 210
      return unless self.authorized_keys_enabled?

211
      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io|
212
        yield(KeyAdder.new(io))
213 214 215
      end
    end

216
    # Remove ssh key from gitlab shell
217 218
    #
    # Ex.
219
    #   remove_key("key-342", "sha-rsa ...")
220
    #
221
    def remove_key(key_id, key_content = nil)
222 223
      return unless self.authorized_keys_enabled?

224 225 226
      args = [gitlab_shell_keys_path, 'rm-key', key_id]
      args << key_content if key_content
      gitlab_shell_fast_execute(args)
227 228
    end

229 230 231
    # Remove all ssh keys from gitlab shell
    #
    # Ex.
Johannes Schleifenbaum committed
232
    #   remove_all_keys
233 234
    #
    def remove_all_keys
235 236
      return unless self.authorized_keys_enabled?

237
      gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
238 239
    end

240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
    # Remove ssh keys from gitlab shell that are not in the DB
    #
    # Ex.
    #   remove_keys_not_found_in_db
    #
    def remove_keys_not_found_in_db
      return unless self.authorized_keys_enabled?

      Rails.logger.info("Removing keys not found in DB")

      batch_read_key_ids do |ids_in_file|
        ids_in_file.uniq!
        keys_in_db = Key.where(id: ids_in_file)

        next unless ids_in_file.size > keys_in_db.count # optimization

        ids_to_remove = ids_in_file - keys_in_db.pluck(:id)
        ids_to_remove.each do |id|
          Rails.logger.info("Removing key-#{id} not found in DB")
          remove_key("key-#{id}")
        end
      end
    end

    # Iterate over all ssh key IDs from gitlab shell, in batches
    #
    # Ex.
    #   batch_read_key_ids { |batch| keys = Key.where(id: batch) }
    #
    def batch_read_key_ids(batch_size: 100, &block)
      return unless self.authorized_keys_enabled?

      list_key_ids do |key_id_stream|
        key_id_stream.lazy.each_slice(batch_size) do |lines|
          key_ids = lines.map { |l| l.chomp.to_i }
          yield(key_ids)
        end
      end
    end

    # Stream all ssh key IDs from gitlab shell, separated by newlines
    #
    # Ex.
    #   list_key_ids
    #
    def list_key_ids(&block)
      return unless self.authorized_keys_enabled?

      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block)
    end

291 292 293
    # Add empty directory for storing repositories
    #
    # Ex.
294
    #   add_namespace("/path/to/storage", "gitlab")
295
    #
296
    def add_namespace(storage, name)
297 298 299 300 301 302 303 304
      Gitlab::GitalyClient.migrate(:add_namespace) do |enabled|
        if enabled
          gitaly_namespace_client(storage).add(name)
        else
          path = full_path(storage, name)
          FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
        end
      end
305 306
    rescue Errno::EEXIST => e
      Rails.logger.warn("Directory exists as a file: #{e} at: #{path}")
307 308
    rescue GRPC::InvalidArgument => e
      raise ArgumentError, e.message
309 310 311 312 313 314
    end

    # Remove directory from repositories storage
    # Every repository inside this directory will be removed too
    #
    # Ex.
315
    #   rm_namespace("/path/to/storage", "gitlab")
316
    #
317
    def rm_namespace(storage, name)
318 319 320 321 322 323 324 325 326
      Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled|
        if enabled
          gitaly_namespace_client(storage).remove(name)
        else
          FileUtils.rm_r(full_path(storage, name), force: true)
        end
      end
    rescue GRPC::InvalidArgument => e
      raise ArgumentError, e.message
327
    end
328
    alias_method :rm_directory, :rm_namespace
329 330 331 332

    # Move namespace directory inside repositories storage
    #
    # Ex.
333
    #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
334
    #
335
    def mv_namespace(storage, old_name, new_name)
336 337 338 339 340
      Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled|
        if enabled
          gitaly_namespace_client(storage).rename(old_name, new_name)
        else
          return false if exists?(storage, new_name) || !exists?(storage, old_name)
341

342 343 344
          FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
        end
      end
345 346
    rescue GRPC::InvalidArgument
      false
347
    end
348
    alias_method :mv_directory, :mv_namespace
349

350
    def url_to_repo(path)
351
      Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
352
    end
353

354 355
    # Return GitLab shell version
    def version
356
      gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION"
357 358

      if File.readable?(gitlab_shell_version_file)
359
        File.read(gitlab_shell_version_file).chomp
360 361 362
      end
    end

363 364 365
    # Check if such directory exists in repositories.
    #
    # Usage:
366 367
    #   exists?(storage, 'gitlab')
    #   exists?(storage, 'gitlab/cookies.git')
368
    #
369
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
370
    def exists?(storage, dir_name)
371 372 373 374 375 376 377
      Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled|
        if enabled
          gitaly_namespace_client(storage).exists?(dir_name)
        else
          File.exist?(full_path(storage, dir_name))
        end
      end
378 379
    end

380 381
    protected

382
    def gitlab_shell_path
383 384 385 386 387
      File.expand_path(Gitlab.config.gitlab_shell.path)
    end

    def gitlab_shell_hooks_path
      File.expand_path(Gitlab.config.gitlab_shell.hooks_path)
388 389
    end

390 391 392 393
    def gitlab_shell_user_home
      File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
    end

394
    def full_path(storage, dir_name)
395 396
      raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?

397
      File.join(storage, dir_name)
398 399
    end

400 401 402 403 404 405 406
    def gitlab_shell_projects_path
      File.join(gitlab_shell_path, 'bin', 'gitlab-projects')
    end

    def gitlab_shell_keys_path
      File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
    end
407

408 409 410 411 412 413 414 415
    def authorized_keys_enabled?
      # Return true if nil to ensure the authorized_keys methods work while
      # fixing the authorized_keys file during migration.
      return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil?

      Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
    end

416 417
    private

418 419 420 421 422 423 424 425
    def gitlab_projects(shard_path, disk_path)
      Gitlab::Git::GitlabProjects.new(
        shard_path,
        disk_path,
        global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
        logger: Rails.logger
      )
    end
426

427 428
    def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false)
      vars = { force: forced, tags: !no_tags }
429 430 431

      if ssh_auth&.ssh_import?
        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
432
          vars[:ssh_key] = ssh_auth.ssh_private_key
433 434 435
        end

        if ssh_auth.ssh_known_hosts.present?
436
          vars[:known_hosts] = ssh_auth.ssh_known_hosts
437 438 439
        end
      end

440 441 442 443 444 445 446
      cmd = gitlab_projects(storage_path, repository_relative_path)

      success = cmd.fetch_remote(remote, git_timeout, vars)

      raise Error, cmd.output unless success

      success
447 448
    end

449 450 451 452 453 454 455 456 457
    def gitlab_shell_fast_execute(cmd)
      output, status = gitlab_shell_fast_execute_helper(cmd)

      return true if status.zero?

      Rails.logger.error("gitlab-shell failed with error #{status}: #{output}")
      false
    end

458 459
    def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
      output, status = gitlab_shell_fast_execute_helper(cmd, vars)
460 461

      raise Error, output unless status.zero?
462

463 464 465
      true
    end

466 467
    def gitlab_shell_fast_execute_helper(cmd, vars = {})
      vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
468 469 470 471 472

      # Don't pass along the entire parent environment to prevent gitlab-shell
      # from wasting I/O by searching through GEM_PATH
      Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
    end
473

474 475 476 477 478 479 480 481
    def gitaly_namespace_client(storage_path)
      storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
        value['path'] == storage_path
      end

      Gitlab::GitalyClient::NamespaceService.new(storage)
    end

482 483 484 485
    def git_timeout
      Gitlab.config.gitlab_shell.git_timeout
    end

486 487 488 489 490 491 492
    def gitaly_migrate(method, &block)
      Gitlab::GitalyClient.migrate(method, &block)
    rescue GRPC::NotFound, GRPC::BadStatus => e
      # Old Popen code returns [Error, output] to the caller, so we
      # need to do the same here...
      raise Error, e
    end
493 494
  end
end