BigW Consortium Gitlab

shell.rb 10.2 KB
Newer Older
1 2 3
# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
# SSH key operations are not part of Gitaly so will never be migrated.

4 5
require 'securerandom'

6
module Gitlab
7
  class Shell
8 9
    GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze

10
    Error = Class.new(StandardError)
11

12
    KeyAdder = Struct.new(:io) do
13
      def add_key(id, key)
14 15 16 17 18 19
        key = Gitlab::Shell.strip_key(key)
        # Newline and tab are part of the 'protocol' used to transmit id+key to the other end
        if key.include?("\t") || key.include?("\n")
          raise Error.new("Invalid key: #{key.inspect}")
        end

20
        io.puts("#{id}\t#{key}")
21 22 23
      end
    end

24
    class << self
25 26 27 28 29 30 31 32 33 34 35 36
      def secret_token
        @secret_token ||= begin
          File.read(Gitlab.config.gitlab_shell.secret_file).chomp
        end
      end

      def ensure_secret_token!
        return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))

        generate_and_link_secret_token
      end

37
      def version_required
38 39
        @version_required ||= File.read(Rails.root
                                        .join('GITLAB_SHELL_VERSION')).strip
40
      end
41 42

      def strip_key(key)
43
        key.split(/[ ]+/)[0, 2].join(' ')
44
      end
45 46 47 48 49 50 51 52 53 54

      private

      # Create (if necessary) and link the secret token file
      def generate_and_link_secret_token
        secret_file = Gitlab.config.gitlab_shell.secret_file
        shell_path = Gitlab.config.gitlab_shell.path

        unless File.size?(secret_file)
          # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
55 56
          @secret_token = SecureRandom.hex(16)
          File.write(secret_file, @secret_token)
57 58 59 60 61 62 63
        end

        link_path = File.join(shell_path, '.gitlab_shell_secret')
        if File.exist?(shell_path) && !File.exist?(link_path)
          FileUtils.symlink(secret_file, link_path)
        end
      end
64 65
    end

66
    # Init new repository
67
    #
68
    # storage - project's storage path
69
    # name - project path with namespace
70 71
    #
    # Ex.
72
    #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
73
    #
74
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
75
    def add_repository(storage, name)
76 77 78 79
      Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
    rescue => err
      Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}")
      false
80 81
    end

82 83
    # Import repository
    #
84
    # storage - project's storage path
85 86 87
    # name - project path with namespace
    #
    # Ex.
88
    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
89
    #
90
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
91
    def import_repository(storage, name, url)
92 93
      # Timeout should be less than 900 ideally, to prevent the memory killer
      # to silently kill the process without knowing we are timing out here.
94 95 96
      cmd = [gitlab_shell_projects_path, 'import-project',
             storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]
      gitlab_shell_fast_execute_raise_error(cmd)
97 98
    end

99 100 101 102 103
    # Fetch remote for repository
    #
    # name - project path with namespace
    # remote - remote name
    # forced - should we use --force flag?
104
    # no_tags - should we use --no-tags flag?
105 106 107 108
    #
    # Ex.
    #   fetch_remote("gitlab/gitlab-ci", "upstream")
    #
109
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
110
    def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
111
      args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
112 113 114
      args << '--force' if forced
      args << '--no-tags' if no_tags

115 116 117 118 119 120 121 122 123 124 125 126 127
      vars = {}

      if ssh_auth&.ssh_import?
        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
          vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
        end

        if ssh_auth.ssh_known_hosts.present?
          vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
        end
      end

      gitlab_shell_fast_execute_raise_error(args, vars)
128 129
    end

130
    # Move repository
131
    # storage - project's storage path
132 133 134 135
    # path - project path with namespace
    # new_path - new project path with namespace
    #
    # Ex.
136
    #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
137
    #
138
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
139
    def mv_repository(storage, path, new_path)
140 141
      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
                                 storage, "#{path}.git", "#{new_path}.git"])
142 143
    end

144
    # Fork repository to new namespace
145
    # forked_from_storage - forked-from project's storage path
146
    # path - project path with namespace
147
    # forked_to_storage - forked-to project's storage path
148 149 150
    # fork_namespace - namespace for forked project
    #
    # Ex.
151
    #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
152
    #
153
    # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
154
    def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
155 156 157
      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
                                 forked_from_storage, "#{path}.git", forked_to_storage,
                                 fork_namespace])
158 159
    end

160
    # Remove repository from file system
161
    #
162
    # storage - project's storage path
163
    # name - project path with namespace
164 165
    #
    # Ex.
166
    #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
167
    #
168
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
169
    def remove_repository(storage, name)
170 171
      gitlab_shell_fast_execute([gitlab_shell_projects_path,
                                 'rm-project', storage, "#{name}.git"])
172 173
    end

174
    # Add new key to gitlab-shell
175
    #
176
    # Ex.
177
    #   add_key("key-42", "sha-rsa ...")
178
    #
179
    def add_key(key_id, key_content)
180 181
      gitlab_shell_fast_execute([gitlab_shell_keys_path,
                                 'add-key', key_id, self.class.strip_key(key_content)])
182 183
    end

184 185 186 187 188 189
    # Batch-add keys to authorized_keys
    #
    # Ex.
    #   batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") }
    def batch_add_keys(&block)
      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io|
190
        yield(KeyAdder.new(io))
191 192 193
      end
    end

194
    # Remove ssh key from gitlab shell
195 196
    #
    # Ex.
197
    #   remove_key("key-342", "sha-rsa ...")
198
    #
199
    def remove_key(key_id, key_content)
200 201 202 203
      args = [gitlab_shell_keys_path, 'rm-key', key_id]
      args << key_content if key_content

      gitlab_shell_fast_execute(args)
204 205
    end

206 207 208
    # Remove all ssh keys from gitlab shell
    #
    # Ex.
Johannes Schleifenbaum committed
209
    #   remove_all_keys
210 211
    #
    def remove_all_keys
212
      gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
213 214
    end

215 216 217
    # Add empty directory for storing repositories
    #
    # Ex.
218
    #   add_namespace("/path/to/storage", "gitlab")
219
    #
220
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
221
    def add_namespace(storage, name)
222 223 224 225
      path = full_path(storage, name)
      FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
    rescue Errno::EEXIST => e
      Rails.logger.warn("Directory exists as a file: #{e} at: #{path}")
226 227 228 229 230 231
    end

    # Remove directory from repositories storage
    # Every repository inside this directory will be removed too
    #
    # Ex.
232
    #   rm_namespace("/path/to/storage", "gitlab")
233
    #
234
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
235 236
    def rm_namespace(storage, name)
      FileUtils.rm_r(full_path(storage, name), force: true)
237 238 239 240 241
    end

    # Move namespace directory inside repositories storage
    #
    # Ex.
242
    #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
243
    #
244
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
245 246
    def mv_namespace(storage, old_name, new_name)
      return false if exists?(storage, new_name) || !exists?(storage, old_name)
247

248
      FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
249 250
    end

251
    def url_to_repo(path)
252
      Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
253
    end
254

255 256
    # Return GitLab shell version
    def version
257
      gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION"
258 259

      if File.readable?(gitlab_shell_version_file)
260
        File.read(gitlab_shell_version_file).chomp
261 262 263
      end
    end

264 265 266
    # Check if such directory exists in repositories.
    #
    # Usage:
267 268
    #   exists?(storage, 'gitlab')
    #   exists?(storage, 'gitlab/cookies.git')
269
    #
270
    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
271 272
    def exists?(storage, dir_name)
      File.exist?(full_path(storage, dir_name))
273 274
    end

275 276
    protected

277
    def gitlab_shell_path
278 279 280 281 282
      File.expand_path(Gitlab.config.gitlab_shell.path)
    end

    def gitlab_shell_hooks_path
      File.expand_path(Gitlab.config.gitlab_shell.hooks_path)
283 284
    end

285 286 287 288
    def gitlab_shell_user_home
      File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
    end

289
    def full_path(storage, dir_name)
290 291
      raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?

292
      File.join(storage, dir_name)
293 294
    end

295 296 297 298 299 300 301
    def gitlab_shell_projects_path
      File.join(gitlab_shell_path, 'bin', 'gitlab-projects')
    end

    def gitlab_shell_keys_path
      File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
    end
302 303 304 305 306 307 308 309 310 311 312 313

    private

    def gitlab_shell_fast_execute(cmd)
      output, status = gitlab_shell_fast_execute_helper(cmd)

      return true if status.zero?

      Rails.logger.error("gitlab-shell failed with error #{status}: #{output}")
      false
    end

314 315
    def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
      output, status = gitlab_shell_fast_execute_helper(cmd, vars)
316 317 318 319 320

      raise Error, output unless status.zero?
      true
    end

321 322
    def gitlab_shell_fast_execute_helper(cmd, vars = {})
      vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
323 324 325 326 327

      # Don't pass along the entire parent environment to prevent gitlab-shell
      # from wasting I/O by searching through GEM_PATH
      Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
    end
328 329
  end
end