BigW Consortium Gitlab

check.rake 15.3 KB
Newer Older
1 2 3 4
# Temporary hack, until we migrate all checks to SystemCheck format
require 'system_check'
require 'system_check/helpers'

5
namespace :gitlab do
6
  desc 'GitLab | Check the configuration of GitLab and its environment'
7
  task check: %w{gitlab:gitlab_shell:check
8
                 gitlab:sidekiq:check
9
                 gitlab:incoming_email:check
10
                 gitlab:ldap:check
Riyad Preukschas committed
11 12
                 gitlab:app:check}

13
  namespace :app do
14
    desc 'GitLab | Check the configuration of the GitLab Rails app'
15
    task check: :environment  do
Riyad Preukschas committed
16 17
      warn_user_is_not_gitlab

18 19 20 21 22 23
      checks = [
        SystemCheck::App::GitConfigCheck,
        SystemCheck::App::DatabaseConfigExistsCheck,
        SystemCheck::App::MigrationsAreUpCheck,
        SystemCheck::App::OrphanedGroupMembersCheck,
        SystemCheck::App::GitlabConfigExistsCheck,
24
        SystemCheck::App::GitlabConfigUpToDateCheck,
25 26 27 28 29 30 31 32 33 34
        SystemCheck::App::LogWritableCheck,
        SystemCheck::App::TmpWritableCheck,
        SystemCheck::App::UploadsDirectoryExistsCheck,
        SystemCheck::App::UploadsPathPermissionCheck,
        SystemCheck::App::UploadsPathTmpPermissionCheck,
        SystemCheck::App::InitScriptExistsCheck,
        SystemCheck::App::InitScriptUpToDateCheck,
        SystemCheck::App::ProjectsHaveNamespaceCheck,
        SystemCheck::App::RedisVersionCheck,
        SystemCheck::App::RubyVersionCheck,
35
        SystemCheck::App::GitVersionCheck,
36
        SystemCheck::App::GitUserDefaultSSHConfigCheck,
37
        SystemCheck::App::ActiveUsersCheck
38 39 40
      ]

      SystemCheck.run('GitLab', checks)
41
    end
42 43
  end

44
  namespace :gitlab_shell do
45
    desc "GitLab | Check the configuration of GitLab Shell"
46
    task check: :environment  do
Riyad Preukschas committed
47
      warn_user_is_not_gitlab
48
      start_checking "GitLab Shell"
Riyad Preukschas committed
49

50
      check_gitlab_shell
Riyad Preukschas committed
51
      check_repo_base_exists
52
      check_repo_base_is_not_symlink
Riyad Preukschas committed
53 54
      check_repo_base_user_and_group
      check_repo_base_permissions
55
      check_repos_hooks_directory_is_link
56
      check_gitlab_shell_self_test
Riyad Preukschas committed
57

58
      finished_checking "GitLab Shell"
Riyad Preukschas committed
59 60 61 62 63 64
    end

    # Checks
    ########################

    def check_repo_base_exists
65
      puts "Repo base directory exists?"
Riyad Preukschas committed
66

67 68
      Gitlab.config.repositories.storages.each do |name, repository_storage|
        repo_base_path = repository_storage['path']
69
        print "#{name}... "
Riyad Preukschas committed
70

71
        if File.exist?(repo_base_path)
72 73 74 75 76 77 78 79 80 81 82 83 84 85
          puts "yes".color(:green)
        else
          puts "no".color(:red)
          puts "#{repo_base_path} is missing".color(:red)
          try_fixing_it(
            "This should have been created when setting up GitLab Shell.",
            "Make sure it's set correctly in config/gitlab.yml",
            "Make sure GitLab Shell is installed correctly."
          )
          for_more_information(
            see_installation_guide_section "GitLab Shell"
          )
          fix_and_rerun
        end
Riyad Preukschas committed
86 87 88
      end
    end

89
    def check_repo_base_is_not_symlink
90
      puts "Repo storage directories are symlinks?"
91

92 93
      Gitlab.config.repositories.storages.each do |name, repository_storage|
        repo_base_path = repository_storage['path']
94
        print "#{name}... "
95

96
        unless File.exist?(repo_base_path)
97
          puts "can't check because of previous errors".color(:magenta)
98
          break
99 100 101 102 103 104 105 106 107 108 109
        end

        unless File.symlink?(repo_base_path)
          puts "no".color(:green)
        else
          puts "yes".color(:red)
          try_fixing_it(
            "Make sure it's set to the real directory in config/gitlab.yml"
          )
          fix_and_rerun
        end
110 111 112
      end
    end

Riyad Preukschas committed
113
    def check_repo_base_permissions
114
      puts "Repo paths access is drwxrws---?"
Riyad Preukschas committed
115

116 117
      Gitlab.config.repositories.storages.each do |name, repository_storage|
        repo_base_path = repository_storage['path']
118
        print "#{name}... "
119

120
        unless File.exist?(repo_base_path)
121
          puts "can't check because of previous errors".color(:magenta)
122
          break
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
        end

        if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
          puts "yes".color(:green)
        else
          puts "no".color(:red)
          try_fixing_it(
            "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
            "sudo chmod -R ug-s #{repo_base_path}",
            "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
          )
          for_more_information(
            see_installation_guide_section "GitLab Shell"
          )
          fix_and_rerun
        end
Riyad Preukschas committed
139 140 141 142
      end
    end

    def check_repo_base_user_and_group
143
      gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
144
      puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
Riyad Preukschas committed
145

146 147
      Gitlab.config.repositories.storages.each do |name, repository_storage|
        repo_base_path = repository_storage['path']
148
        print "#{name}... "
149

150
        unless File.exist?(repo_base_path)
151
          puts "can't check because of previous errors".color(:magenta)
152
          break
153 154
        end

155 156 157 158
        user_id = uid_for(gitlab_shell_ssh_user)
        root_group_id = gid_for('root')
        group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)]
        if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid)
159 160 161
          puts "yes".color(:green)
        else
          puts "no".color(:red)
162
          puts "  User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue)
163
          try_fixing_it(
164
            "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}"
165 166 167 168 169 170
          )
          for_more_information(
            see_installation_guide_section "GitLab Shell"
          )
          fix_and_rerun
        end
Riyad Preukschas committed
171 172 173
      end
    end

174 175
    def check_repos_hooks_directory_is_link
      print "hooks directories in repos are links: ... "
Riyad Preukschas committed
176

177
      gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path
178

Riyad Preukschas committed
179
      unless Project.count > 0
180
        puts "can't check, you have no projects".color(:magenta)
Riyad Preukschas committed
181 182 183
        return
      end
      puts ""
184

Riyad Preukschas committed
185
      Project.find_each(batch_size: 100) do |project|
186
        print sanitized_message(project)
187
        project_hook_directory = File.join(project.repository.path_to_repo, "hooks")
188

189
        if project.empty_repo?
190
          puts "repository is empty".color(:magenta)
191 192
        elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) &&
            (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path))
193
          puts 'ok'.color(:green)
Riyad Preukschas committed
194
        else
195
          puts "wrong or missing hooks".color(:red)
196
          try_fixing_it(
197
            sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
198 199 200 201 202 203 204
            'Check the hooks_path in config/gitlab.yml',
            'Check your gitlab-shell installation'
          )
          for_more_information(
            see_installation_guide_section "GitLab Shell"
          )
          fix_and_rerun
205 206
        end
      end
207
    end
Riyad Preukschas committed
208

209
    def check_gitlab_shell_self_test
210
      gitlab_shell_repo_base = gitlab_shell_path
211 212 213
      check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
      puts "Running #{check_cmd}"
      if system(check_cmd, chdir: gitlab_shell_repo_base)
214
        puts 'gitlab-shell self-check successful'.color(:green)
215
      else
216
        puts 'gitlab-shell self-check failed'.color(:red)
217 218 219 220 221 222 223 224 225
        try_fixing_it(
          'Make sure GitLab is running;',
          'Check the gitlab-shell configuration file:',
          sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}")
        )
        fix_and_rerun
      end
    end

Riyad Preukschas committed
226 227 228
    # Helper methods
    ########################

229 230
    def gitlab_shell_path
      Gitlab.config.gitlab_shell.path
Riyad Preukschas committed
231 232
    end

233
    def gitlab_shell_version
234
      Gitlab::Shell.new.version
Riyad Preukschas committed
235 236
    end

237
    def gitlab_shell_major_version
238
      Gitlab::Shell.version_required.split('.')[0].to_i
239 240 241
    end

    def gitlab_shell_minor_version
242
      Gitlab::Shell.version_required.split('.')[1].to_i
243 244 245
    end

    def gitlab_shell_patch_version
246
      Gitlab::Shell.version_required.split('.')[2].to_i
247
    end
248
  end
249

250
  namespace :sidekiq do
251
    desc "GitLab | Check the configuration of Sidekiq"
252
    task check: :environment  do
Riyad Preukschas committed
253
      warn_user_is_not_gitlab
254
      start_checking "Sidekiq"
Riyad Preukschas committed
255

256
      check_sidekiq_running
257
      only_one_sidekiq_running
Riyad Preukschas committed
258

259
      finished_checking "Sidekiq"
Riyad Preukschas committed
260 261 262 263 264
    end

    # Checks
    ########################

265
    def check_sidekiq_running
Riyad Preukschas committed
266 267
      print "Running? ... "

268
      if sidekiq_process_count > 0
269
        puts "yes".color(:green)
Riyad Preukschas committed
270
      else
271
        puts "no".color(:red)
Riyad Preukschas committed
272
        try_fixing_it(
273
          sudo_gitlab("RAILS_ENV=production bin/background_jobs start")
Riyad Preukschas committed
274 275 276
        )
        for_more_information(
          see_installation_guide_section("Install Init Script"),
277
          "see log/sidekiq.log for possible errors"
Riyad Preukschas committed
278
        )
Riyad Preukschas committed
279
        fix_and_rerun
Riyad Preukschas committed
280 281
      end
    end
282 283

    def only_one_sidekiq_running
284 285
      process_count = sidekiq_process_count
      return if process_count.zero?
286 287

      print 'Number of Sidekiq processes ... '
288
      if process_count == 1
289
        puts '1'.color(:green)
290
      else
291
        puts "#{process_count}".color(:red)
292 293
        try_fixing_it(
          'sudo service gitlab stop',
294 295
          "sudo pkill -u #{gitlab_user} -f sidekiq",
          "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq",
296 297 298 299 300 301
          'sudo service gitlab start'
        )
        fix_and_rerun
      end
    end

302
    def sidekiq_process_count
303
      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
304
      ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
305
    end
Riyad Preukschas committed
306 307
  end

308
  namespace :incoming_email do
309 310 311 312
    desc "GitLab | Check the configuration of Reply by email"
    task check: :environment  do
      warn_user_is_not_gitlab

313
      if Gitlab.config.incoming_email.enabled
314 315 316
        checks = [
          SystemCheck::IncomingEmail::ImapAuthenticationCheck
        ]
317

318
        if Rails.env.production?
319 320
          checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
          checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
321
        else
322
          checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
323 324
        end

325
        SystemCheck.run('Reply by email', checks)
326
      else
327
        puts 'Reply by email is disabled in config/gitlab.yml'
328 329 330 331
      end
    end
  end

332
  namespace :ldap do
333
    task :check, [:limit] => :environment do |_, args|
334 335
      # Only show up to 100 results because LDAP directories can be very big.
      # This setting only affects the `rake gitlab:check` script.
336
      args.with_defaults(limit: 100)
337 338 339
      warn_user_is_not_gitlab
      start_checking "LDAP"

340
      if Gitlab::LDAP::Config.enabled?
341
        check_ldap(args.limit)
342 343 344
      else
        puts 'LDAP is disabled in config/gitlab.yml'
      end
345 346 347 348

      finished_checking "LDAP"
    end

349
    def check_ldap(limit)
350
      servers = Gitlab::LDAP::Config.providers
351

352 353
      servers.each do |server|
        puts "Server: #{server}"
354 355 356 357 358 359 360 361 362 363 364

        begin
          Gitlab::LDAP::Adapter.open(server) do |adapter|
            check_ldap_auth(adapter)

            puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"

            users = adapter.users(adapter.config.uid, '*', limit)
            users.each do |user|
              puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
            end
365
          end
366 367
        rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
          puts "Could not connect to the LDAP server: #{e.message}".color(:red)
368
        end
369
      end
370
    end
371 372 373 374

    def check_ldap_auth(adapter)
      auth = adapter.config.has_auth?

375 376 377 378 379 380 381
      message = if auth && adapter.ldap.bind
                  'Success'.color(:green)
                elsif auth
                  'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
                else
                  'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
                end
382 383 384

      puts "LDAP authentication... #{message}"
    end
385
  end
Riyad Preukschas committed
386

Vinnie Okada committed
387
  namespace :repo do
388
    desc "GitLab | Check the integrity of the repositories managed by GitLab"
Vinnie Okada committed
389
    task check: :environment do
390 391
      Gitlab.config.repositories.storages.each do |name, repository_storage|
        namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
Vinnie Okada committed
392

393 394 395 396
        namespace_dirs.each do |namespace_dir|
          repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
          repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
        end
397 398 399 400
      end
    end
  end

401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
  namespace :orphans do
    desc 'Gitlab | Check for orphaned namespaces and repositories'
    task check: :environment do
      warn_user_is_not_gitlab
      checks = [
        SystemCheck::Orphans::NamespaceCheck,
        SystemCheck::Orphans::RepositoryCheck
      ]

      SystemCheck.run('Orphans', checks)
    end

    desc 'GitLab | Check for orphaned namespaces in the repositories path'
    task check_namespaces: :environment do
      warn_user_is_not_gitlab
      checks = [SystemCheck::Orphans::NamespaceCheck]

      SystemCheck.run('Orphans', checks)
    end

    desc 'GitLab | Check for orphaned repositories in the repositories path'
    task check_repositories: :environment do
      warn_user_is_not_gitlab
      checks = [SystemCheck::Orphans::RepositoryCheck]

      SystemCheck.run('Orphans', checks)
    end
  end

430 431 432
  namespace :user do
    desc "GitLab | Check the integrity of a specific user's repositories"
    task :check_repos, [:username] => :environment do |t, args|
433
      username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
434 435 436
      user = User.find_by(username: username)
      if user
        repo_dirs = user.authorized_projects.map do |p|
437 438
          File.join(
            p.repository_storage_path,
439
            "#{p.disk_path}.git"
440 441
          )
        end
442 443 444

        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
      else
445
        puts "\nUser '#{username}' not found".color(:red)
Vinnie Okada committed
446 447 448 449
      end
    end
  end

Riyad Preukschas committed
450 451 452
  # Helper methods
  ##########################

453
  def check_gitlab_shell
454
    required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version)
455
    current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
456

457
    print "GitLab Shell version >= #{required_version} ? ... "
458
    if current_version.valid? && required_version <= current_version
459
      puts "OK (#{current_version})".color(:green)
460
    else
461
      puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
462 463
    end
  end
464

465
  def check_repo_integrity(repo_dir)
466
    puts "\nChecking repo at #{repo_dir.color(:yellow)}"
467 468 469 470 471 472 473

    git_fsck(repo_dir)
    check_config_lock(repo_dir)
    check_ref_locks(repo_dir)
  end

  def git_fsck(repo_dir)
474
    puts "Running `git fsck`".color(:yellow)
475 476 477 478
    system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
  end

  def check_config_lock(repo_dir)
479
    config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
480 481
    config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
    puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
482 483 484
  end

  def check_ref_locks(repo_dir)
485
    lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
486
    if lock_files.present?
487
      puts "Ref lock files exist:".color(:red)
488 489 490 491
      lock_files.each do |lock_file|
        puts "  #{lock_file}"
      end
    else
492
      puts "No ref lock files exist".color(:green)
493 494
    end
  end
495
end