BigW Consortium Gitlab

stuck_import_jobs_worker.rb 1.71 KB
Newer Older
1 2 3 4
class StuckImportJobsWorker
  include Sidekiq::Worker
  include CronjobQueue

5
  IMPORT_JOBS_EXPIRATION = 15.hours.to_i
6 7

  def perform
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
    projects_without_jid_count = mark_projects_without_jid_as_failed!
    projects_with_jid_count = mark_projects_with_jid_as_failed!

    Gitlab::Metrics.add_event(:stuck_import_jobs,
                             projects_without_jid_count: projects_without_jid_count,
                             projects_with_jid_count: projects_with_jid_count)
  end

  private

  def mark_projects_without_jid_as_failed!
    started_projects_without_jid.each do |project|
      project.mark_import_as_failed(error_message)
    end.count
  end

  def mark_projects_with_jid_as_failed!
    completed_jids_count = 0

    started_projects_with_jid.find_in_batches(batch_size: 500) do |group|
28 29 30
      jids = group.map(&:import_jid)

      # Find the jobs that aren't currently running or that exceeded the threshold.
31
      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set
32 33

      if completed_jids.any?
34 35 36 37
        completed_jids_count += completed_jids.count
        group.each do |project|
          project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid)
        end
38

39
        Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}")
40 41 42
      end
    end

43 44
    completed_jids_count
  end
45

46 47
  def started_projects
    Project.with_import_status(:started)
48 49
  end

50 51 52
  def started_projects_with_jid
    started_projects.where.not(import_jid: nil)
  end
53

54 55
  def started_projects_without_jid
    started_projects.where(import_jid: nil)
56 57 58
  end

  def error_message
59
    "Import timed out. Import took longer than #{IMPORT_JOBS_EXPIRATION} seconds"
60 61
  end
end