BigW Consortium Gitlab

pipeline.rb 14.8 KB
Newer Older
1
module Ci
2
  class Pipeline < ActiveRecord::Base
3
    extend Gitlab::Ci::Model
4
    include HasStatus
5
    include Importable
6
    include AfterCommitQueue
7
    include Presentable
8
    include Gitlab::OptimisticLocking
Kamil Trzcinski committed
9

10
    belongs_to :project, inverse_of: :pipelines
11
    belongs_to :user
12
    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
13
    belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
14

15
    has_many :stages
16
    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
17
    has_many :builds, foreign_key: :commit_id
18
    has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
Shinya Maeda committed
19
    has_many :variables, class_name: 'Ci::PipelineVariable'
Felipe Artur committed
20 21 22

    # Merge requests for which the current pipeline is running against
    # the merge request's latest commit.
23
    has_many :merge_requests, foreign_key: "head_pipeline_id"
24

25
    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
Shinya Maeda committed
26
    has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
27
    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
28 29
    has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
30

31 32
    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
33

Douwe Maan committed
34
    delegate :id, to: :project, prefix: true
35
    delegate :full_path, to: :project, prefix: true
Douwe Maan committed
36

37
    validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
Douwe Maan committed
38 39 40
    validates :sha, presence: { unless: :importing? }
    validates :ref, presence: { unless: :importing? }
    validates :status, presence: { unless: :importing? }
41
    validate :valid_commit_sha, unless: :importing?
42

43
    after_create :keep_around_commits, unless: :importing?
Kamil Trzcinski committed
44

45 46 47 48 49 50 51 52 53 54
    enum source: {
      unknown: nil,
      push: 1,
      web: 2,
      trigger: 3,
      schedule: 4,
      api: 5,
      external: 6
    }

55
    enum config_source: {
56 57 58
      unknown_source: nil,
      repository_source: 1,
      auto_devops_source: 2
59 60
    }

61
    enum failure_reason: {
62 63
      unknown_failure: 0,
      config_error: 1
64 65
    }

66
    state_machine :status, initial: :created do
67
      event :enqueue do
68 69
        transition [:created, :skipped] => :pending
        transition [:success, :failed, :canceled] => :running
70 71 72
      end

      event :run do
73
        transition any - [:running] => :running
74 75
      end

76
      event :skip do
77
        transition any - [:skipped] => :skipped
78 79 80
      end

      event :drop do
81
        transition any - [:failed] => :failed
82 83
      end

84
      event :succeed do
85
        transition any - [:success] => :success
86 87 88
      end

      event :cancel do
89
        transition any - [:canceled] => :canceled
90 91
      end

92
      event :block do
93
        transition any - [:manual] => :manual
94 95
      end

96 97 98 99
      # IMPORTANT
      # Do not add any operations to this state_machine
      # Create a separate worker for each new operation

100
      before_transition [:created, :pending] => :running do |pipeline|
101
        pipeline.started_at = Time.now
102 103
      end

104
      before_transition any => [:success, :failed, :canceled] do |pipeline|
105
        pipeline.finished_at = Time.now
106 107 108
        pipeline.update_duration
      end

109 110 111 112
      before_transition any => [:manual] do |pipeline|
        pipeline.update_duration
      end

Lin Jen-Shin committed
113
      before_transition canceled: any - [:canceled] do |pipeline|
Lin Jen-Shin committed
114 115 116
        pipeline.auto_canceled_by = nil
      end

117 118 119 120 121 122
      before_transition any => :failed do |pipeline, transition|
        transition.args.first.try do |reason|
          pipeline.failure_reason = reason
        end
      end

123
      after_transition [:created, :pending] => :running do |pipeline|
124
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
125 126 127
      end

      after_transition any => [:success] do |pipeline|
128
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
129 130
      end

131
      after_transition [:created, :pending, :running] => :success do |pipeline|
132
        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
133
      end
134 135

      after_transition do |pipeline, transition|
136 137 138
        next if transition.loopback?

        pipeline.run_after_commit do
139
          PipelineHooksWorker.perform_async(pipeline.id)
140
          ExpirePipelineCacheWorker.perform_async(pipeline.id)
141
        end
142
      end
143

144
      after_transition any => [:success, :failed] do |pipeline|
145
        pipeline.run_after_commit do
146
          PipelineNotificationWorker.perform_async(pipeline.id)
147
        end
148
      end
149 150
    end

151
    scope :internal, -> { where(source: internal_sources) }
152

153 154 155 156 157 158 159 160 161
    # Returns the pipelines in descending order (= newest first), optionally
    # limited to a number of references.
    #
    # ref - The name (or names) of the branch(es)/tag(s) to limit the list of
    #       pipelines to.
    def self.newest_first(ref = nil)
      relation = order(id: :desc)

      ref ? relation.where(ref: ref) : relation
162
    end
163

164
    def self.latest_status(ref = nil)
165
      newest_first(ref).pluck(:status).first
166 167
    end

168
    def self.latest_successful_for(ref)
169
      newest_first(ref).success.take
170 171
    end

172
    def self.latest_successful_for_refs(refs)
173 174 175
      relation = newest_first(refs).success

      relation.each_with_object({}) do |pipeline, hash|
176 177 178 179
        hash[pipeline.ref] ||= pipeline
      end
    end

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
    # Returns a Hash containing the latest pipeline status for every given
    # commit.
    #
    # The keys of this Hash are the commit SHAs, the values the statuses.
    #
    # commits - The list of commit SHAs to get the status for.
    # ref - The ref to scope the data to (e.g. "master"). If the ref is not
    #       given we simply get the latest status for the commits, regardless
    #       of what refs their pipelines belong to.
    def self.latest_status_per_commit(commits, ref = nil)
      p1 = arel_table
      p2 = arel_table.alias

      # This LEFT JOIN will filter out all but the newest row for every
      # combination of (project_id, sha) or (project_id, sha, ref) if a ref is
      # given.
      cond = p1[:sha].eq(p2[:sha])
        .and(p1[:project_id].eq(p2[:project_id]))
        .and(p1[:id].lt(p2[:id]))

      cond = cond.and(p1[:ref].eq(p2[:ref])) if ref
      join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond)

      relation = select(:sha, :status)
        .where(sha: commits)
        .where(p2[:id].eq(nil))
        .joins(join.join_sources)

      relation = relation.where(ref: ref) if ref

      relation.each_with_object({}) do |row, hash|
        hash[row[:sha]] = row[:status]
      end
    end

215 216 217 218
    def self.truncate_sha(sha)
      sha[0...8]
    end

219
    def self.total_duration
220
      where.not(duration: nil).sum(:duration)
221 222
    end

223 224 225 226
    def self.internal_sources
      sources.reject { |source| source == "external" }.values
    end

227 228
    def stages_count
      statuses.select(:stage).distinct.count
229 230
    end

231 232 233 234
    def total_size
      statuses.count(:id)
    end

235
    def stages_names
236 237
      statuses.order(:stage_idx).distinct
        .pluck(:stage, :stage_idx).map(&:first)
238 239
    end

240
    def legacy_stage(name)
241
      stage = Ci::LegacyStage.new(self, name: name)
242 243 244 245
      stage unless stage.statuses_count.zero?
    end

    def legacy_stages
246 247
      # TODO, this needs refactoring, see gitlab-ce#26481.

248 249
      stages_query = statuses
        .group('stage').select(:stage).order('max(stage_idx)')
250

251 252
      status_sql = statuses.latest.where('stage=sg.stage').status_sql

253
      warnings_sql = statuses.latest.select('COUNT(*)')
254
        .where('stage=sg.stage').failed_but_allowed.to_sql
255

256 257
      stages_with_statuses = CommitStatus.from(stages_query, :sg)
        .pluck('sg.stage', status_sql, "(#{warnings_sql})")
258 259

      stages_with_statuses.map do |stage|
260
        Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
261 262 263
      end
    end

264
    def valid_commit_sha
265
      if self.sha == Gitlab::Git::BLANK_SHA
266 267 268 269 270
        self.errors.add(:sha, " cant be 00000000 (branch removal)")
      end
    end

    def git_author_name
271
      commit.try(:author_name)
272 273 274
    end

    def git_author_email
275
      commit.try(:author_email)
276 277 278
    end

    def git_commit_message
279
      commit.try(:message)
280 281
    end

282 283 284 285
    def git_commit_title
      commit.try(:title)
    end

286
    def short_sha
287
      Ci::Pipeline.truncate_sha(sha)
288 289
    end

290 291 292 293
    # NOTE: This is loaded lazily and will never be nil, even if the commit
    # cannot be found.
    #
    # Use constructs like: `pipeline.commit.present?`
294
    def commit
295
      @commit ||= Commit.lazy(project, sha)
296 297
    end

298 299 300 301
    def branch?
      !tag?
    end

302
    def stuck?
303
      pending_builds.any?(&:stuck?)
304 305
    end

306
    def retryable?
307
      retryable_builds.any?
308 309
    end

310
    def cancelable?
311
      cancelable_statuses.any?
312 313
    end

Lin Jen-Shin committed
314 315 316 317
    def auto_canceled?
      canceled? && auto_canceled_by_id?
    end

318
    def cancel_running
319
      retry_optimistic_lock(cancelable_statuses) do |cancelable|
320 321 322
        cancelable.find_each do |job|
          yield(job) if block_given?
          job.cancel
323
        end
324
      end
325 326
    end

327 328 329 330 331
    def auto_cancel_running(pipeline)
      update(auto_canceled_by: pipeline)

      cancel_running do |job|
        job.auto_canceled_by = pipeline
332
      end
333 334
    end

335
    def retry_failed(current_user)
336 337
      Ci::RetryPipelineService.new(project, current_user)
        .execute(self)
338 339
    end

340
    def mark_as_processable_after_stage(stage_idx)
341
      builds.skipped.after_stage(stage_idx).find_each(&:process)
342 343
    end

Kamil Trzcinski committed
344
    def latest?
345
      return false unless ref && commit.present?
346

347
      project.commit(ref) == commit
Kamil Trzcinski committed
348 349
    end

350 351
    def retried
      @retried ||= (statuses.order(id: :desc) - statuses.latest)
352 353 354
    end

    def coverage
355
      coverage_array = statuses.latest.map(&:coverage).compact
356 357
      if coverage_array.size >= 1
        '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
358 359 360
      end
    end

361
    def stage_seeds
362 363
      return [] unless config_processor

364
      @stage_seeds ||= config_processor.stage_seeds(self)
365 366
    end

367 368 369 370
    def seeds_size
      @seeds_size ||= stage_seeds.sum(&:size)
    end

371
    def has_kubernetes_active?
372
      project.deployment_platform&.active?
373 374
    end

375 376
    def has_stage_seeds?
      stage_seeds.any?
377 378
    end

Connor Shea committed
379
    def has_warnings?
380
      builds.latest.failed_but_allowed.any?
381 382
    end

383
    def set_config_source
384 385 386 387 388
      if ci_yaml_from_repo
        self.config_source = :repository_source
      elsif implied_ci_yaml_file
        self.config_source = :auto_devops_source
      end
389 390
    end

391
    def config_processor
392
      return unless ci_yaml_file
393 394 395
      return @config_processor if defined?(@config_processor)

      @config_processor ||= begin
396
        Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
397
      rescue Gitlab::Ci::YamlProcessor::ValidationError, Psych::SyntaxError => e
398
        self.yaml_errors = e.message
399 400
        nil
      rescue
401
        self.yaml_errors = 'Undefined error'
402 403
        nil
      end
404 405
    end

406
    def ci_yaml_file_path
407
      if project.ci_config_path.blank?
408 409
        '.gitlab-ci.yml'
      else
410
        project.ci_config_path
411 412 413
      end
    end

414
    def ci_yaml_file
415 416
      return @ci_yaml_file if defined?(@ci_yaml_file)

417
      @ci_yaml_file =
418
        if auto_devops_source?
419
          implied_ci_yaml_file
420 421
        else
          ci_yaml_from_repo
422
        end
423 424 425 426 427 428

      if @ci_yaml_file
        @ci_yaml_file
      else
        self.yaml_errors = "Failed to load CI/CD config file for #{sha}"
        nil
429
      end
430 431
    end

432 433 434 435
    def has_yaml_errors?
      yaml_errors.present?
    end

Kamil Trzcinski committed
436 437 438 439
    def environments
      builds.where.not(environment: nil).success.pluck(:environment).uniq
    end

440 441 442 443 444 445 446 447 448 449 450 451 452
    # Manually set the notes for a Ci::Pipeline
    # There is no ActiveRecord relation between Ci::Pipeline and notes
    # as they are related to a commit sha. This method helps importing
    # them using the +Gitlab::ImportExport::RelationFactory+ class.
    def notes=(notes)
      notes.each do |note|
        note[:id] = nil
        note[:commit_id] = sha
        note[:noteable_id] = self['id']
        note.save!
      end
    end

453
    def notes
454
      project.notes.for_commit_id(sha)
455 456
    end

457 458 459
    def process!
      Ci::ProcessPipelineService.new(project, user).execute(self)
    end
460

461
    def update_status
462
      retry_optimistic_lock(self) do
Kamil Trzcinski committed
463
        case latest_builds_status
464 465 466 467 468 469
        when 'pending' then enqueue
        when 'running' then run
        when 'success' then succeed
        when 'failed' then drop
        when 'canceled' then cancel
        when 'skipped' then skip
470
        when 'manual' then block
471
        end
472
      end
473 474
    end

475 476
    def predefined_variables
      [
477
        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
478 479
        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true },
        { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
480 481 482
      ]
    end

483 484 485 486 487 488 489
    def queued_duration
      return unless started_at

      seconds = (started_at - created_at).to_i
      seconds unless seconds.zero?
    end

490
    def update_duration
491 492
      return unless started_at

493
      self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
494 495 496
    end

    def execute_hooks
497 498 499
      data = pipeline_data
      project.execute_hooks(data, :pipeline_hooks)
      project.execute_services(data, :pipeline_hooks)
500 501
    end

502 503
    # All the merge requests for which the current pipeline runs/ran against
    def all_merge_requests
504
      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
505 506
    end

507
    def detailed_status(current_user)
508 509 510
      Gitlab::Ci::Status::Pipeline::Factory
        .new(self, current_user)
        .fabricate!
511 512
    end

513
    def latest_builds_with_artifacts
514 515 516 517
      # We purposely cast the builds to an Array here. Because we always use the
      # rows if there are more than 0 this prevents us from having to run two
      # queries: one to get the count and one to get the rows.
      @latest_builds_with_artifacts ||= builds.latest.with_artifacts.to_a
518 519
    end

520 521
    private

522
    def ci_yaml_from_repo
523 524 525
      return unless project
      return unless sha

526
      project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
527
    rescue GRPC::NotFound, GRPC::Internal
528 529 530
      nil
    end

531
    def implied_ci_yaml_file
532 533
      return unless project

534 535 536 537 538
      if project.auto_devops_enabled?
        Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
      end
    end

539
    def pipeline_data
540
      Gitlab::DataBuilder::Pipeline.build(self)
Kamil Trzcinski committed
541
    end
542

543
    def latest_builds_status
544 545 546
      return 'failed' unless yaml_errors.blank?

      statuses.latest.status || 'skipped'
Kamil Trzcinski committed
547
    end
548 549

    def keep_around_commits
550
      return unless project
551

552 553 554
      project.repository.keep_around(self.sha)
      project.repository.keep_around(self.before_sha)
    end
555 556
  end
end