BigW Consortium Gitlab

pipeline.rb 12.9 KB
Newer Older
1
module Ci
2
  class Pipeline < ActiveRecord::Base
3
    extend Gitlab::Ci::Model
4
    include HasStatus
5
    include Importable
6
    include AfterCommitQueue
7
    include Presentable
Kamil Trzcinski committed
8

9
    belongs_to :project
10
    belongs_to :user
11
    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
12
    belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
13

14
    has_many :stages
15
    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
16
    has_many :builds, foreign_key: :commit_id
17
    has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
Shinya Maeda committed
18
    has_many :variables, class_name: 'Ci::PipelineVariable'
Felipe Artur committed
19 20 21

    # Merge requests for which the current pipeline is running against
    # the merge request's latest commit.
22
    has_many :merge_requests, foreign_key: "head_pipeline_id"
23

24
    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
Shinya Maeda committed
25
    has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
26
    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
27 28
    has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
29

30 31
    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
32

Douwe Maan committed
33
    delegate :id, to: :project, prefix: true
34
    delegate :full_path, to: :project, prefix: true
Douwe Maan committed
35

36
    validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
Douwe Maan committed
37 38 39
    validates :sha, presence: { unless: :importing? }
    validates :ref, presence: { unless: :importing? }
    validates :status, presence: { unless: :importing? }
40
    validate :valid_commit_sha, unless: :importing?
41

42
    after_initialize :set_config_source, if: :new_record?
43
    after_create :keep_around_commits, unless: :importing?
Kamil Trzcinski committed
44

45 46 47 48 49 50 51 52 53 54
    enum source: {
      unknown: nil,
      push: 1,
      web: 2,
      trigger: 3,
      schedule: 4,
      api: 5,
      external: 6
    }

55
    enum config_source: {
56 57 58
      unknown_source: nil,
      repository_source: 1,
      auto_devops_source: 2
59 60
    }

61
    state_machine :status, initial: :created do
62
      event :enqueue do
Kamil Trzcinski committed
63
        transition created: :pending
64
        transition [:success, :failed, :canceled, :skipped] => :running
65 66 67
      end

      event :run do
68
        transition any - [:running] => :running
69 70
      end

71
      event :skip do
72
        transition any - [:skipped] => :skipped
73 74 75
      end

      event :drop do
76
        transition any - [:failed] => :failed
77 78
      end

79
      event :succeed do
80
        transition any - [:success] => :success
81 82 83
      end

      event :cancel do
84
        transition any - [:canceled] => :canceled
85 86
      end

87
      event :block do
88
        transition any - [:manual] => :manual
89 90
      end

91 92 93 94
      # IMPORTANT
      # Do not add any operations to this state_machine
      # Create a separate worker for each new operation

95
      before_transition [:created, :pending] => :running do |pipeline|
96
        pipeline.started_at = Time.now
97 98
      end

99
      before_transition any => [:success, :failed, :canceled] do |pipeline|
100
        pipeline.finished_at = Time.now
101 102 103
        pipeline.update_duration
      end

104 105 106 107
      before_transition any => [:manual] do |pipeline|
        pipeline.update_duration
      end

Lin Jen-Shin committed
108
      before_transition canceled: any - [:canceled] do |pipeline|
Lin Jen-Shin committed
109 110 111
        pipeline.auto_canceled_by = nil
      end

112
      after_transition [:created, :pending] => :running do |pipeline|
113
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
114 115 116
      end

      after_transition any => [:success] do |pipeline|
117
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
118 119
      end

120
      after_transition [:created, :pending, :running] => :success do |pipeline|
121
        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
122
      end
123 124

      after_transition do |pipeline, transition|
125 126 127
        next if transition.loopback?

        pipeline.run_after_commit do
128
          PipelineHooksWorker.perform_async(pipeline.id)
129
          ExpirePipelineCacheWorker.perform_async(pipeline.id)
130
        end
131
      end
132

133
      after_transition any => [:success, :failed] do |pipeline|
134
        pipeline.run_after_commit do
135
          PipelineNotificationWorker.perform_async(pipeline.id)
136
        end
137
      end
138 139
    end

140
    # ref can't be HEAD or SHA, can only be branch/tag name
141
    scope :latest, ->(ref = nil) do
142 143 144
      max_id = unscope(:select)
        .select("max(#{quoted_table_name}.id)")
        .group(:ref, :sha)
145

146 147 148 149 150
      if ref
        where(ref: ref, id: max_id.where(ref: ref))
      else
        where(id: max_id)
      end
151
    end
152
    scope :internal, -> { where(source: internal_sources) }
153

154 155 156 157
    def self.latest_status(ref = nil)
      latest(ref).status
    end

158
    def self.latest_successful_for(ref)
159
      success.latest(ref).order(id: :desc).first
160 161
    end

162 163 164 165 166 167
    def self.latest_successful_for_refs(refs)
      success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash|
        hash[pipeline.ref] ||= pipeline
      end
    end

168 169 170 171
    def self.truncate_sha(sha)
      sha[0...8]
    end

172
    def self.total_duration
173
      where.not(duration: nil).sum(:duration)
174 175
    end

176 177 178 179
    def self.internal_sources
      sources.reject { |source| source == "external" }.values
    end

180 181
    def stages_count
      statuses.select(:stage).distinct.count
182 183
    end

184
    def stages_names
185 186
      statuses.order(:stage_idx).distinct
        .pluck(:stage, :stage_idx).map(&:first)
187 188
    end

189
    def legacy_stage(name)
190
      stage = Ci::LegacyStage.new(self, name: name)
191 192 193 194
      stage unless stage.statuses_count.zero?
    end

    def legacy_stages
195 196
      # TODO, this needs refactoring, see gitlab-ce#26481.

197 198
      stages_query = statuses
        .group('stage').select(:stage).order('max(stage_idx)')
199

200 201
      status_sql = statuses.latest.where('stage=sg.stage').status_sql

202
      warnings_sql = statuses.latest.select('COUNT(*)')
203
        .where('stage=sg.stage').failed_but_allowed.to_sql
204

205 206
      stages_with_statuses = CommitStatus.from(stages_query, :sg)
        .pluck('sg.stage', status_sql, "(#{warnings_sql})")
207 208

      stages_with_statuses.map do |stage|
209
        Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
210 211 212
      end
    end

213
    def valid_commit_sha
214
      if self.sha == Gitlab::Git::BLANK_SHA
215 216 217 218 219
        self.errors.add(:sha, " cant be 00000000 (branch removal)")
      end
    end

    def git_author_name
220
      commit.try(:author_name)
221 222 223
    end

    def git_author_email
224
      commit.try(:author_email)
225 226 227
    end

    def git_commit_message
228
      commit.try(:message)
229 230
    end

231 232 233 234
    def git_commit_title
      commit.try(:title)
    end

235
    def short_sha
236
      Ci::Pipeline.truncate_sha(sha)
237 238
    end

239
    def commit
240
      @commit ||= project.commit(sha)
241 242 243 244
    rescue
      nil
    end

245 246 247 248
    def branch?
      !tag?
    end

249
    def stuck?
250
      pending_builds.any?(&:stuck?)
251 252
    end

253
    def retryable?
254
      retryable_builds.any?
255 256
    end

257
    def cancelable?
258
      cancelable_statuses.any?
259 260
    end

Lin Jen-Shin committed
261 262 263 264
    def auto_canceled?
      canceled? && auto_canceled_by_id?
    end

265
    def cancel_running
266
      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
267 268 269
        cancelable.find_each do |job|
          yield(job) if block_given?
          job.cancel
270
        end
271
      end
272 273
    end

274 275 276 277 278
    def auto_cancel_running(pipeline)
      update(auto_canceled_by: pipeline)

      cancel_running do |job|
        job.auto_canceled_by = pipeline
279
      end
280 281
    end

282
    def retry_failed(current_user)
283 284
      Ci::RetryPipelineService.new(project, current_user)
        .execute(self)
285 286
    end

287
    def mark_as_processable_after_stage(stage_idx)
288
      builds.skipped.after_stage(stage_idx).find_each(&:process)
289 290
    end

Kamil Trzcinski committed
291 292 293 294 295 296 297
    def latest?
      return false unless ref
      commit = project.commit(ref)
      return false unless commit
      commit.sha == sha
    end

298 299
    def retried
      @retried ||= (statuses.order(id: :desc) - statuses.latest)
300 301 302
    end

    def coverage
303
      coverage_array = statuses.latest.map(&:coverage).compact
304 305
      if coverage_array.size >= 1
        '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
306 307 308
      end
    end

309
    def stage_seeds
310 311
      return [] unless config_processor

312
      @stage_seeds ||= config_processor.stage_seeds(self)
313 314
    end

315 316
    def has_kubernetes_active?
      project.kubernetes_service&.active?
317 318
    end

319 320
    def has_stage_seeds?
      stage_seeds.any?
321 322
    end

Connor Shea committed
323
    def has_warnings?
324
      builds.latest.failed_but_allowed.any?
325 326
    end

327
    def set_config_source
328 329 330 331 332
      if ci_yaml_from_repo
        self.config_source = :repository_source
      elsif implied_ci_yaml_file
        self.config_source = :auto_devops_source
      end
333 334
    end

335
    def config_processor
336
      return unless ci_yaml_file
337 338 339
      return @config_processor if defined?(@config_processor)

      @config_processor ||= begin
340
        Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
341
      rescue Gitlab::Ci::YamlProcessor::ValidationError, Psych::SyntaxError => e
342
        self.yaml_errors = e.message
343 344
        nil
      rescue
345
        self.yaml_errors = 'Undefined error'
346 347
        nil
      end
348 349
    end

350
    def ci_yaml_file_path
351
      if project.ci_config_path.blank?
352 353
        '.gitlab-ci.yml'
      else
354
        project.ci_config_path
355 356 357
      end
    end

358
    def ci_yaml_file
359 360
      return @ci_yaml_file if defined?(@ci_yaml_file)

361
      @ci_yaml_file =
362
        if auto_devops_source?
363
          implied_ci_yaml_file
364 365
        else
          ci_yaml_from_repo
366
        end
367 368 369 370 371 372

      if @ci_yaml_file
        @ci_yaml_file
      else
        self.yaml_errors = "Failed to load CI/CD config file for #{sha}"
        nil
373
      end
374 375
    end

376 377 378 379
    def has_yaml_errors?
      yaml_errors.present?
    end

Kamil Trzcinski committed
380 381 382 383
    def environments
      builds.where.not(environment: nil).success.pluck(:environment).uniq
    end

384 385 386 387 388 389 390 391 392 393 394 395 396
    # Manually set the notes for a Ci::Pipeline
    # There is no ActiveRecord relation between Ci::Pipeline and notes
    # as they are related to a commit sha. This method helps importing
    # them using the +Gitlab::ImportExport::RelationFactory+ class.
    def notes=(notes)
      notes.each do |note|
        note[:id] = nil
        note[:commit_id] = sha
        note[:noteable_id] = self['id']
        note.save!
      end
    end

397 398 399 400
    def notes
      Note.for_commit_id(sha)
    end

401 402 403
    def process!
      Ci::ProcessPipelineService.new(project, user).execute(self)
    end
404

405
    def update_status
406
      Gitlab::OptimisticLocking.retry_lock(self) do
Kamil Trzcinski committed
407
        case latest_builds_status
408 409 410 411 412 413
        when 'pending' then enqueue
        when 'running' then run
        when 'success' then succeed
        when 'failed' then drop
        when 'canceled' then cancel
        when 'skipped' then skip
414
        when 'manual' then block
415
        end
416
      end
417 418
    end

419 420
    def predefined_variables
      [
421
        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
422 423
        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true },
        { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
424 425 426
      ]
    end

427 428 429 430 431 432 433
    def queued_duration
      return unless started_at

      seconds = (started_at - created_at).to_i
      seconds unless seconds.zero?
    end

434
    def update_duration
435 436
      return unless started_at

437
      self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
438 439 440
    end

    def execute_hooks
441 442 443
      data = pipeline_data
      project.execute_hooks(data, :pipeline_hooks)
      project.execute_services(data, :pipeline_hooks)
444 445
    end

446 447
    # All the merge requests for which the current pipeline runs/ran against
    def all_merge_requests
448
      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
449 450
    end

451
    def detailed_status(current_user)
452 453 454
      Gitlab::Ci::Status::Pipeline::Factory
        .new(self, current_user)
        .fabricate!
455 456
    end

457 458 459 460
    def latest_builds_with_artifacts
      @latest_builds_with_artifacts ||= builds.latest.with_artifacts
    end

461 462
    private

463
    def ci_yaml_from_repo
464 465 466
      return unless project
      return unless sha

467 468 469 470 471
      project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
    rescue GRPC::NotFound, Rugged::ReferenceError, GRPC::Internal
      nil
    end

472
    def implied_ci_yaml_file
473 474
      return unless project

475 476 477 478 479
      if project.auto_devops_enabled?
        Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
      end
    end

480
    def pipeline_data
481
      Gitlab::DataBuilder::Pipeline.build(self)
Kamil Trzcinski committed
482
    end
483

484
    def latest_builds_status
485 486 487
      return 'failed' unless yaml_errors.blank?

      statuses.latest.status || 'skipped'
Kamil Trzcinski committed
488
    end
489 490

    def keep_around_commits
491
      return unless project
492

493 494 495
      project.repository.keep_around(self.sha)
      project.repository.keep_around(self.before_sha)
    end
496 497
  end
end