BigW Consortium Gitlab

pipeline.rb 11.8 KB
Newer Older
1
module Ci
2
  class Pipeline < ActiveRecord::Base
3
    extend Ci::Model
4
    include HasStatus
5
    include Importable
6
    include AfterCommitQueue
7
    include Presentable
Kamil Trzcinski committed
8

9
    belongs_to :project
10
    belongs_to :user
11
    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
12
    belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
13

14
    has_many :stages
15
    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
16
    has_many :builds, foreign_key: :commit_id
17
    has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
Shinya Maeda committed
18
    has_many :variables, class_name: 'Ci::PipelineVariable'
Felipe Artur committed
19 20 21

    # Merge requests for which the current pipeline is running against
    # the merge request's latest commit.
22
    has_many :merge_requests, foreign_key: "head_pipeline_id"
23

24
    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
Shinya Maeda committed
25
    has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
26
    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
27 28
    has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
29

30 31
    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
32

Douwe Maan committed
33 34
    delegate :id, to: :project, prefix: true

35
    validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
Douwe Maan committed
36 37 38
    validates :sha, presence: { unless: :importing? }
    validates :ref, presence: { unless: :importing? }
    validates :status, presence: { unless: :importing? }
39
    validate :valid_commit_sha, unless: :importing?
40

41
    after_create :keep_around_commits, unless: :importing?
Kamil Trzcinski committed
42

43 44 45 46 47 48 49 50 51 52
    enum source: {
      unknown: nil,
      push: 1,
      web: 2,
      trigger: 3,
      schedule: 4,
      api: 5,
      external: 6
    }

53
    state_machine :status, initial: :created do
54
      event :enqueue do
Kamil Trzcinski committed
55
        transition created: :pending
56
        transition [:success, :failed, :canceled, :skipped] => :running
57 58 59
      end

      event :run do
60
        transition any - [:running] => :running
61 62
      end

63
      event :skip do
64
        transition any - [:skipped] => :skipped
65 66 67
      end

      event :drop do
68
        transition any - [:failed] => :failed
69 70
      end

71
      event :succeed do
72
        transition any - [:success] => :success
73 74 75
      end

      event :cancel do
76
        transition any - [:canceled] => :canceled
77 78
      end

79
      event :block do
80
        transition any - [:manual] => :manual
81 82
      end

83 84 85 86
      # IMPORTANT
      # Do not add any operations to this state_machine
      # Create a separate worker for each new operation

87
      before_transition [:created, :pending] => :running do |pipeline|
88
        pipeline.started_at = Time.now
89 90
      end

91
      before_transition any => [:success, :failed, :canceled] do |pipeline|
92
        pipeline.finished_at = Time.now
93 94 95
        pipeline.update_duration
      end

96 97 98 99
      before_transition any => [:manual] do |pipeline|
        pipeline.update_duration
      end

Lin Jen-Shin committed
100
      before_transition canceled: any - [:canceled] do |pipeline|
Lin Jen-Shin committed
101 102 103
        pipeline.auto_canceled_by = nil
      end

104
      after_transition [:created, :pending] => :running do |pipeline|
105
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
106 107 108
      end

      after_transition any => [:success] do |pipeline|
109
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
110 111
      end

112
      after_transition [:created, :pending, :running] => :success do |pipeline|
113
        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
114
      end
115 116

      after_transition do |pipeline, transition|
117 118 119
        next if transition.loopback?

        pipeline.run_after_commit do
120
          PipelineHooksWorker.perform_async(pipeline.id)
121
          ExpirePipelineCacheWorker.perform_async(pipeline.id)
122
        end
123
      end
124

125
      after_transition any => [:success, :failed] do |pipeline|
126
        pipeline.run_after_commit do
127
          PipelineNotificationWorker.perform_async(pipeline.id)
128
        end
129
      end
130 131
    end

132
    # ref can't be HEAD or SHA, can only be branch/tag name
133
    scope :latest, ->(ref = nil) do
134 135 136
      max_id = unscope(:select)
        .select("max(#{quoted_table_name}.id)")
        .group(:ref, :sha)
137

138 139 140 141 142
      if ref
        where(ref: ref, id: max_id.where(ref: ref))
      else
        where(id: max_id)
      end
143
    end
144
    scope :internal, -> { where(source: internal_sources) }
145

146 147 148 149
    def self.latest_status(ref = nil)
      latest(ref).status
    end

150
    def self.latest_successful_for(ref)
151
      success.latest(ref).order(id: :desc).first
152 153
    end

154 155 156 157 158 159
    def self.latest_successful_for_refs(refs)
      success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash|
        hash[pipeline.ref] ||= pipeline
      end
    end

160 161 162 163
    def self.truncate_sha(sha)
      sha[0...8]
    end

164
    def self.total_duration
165
      where.not(duration: nil).sum(:duration)
166 167
    end

168 169 170 171
    def self.internal_sources
      sources.reject { |source| source == "external" }.values
    end

172 173
    def stages_count
      statuses.select(:stage).distinct.count
174 175
    end

176
    def stages_names
177 178
      statuses.order(:stage_idx).distinct
        .pluck(:stage, :stage_idx).map(&:first)
179 180
    end

181
    def legacy_stage(name)
182
      stage = Ci::LegacyStage.new(self, name: name)
183 184 185 186
      stage unless stage.statuses_count.zero?
    end

    def legacy_stages
187 188
      # TODO, this needs refactoring, see gitlab-ce#26481.

189 190
      stages_query = statuses
        .group('stage').select(:stage).order('max(stage_idx)')
191

192 193
      status_sql = statuses.latest.where('stage=sg.stage').status_sql

194
      warnings_sql = statuses.latest.select('COUNT(*)')
195
        .where('stage=sg.stage').failed_but_allowed.to_sql
196

197 198
      stages_with_statuses = CommitStatus.from(stages_query, :sg)
        .pluck('sg.stage', status_sql, "(#{warnings_sql})")
199 200

      stages_with_statuses.map do |stage|
201
        Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
202 203 204
      end
    end

205
    def valid_commit_sha
206
      if self.sha == Gitlab::Git::BLANK_SHA
207 208 209 210 211
        self.errors.add(:sha, " cant be 00000000 (branch removal)")
      end
    end

    def git_author_name
212
      commit.try(:author_name)
213 214 215
    end

    def git_author_email
216
      commit.try(:author_email)
217 218 219
    end

    def git_commit_message
220
      commit.try(:message)
221 222
    end

223 224 225 226
    def git_commit_title
      commit.try(:title)
    end

227
    def short_sha
228
      Ci::Pipeline.truncate_sha(sha)
229 230
    end

231
    def commit
232
      @commit ||= project.commit(sha)
233 234 235 236
    rescue
      nil
    end

237 238 239 240
    def branch?
      !tag?
    end

241
    def stuck?
242
      pending_builds.any?(&:stuck?)
243 244
    end

245
    def retryable?
246
      retryable_builds.any?
247 248
    end

249
    def cancelable?
250
      cancelable_statuses.any?
251 252
    end

Lin Jen-Shin committed
253 254 255 256
    def auto_canceled?
      canceled? && auto_canceled_by_id?
    end

257
    def cancel_running
258
      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
259 260 261
        cancelable.find_each do |job|
          yield(job) if block_given?
          job.cancel
262
        end
263
      end
264 265
    end

266 267 268 269 270
    def auto_cancel_running(pipeline)
      update(auto_canceled_by: pipeline)

      cancel_running do |job|
        job.auto_canceled_by = pipeline
271
      end
272 273
    end

274
    def retry_failed(current_user)
275 276
      Ci::RetryPipelineService.new(project, current_user)
        .execute(self)
277 278
    end

279
    def mark_as_processable_after_stage(stage_idx)
280
      builds.skipped.after_stage(stage_idx).find_each(&:process)
281 282
    end

Kamil Trzcinski committed
283 284 285 286 287 288 289
    def latest?
      return false unless ref
      commit = project.commit(ref)
      return false unless commit
      commit.sha == sha
    end

290 291
    def retried
      @retried ||= (statuses.order(id: :desc) - statuses.latest)
292 293 294
    end

    def coverage
295
      coverage_array = statuses.latest.map(&:coverage).compact
296 297
      if coverage_array.size >= 1
        '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
298 299 300
      end
    end

301
    def stage_seeds
302 303
      return [] unless config_processor

304
      @stage_seeds ||= config_processor.stage_seeds(self)
305 306
    end

307 308
    def has_stage_seeds?
      stage_seeds.any?
309 310
    end

Connor Shea committed
311
    def has_warnings?
312
      builds.latest.failed_but_allowed.any?
313 314
    end

315
    def config_processor
316
      return unless ci_yaml_file
317 318 319
      return @config_processor if defined?(@config_processor)

      @config_processor ||= begin
320
        Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.full_path)
321
      rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
322
        self.yaml_errors = e.message
323 324
        nil
      rescue
325
        self.yaml_errors = 'Undefined error'
326 327
        nil
      end
328 329
    end

330
    def ci_yaml_file_path
331
      if project.ci_config_path.blank?
332 333
        '.gitlab-ci.yml'
      else
334
        project.ci_config_path
335 336 337
      end
    end

338
    def ci_yaml_file
339 340
      return @ci_yaml_file if defined?(@ci_yaml_file)

341
      @ci_yaml_file = begin
342
        project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
Lin Jen-Shin committed
343
      rescue Rugged::ReferenceError, GRPC::NotFound, GRPC::Internal
344
        self.yaml_errors =
345
          "Failed to load CI/CD config file at #{ci_yaml_file_path}"
346
        nil
347
      end
348 349
    end

350 351 352 353
    def has_yaml_errors?
      yaml_errors.present?
    end

Kamil Trzcinski committed
354 355 356 357
    def environments
      builds.where.not(environment: nil).success.pluck(:environment).uniq
    end

358 359 360 361 362 363 364 365 366 367 368 369 370
    # Manually set the notes for a Ci::Pipeline
    # There is no ActiveRecord relation between Ci::Pipeline and notes
    # as they are related to a commit sha. This method helps importing
    # them using the +Gitlab::ImportExport::RelationFactory+ class.
    def notes=(notes)
      notes.each do |note|
        note[:id] = nil
        note[:commit_id] = sha
        note[:noteable_id] = self['id']
        note.save!
      end
    end

371 372 373 374
    def notes
      Note.for_commit_id(sha)
    end

375 376 377
    def process!
      Ci::ProcessPipelineService.new(project, user).execute(self)
    end
378

379
    def update_status
380
      Gitlab::OptimisticLocking.retry_lock(self) do
Kamil Trzcinski committed
381
        case latest_builds_status
382 383 384 385 386 387
        when 'pending' then enqueue
        when 'running' then run
        when 'success' then succeed
        when 'failed' then drop
        when 'canceled' then cancel
        when 'skipped' then skip
388
        when 'manual' then block
389
        end
390
      end
391 392
    end

393 394
    def predefined_variables
      [
395
        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
396 397
        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true },
        { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
398 399 400
      ]
    end

401 402 403 404 405 406 407
    def queued_duration
      return unless started_at

      seconds = (started_at - created_at).to_i
      seconds unless seconds.zero?
    end

408
    def update_duration
409 410
      return unless started_at

411
      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
412 413 414
    end

    def execute_hooks
415 416 417
      data = pipeline_data
      project.execute_hooks(data, :pipeline_hooks)
      project.execute_services(data, :pipeline_hooks)
418 419
    end

420 421
    # All the merge requests for which the current pipeline runs/ran against
    def all_merge_requests
422
      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
423 424
    end

425
    def detailed_status(current_user)
426 427 428
      Gitlab::Ci::Status::Pipeline::Factory
        .new(self, current_user)
        .fabricate!
429 430
    end

431 432
    private

433
    def pipeline_data
434
      Gitlab::DataBuilder::Pipeline.build(self)
Kamil Trzcinski committed
435
    end
436

437
    def latest_builds_status
438 439 440
      return 'failed' unless yaml_errors.blank?

      statuses.latest.status || 'skipped'
Kamil Trzcinski committed
441
    end
442 443

    def keep_around_commits
444
      return unless project
445

446 447 448
      project.repository.keep_around(self.sha)
      project.repository.keep_around(self.before_sha)
    end
449 450
  end
end