BigW Consortium Gitlab

Skip to content

G
gitlab-ce
Commit 9d6fe7bf by Shinya Maeda
Options

Refactoring ci_job_trace to ci_build_trace

parent 4eb67ccf
Showing with 130 additions and 131 deletions
app/models/ci/build.rb
......@@ -19,14 +19,13 @@ module Ci
has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_one :metadata, class_name: 'Ci::BuildMetadata'
delegate :timeout, to: :metadata, prefix: true, allow_nil: true
delegate :gitlab_deploy_token, to: :project
......
app/models/ci/job_trace_chunk.rb → app/models/ci/build_trace_chunk.rb
module Ci
class JobTraceChunk < ActiveRecord::Base
class BuildTraceChunk < ActiveRecord::Base
extend Gitlab::Ci::Model
belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
after_destroy :redis_delete_data, if: :redis?
......@@ -118,11 +118,11 @@ module Ci
end
def redis_data_key
"gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:data"
"gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:data"
end
def redis_lock_key
"gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:lock"
"gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:lock"
end
def in_lock
......
app/workers/build_trace_swap_chunk_worker.rb
......@@ -4,9 +4,9 @@ class BuildTraceSwapChunkWorker
queue_namespace :pipeline_processing
def perform(job_trace_chunk_id)
Ci::JobTraceChunk.find_by(id: job_trace_chunk_id).try do |job_trace_chunk|
job_trace_chunk.use_database!
def perform(build_trace_chunk_id)
Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk|
build_trace_chunk.use_database!
end
end
end
db/migrate/20180326202229_create_ci_job_trace_chunks.rb → db/migrate/20180326202229_create_ci_build_trace_chunks.rb
class CreateCiJobTraceChunks < ActiveRecord::Migration
class CreateCiBuildTraceChunks < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :ci_job_trace_chunks, id: :bigserial do |t|
t.integer :job_id, null: false
create_table :ci_build_trace_chunks, id: :bigserial do |t|
t.integer :build_id, null: false
t.integer :chunk_index, null: false
t.integer :data_store, null: false
t.binary :raw_data
t.foreign_key :ci_builds, column: :job_id, on_delete: :cascade
t.index [:job_id, :chunk_index], unique: true
t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade
t.index [:build_id, :chunk_index], unique: true
end
end
end
db/migrate/20180406204716_add_limits_ci_job_trace_chunks_raw_data_for_mysql.rb → db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql')
require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
class AddLimitsCiJobTraceChunksRawDataForMysql < ActiveRecord::Migration
class AddLimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
LimitsCiJobTraceChunksRawDataForMysql.new.up
LimitsCiBuildTraceChunksRawDataForMysql.new.up
end
end
db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql.rb → db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb
class LimitsCiJobTraceChunksRawDataForMysql < ActiveRecord::Migration
class LimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
def up
return unless Gitlab::Database.mysql?
# Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB)
# Because 'raw_data' is always capped by Ci::JobTraceChunk::CHUNK_SIZE, which is 128KB
change_column :ci_job_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT
# Because 'raw_data' is always capped by Ci::BuildTraceChunk::CHUNK_SIZE, which is 128KB
change_column :ci_build_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT
end
end
db/schema.rb
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20180425131009) do
ActiveRecord::Schema.define(version: 20180425205249) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -246,6 +246,15 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree
create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t|
t.integer "build_id", null: false
t.integer "chunk_index", null: false
t.integer "data_store", null: false
t.binary "raw_data"
end
add_index "ci_build_trace_chunks", ["build_id", "chunk_index"], name: "index_ci_build_trace_chunks_on_build_id_and_chunk_index", unique: true, using: :btree
create_table "ci_build_trace_section_names", force: :cascade do |t|
t.integer "project_id", null: false
t.string "name", null: false
......@@ -371,15 +380,6 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
create_table "ci_job_trace_chunks", id: :bigserial, force: :cascade do |t|
t.integer "job_id", null: false
t.integer "chunk_index", null: false
t.integer "data_store", null: false
t.binary "raw_data"
end
add_index "ci_job_trace_chunks", ["job_id", "chunk_index"], name: "index_ci_job_trace_chunks_on_job_id_and_chunk_index", unique: true, using: :btree
create_table "ci_pipeline_schedule_variables", force: :cascade do |t|
t.string "key", null: false
t.text "value"
......@@ -2075,6 +2075,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade
add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade
add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade
add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade
add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade
......@@ -2087,7 +2088,6 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade
add_foreign_key "ci_job_artifacts", "ci_builds", column: "job_id", on_delete: :cascade
add_foreign_key "ci_job_artifacts", "projects", on_delete: :cascade
add_foreign_key "ci_job_trace_chunks", "ci_builds", column: "job_id", on_delete: :cascade
add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade
add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify
......
lib/gitlab/ci/trace.rb
......@@ -54,14 +54,14 @@ module Gitlab
end
def exist?
trace_artifact&.exists? || job.chunks.any? || current_path.present? || old_trace.present?
trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present?
end
def read
stream = Gitlab::Ci::Trace::Stream.new do
if trace_artifact
trace_artifact.open
elsif job.chunks.any?
elsif job.trace_chunks.any?
Gitlab::Ci::Trace::ChunkedIO.new(job)
elsif current_path
File.open(current_path, "rb")
......@@ -100,7 +100,7 @@ module Gitlab
FileUtils.rm(trace_path, force: true)
end
job.chunks.destroy_all
job.trace_chunks.destroy_all
job.erase_old_trace!
end
......@@ -108,7 +108,7 @@ module Gitlab
raise ArchiveError, 'Already archived' if trace_artifact
raise ArchiveError, 'Job is not finished yet' unless job.complete?
if job.chunks.any?
if job.trace_chunks.any?
Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
archive_stream!(stream)
stream.destroy!
......@@ -130,7 +130,7 @@ module Gitlab
def archive_stream!(stream)
clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
create_job_trace!(job, clone_path)
create_build_trace!(job, clone_path)
end
end
......@@ -146,7 +146,7 @@ module Gitlab
end
end
def create_job_trace!(job, path)
def create_build_trace!(job, path)
File.open(path) do |stream|
job.create_job_artifacts_trace!(
project: job.project,
......
lib/gitlab/ci/trace/chunked_io.rb
......@@ -5,18 +5,18 @@ module Gitlab
module Ci
class Trace
class ChunkedIO
CHUNK_SIZE = ::Ci::JobTraceChunk::CHUNK_SIZE
CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE
FailedToGetChunkError = Class.new(StandardError)
attr_reader :job
attr_reader :build
attr_reader :tell, :size
attr_reader :chunk, :chunk_range
alias_method :pos, :tell
def initialize(job, &block)
@job = job
def initialize(build, &block)
@build = build
@chunks_cache = []
@tell = 0
@size = calculate_size
......@@ -140,7 +140,7 @@ module Gitlab
@size = offset
# remove all next chunks
job_chunks.where('chunk_index > ?', chunk_index).destroy_all
trace_chunks.where('chunk_index > ?', chunk_index).destroy_all
# truncate current chunk
current_chunk.truncate(chunk_offset) if chunk_offset != 0
......@@ -157,7 +157,7 @@ module Gitlab
end
def destroy!
job_chunks.destroy_all
trace_chunks.destroy_all
@tell = @size = 0
ensure
invalidate_chunk_cache
......@@ -206,23 +206,23 @@ module Gitlab
end
def current_chunk
@chunks_cache[chunk_index] ||= job_chunks.find_by(chunk_index: chunk_index)
@chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index)
end
def build_chunk
@chunks_cache[chunk_index] = ::Ci::JobTraceChunk.new(job: job, chunk_index: chunk_index)
@chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index)
end
def ensure_chunk
current_chunk || build_chunk
end
def job_chunks
::Ci::JobTraceChunk.where(job: job)
def trace_chunks
::Ci::BuildTraceChunk.where(build: build)
end
def calculate_size
job_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
end
end
end
......
lib/tasks/migrate/add_limits_mysql.rake
require Rails.root.join('db/migrate/limits_to_mysql')
require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql')
require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
desc "GitLab | Add limits to strings in mysql database"
task add_limits_mysql: :environment do
......@@ -9,5 +9,5 @@ task add_limits_mysql: :environment do
LimitsToMysql.new.up
MarkdownCacheLimitsToMysql.new.up
MergeRequestDiffFileLimitsToMysql.new.up
LimitsCiJobTraceChunksRawDataForMysql.new.up
LimitsCiBuildTraceChunksRawDataForMysql.new.up
end
spec/factories/ci/job_trace_chunks.rb → spec/factories/ci/build_trace_chunks.rb
FactoryBot.define do
factory :ci_job_trace_chunk, class: Ci::JobTraceChunk do
factory :ci_build_trace_chunk, class: Ci::BuildTraceChunk do
job factory: :ci_build
chunk_index 0
data_store :redis
......
spec/lib/gitlab/ci/trace/chunked_io_spec.rb
......@@ -3,8 +3,8 @@ require 'spec_helper'
describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
include ChunkedIOHelpers
set(:job) { create(:ci_build, :running) }
let(:chunked_io) { described_class.new(job) }
set(:build) { create(:ci_build, :running) }
let(:chunked_io) { described_class.new(build) }
before do
stub_feature_flags(ci_enable_live_trace: true)
......@@ -13,7 +13,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context "#initialize" do
context 'when a chunk exists' do
before do
job.trace.set('ABC')
build.trace.set('ABC')
end
it { expect(chunked_io.size).to eq(3) }
......@@ -22,7 +22,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when two chunks exist' do
before do
stub_buffer_size(4)
job.trace.set('ABCDEF')
build.trace.set('ABCDEF')
end
it { expect(chunked_io.size).to eq(6) }
......@@ -37,7 +37,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
subject { chunked_io.seek(pos, where) }
before do
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
context 'when moves pos to end of the file' do
......@@ -68,7 +68,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
subject { chunked_io.eof? }
before do
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
context 'when current pos is at end of the file' do
......@@ -94,7 +94,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'yields lines' do
......@@ -106,7 +106,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'calls get_chunk only once' do
......@@ -127,7 +127,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it { is_expected.to eq(sample_trace_raw) }
......@@ -136,7 +136,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it { is_expected.to eq(sample_trace_raw) }
......@@ -149,7 +149,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'reads a trace' do
......@@ -160,7 +160,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'reads a trace' do
......@@ -175,7 +175,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'reads a trace' do
......@@ -186,7 +186,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'reads a trace' do
......@@ -201,7 +201,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'reads a trace' do
......@@ -212,7 +212,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'reads a trace' do
......@@ -238,7 +238,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it_behaves_like 'all line matching'
......@@ -247,7 +247,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it_behaves_like 'all line matching'
......@@ -256,7 +256,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when pos is at middle of the file' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
chunked_io.seek(chunked_io.size / 2)
string_io.seek(string_io.size / 2)
......@@ -316,7 +316,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(exist_data)
build.trace.set(exist_data)
end
it_behaves_like 'appends a trace'
......@@ -325,7 +325,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(exist_data)
build.trace.set(exist_data)
end
it_behaves_like 'appends a trace'
......@@ -349,7 +349,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is smaller than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize / 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it_behaves_like 'truncates a trace'
......@@ -358,7 +358,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
context 'when buffer size is larger than file size' do
before do
stub_buffer_size(sample_trace_raw.bytesize * 2)
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it_behaves_like 'truncates a trace'
......@@ -370,14 +370,14 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
subject { chunked_io.destroy! }
before do
job.trace.set(sample_trace_raw)
build.trace.set(sample_trace_raw)
end
it 'deletes' do
expect { subject }.to change { chunked_io.size }
.from(sample_trace_raw.bytesize).to(0)
expect(Ci::JobTraceChunk.where(job: job).count).to eq(0)
expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0)
end
end
end
spec/lib/gitlab/ci/trace/stream_spec.rb
require 'spec_helper'
describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
set(:job) { create(:ci_build, :running) }
set(:build) { create(:ci_build, :running) }
before do
stub_feature_flags(ci_enable_live_trace: true)
......@@ -83,7 +83,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write((1..8).to_a.join("\n"))
chunked_io.seek(0, IO::SEEK_SET)
end
......@@ -137,7 +137,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write('12345678')
chunked_io.seek(0, IO::SEEK_SET)
end
......@@ -175,7 +175,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write('12345678')
chunked_io.seek(0, IO::SEEK_SET)
end
......@@ -234,7 +234,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write(File.binread(path))
chunked_io.seek(0, IO::SEEK_SET)
end
......@@ -283,7 +283,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write("1234")
chunked_io.seek(0, IO::SEEK_SET)
end
......@@ -318,7 +318,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write("12\n34\n56")
chunked_io.seek(0, IO::SEEK_SET)
end
......@@ -473,7 +473,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
context 'when stream is ChunkedIO' do
let(:stream) do
described_class.new do
Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
chunked_io.write(data)
chunked_io.seek(0, IO::SEEK_SET)
end
......
spec/lib/gitlab/ci/trace_spec.rb
......@@ -436,7 +436,7 @@ describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
expect(Ci::JobTraceChunk.where(job: build)).not_to be_exist
expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
end
it "returns live trace data" do
......@@ -512,7 +512,7 @@ describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do
expect(build.trace.exist?).to be_truthy
expect(build.job_artifacts_trace.file.exists?).to be_truthy
expect(build.job_artifacts_trace.file.filename).to eq('job.log')
expect(Ci::JobTraceChunk.where(job: build)).not_to be_exist
expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
expect(src_checksum)
.to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
......
spec/models/ci/job_trace_chunk_spec.rb → spec/models/ci/build_trace_chunk_spec.rb
require 'spec_helper'
describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
set(:job) { create(:ci_build, :running) }
describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do
set(:build) { create(:ci_build, :running) }
let(:chunk_index) { 0 }
let(:data_store) { :redis }
let(:raw_data) { nil }
let(:job_trace_chunk) do
described_class.new(job: job, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data)
let(:build_trace_chunk) do
described_class.new(build: build, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data)
end
describe 'CHUNK_SIZE' do
......@@ -17,13 +17,13 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
describe '#data' do
subject { job_trace_chunk.data }
subject { build_trace_chunk.data }
context 'when data_store is redis' do
let(:data_store) { :redis }
before do
job_trace_chunk.send(:redis_set_data, 'Sample data in redis')
build_trace_chunk.send(:redis_set_data, 'Sample data in redis')
end
it { is_expected.to eq('Sample data in redis') }
......@@ -38,7 +38,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
context 'when data_store is others' do
before do
job_trace_chunk.send(:write_attribute, :data_store, -1)
build_trace_chunk.send(:write_attribute, :data_store, -1)
end
it { expect { subject }.to raise_error('Unsupported data store') }
......@@ -46,7 +46,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
describe '#set_data' do
subject { job_trace_chunk.set_data(value) }
subject { build_trace_chunk.set_data(value) }
let(:value) { 'Sample data' }
......@@ -60,11 +60,11 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
let(:data_store) { :redis }
it do
expect(job_trace_chunk.send(:redis_data)).to be_nil
expect(build_trace_chunk.send(:redis_data)).to be_nil
subject
expect(job_trace_chunk.send(:redis_data)).to eq(value)
expect(build_trace_chunk.send(:redis_data)).to eq(value)
end
context 'when fullfilled chunk size' do
......@@ -82,26 +82,26 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
let(:data_store) { :db }
it 'sets data' do
expect(job_trace_chunk.raw_data).to be_nil
expect(build_trace_chunk.raw_data).to be_nil
subject
expect(job_trace_chunk.raw_data).to eq(value)
expect(job_trace_chunk.persisted?).to be_truthy
expect(build_trace_chunk.raw_data).to eq(value)
expect(build_trace_chunk.persisted?).to be_truthy
end
context 'when raw_data is not changed' do
it 'does not execute UPDATE' do
expect(job_trace_chunk.raw_data).to be_nil
job_trace_chunk.save!
expect(build_trace_chunk.raw_data).to be_nil
build_trace_chunk.save!
# First set
expect(ActiveRecord::QueryRecorder.new { subject }.count).to be > 0
expect(job_trace_chunk.raw_data).to eq(value)
expect(job_trace_chunk.persisted?).to be_truthy
expect(build_trace_chunk.raw_data).to eq(value)
expect(build_trace_chunk.persisted?).to be_truthy
# Second set
job_trace_chunk.reload
build_trace_chunk.reload
expect(ActiveRecord::QueryRecorder.new { subject }.count).to be(0)
end
end
......@@ -117,7 +117,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
context 'when data_store is others' do
before do
job_trace_chunk.send(:write_attribute, :data_store, -1)
build_trace_chunk.send(:write_attribute, :data_store, -1)
end
it { expect { subject }.to raise_error('Unsupported data store') }
......@@ -125,7 +125,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
describe '#truncate' do
subject { job_trace_chunk.truncate(offset) }
subject { build_trace_chunk.truncate(offset) }
shared_examples_for 'truncates' do
context 'when offset is negative' do
......@@ -146,7 +146,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
it 'truncates' do
subject
expect(job_trace_chunk.data).to eq(data.byteslice(0, offset))
expect(build_trace_chunk.data).to eq(data.byteslice(0, offset))
end
end
end
......@@ -156,7 +156,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
let(:data) { 'Sample data in redis' }
before do
job_trace_chunk.send(:redis_set_data, data)
build_trace_chunk.send(:redis_set_data, data)
end
it_behaves_like 'truncates'
......@@ -172,7 +172,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
describe '#append' do
subject { job_trace_chunk.append(new_data, offset) }
subject { build_trace_chunk.append(new_data, offset) }
let(:new_data) { 'Sample new data' }
let(:offset) { 0 }
......@@ -203,7 +203,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
it 'appends' do
subject
expect(job_trace_chunk.data).to eq(total_data)
expect(build_trace_chunk.data).to eq(total_data)
end
end
......@@ -213,7 +213,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
it 'appends' do
subject
expect(job_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data)
expect(build_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data)
end
end
end
......@@ -223,7 +223,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
let(:data) { 'Sample data in redis' }
before do
job_trace_chunk.send(:redis_set_data, data)
build_trace_chunk.send(:redis_set_data, data)
end
it_behaves_like 'appends'
......@@ -239,7 +239,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
describe '#size' do
subject { job_trace_chunk.size }
subject { build_trace_chunk.size }
context 'when data_store is redis' do
let(:data_store) { :redis }
......@@ -248,7 +248,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
let(:data) { 'Sample data in redis' }
before do
job_trace_chunk.send(:redis_set_data, data)
build_trace_chunk.send(:redis_set_data, data)
end
it { is_expected.to eq(data.bytesize) }
......@@ -276,7 +276,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
describe '#use_database!' do
subject { job_trace_chunk.use_database! }
subject { build_trace_chunk.use_database! }
context 'when data_store is redis' do
let(:data_store) { :redis }
......@@ -285,19 +285,19 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
let(:data) { 'Sample data in redis' }
before do
job_trace_chunk.send(:redis_set_data, data)
build_trace_chunk.send(:redis_set_data, data)
end
it 'stashes the data' do
expect(job_trace_chunk.data_store).to eq('redis')
expect(job_trace_chunk.send(:redis_data)).to eq(data)
expect(job_trace_chunk.raw_data).to be_nil
expect(build_trace_chunk.data_store).to eq('redis')
expect(build_trace_chunk.send(:redis_data)).to eq(data)
expect(build_trace_chunk.raw_data).to be_nil
subject
expect(job_trace_chunk.data_store).to eq('db')
expect(job_trace_chunk.send(:redis_data)).to be_nil
expect(job_trace_chunk.raw_data).to eq(data)
expect(build_trace_chunk.data_store).to eq('db')
expect(build_trace_chunk.send(:redis_data)).to be_nil
expect(build_trace_chunk.raw_data).to eq(data)
end
end
......@@ -320,11 +320,11 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
describe 'ExclusiveLock' do
before do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
stub_const('Ci::JobTraceChunk::LOCK_RETRY', 1)
stub_const('Ci::BuildTraceChunk::LOCK_RETRY', 1)
end
it 'raise an error' do
expect { job_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock')
expect { build_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock')
end
end
......@@ -338,7 +338,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
end
shared_examples_for 'deletes all job_trace_chunk and data in redis' do
shared_examples_for 'deletes all build_trace_chunk and data in redis' do
it do
project.builds.each do |build|
Gitlab::Redis::SharedState.with do |redis|
......@@ -364,20 +364,20 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
end
end
context 'when job_trace_chunk is destroyed' do
context 'when build_trace_chunk is destroyed' do
let(:subject) do
project.builds.each { |build| build.chunks.destroy_all }
end
it_behaves_like 'deletes all job_trace_chunk and data in redis'
it_behaves_like 'deletes all build_trace_chunk and data in redis'
end
context 'when job is destroyed' do
context 'when build is destroyed' do
let(:subject) do
project.builds.destroy_all
end
it_behaves_like 'deletes all job_trace_chunk and data in redis'
it_behaves_like 'deletes all build_trace_chunk and data in redis'
end
context 'when project is destroyed' do
......@@ -385,7 +385,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
project.destroy!
end
it_behaves_like 'deletes all job_trace_chunk and data in redis'
it_behaves_like 'deletes all build_trace_chunk and data in redis'
end
end
end
spec/support/chunked_io/chunked_io_helpers.rb
......@@ -5,7 +5,7 @@ module ChunkedIOHelpers
end
def stub_buffer_size(size)
stub_const('Ci::JobTraceChunk::CHUNK_SIZE', size)
stub_const('Ci::BuildTraceChunk::CHUNK_SIZE', size)
stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size)
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment