BigW Consortium Gitlab

Skip to content

G
gitlab-ce
Commit 828d81ee by Kamil Trzciński Committed by Rémy Coutable
Options

Optimise trace handling code to use streaming instead of full read

parent 514dc1a0
Showing with 921 additions and 590 deletions
app/assets/javascripts/build.js
......@@ -84,10 +84,10 @@ window.Build = (function() {
var removeRefreshStatuses = ['success', 'failed', 'canceled', 'skipped'];
return $.ajax({
url: this.buildUrl,
url: this.pageUrl + "/trace.json",
dataType: 'json',
success: function(buildData) {
$('.js-build-output').html(buildData.trace_html);
$('.js-build-output').html(buildData.html);
gl.utils.setCiStatusFavicon(`${this.pageUrl}/status.json`);
if (window.location.hash === DOWN_BUILD_TRACE) {
$("html,body").scrollTop(this.$buildTrace.height());
......
app/controllers/projects/builds_controller.rb
......@@ -31,25 +31,25 @@ class Projects::BuildsController < Projects::ApplicationController
@builds = @project.pipelines.find_by_sha(@build.sha).builds.order('id DESC')
@builds = @builds.where("id not in (?)", @build.id)
@pipeline = @build.pipeline
respond_to do |format|
format.html
format.json do
render json: {
id: @build.id,
status: @build.status,
trace_html: @build.trace_html
}
end
end
end
def trace
respond_to do |format|
format.json do
state = params[:state].presence
render json: @build.trace_with_state(state: state).
merge!(id: @build.id, status: @build.status)
build.trace.read do |stream|
respond_to do |format|
format.json do
result = {
id: @build.id, status: @build.status, complete: @build.complete?
}
if stream.valid?
stream.limit
state = params[:state].presence
trace = stream.html_with_state(state)
result.merge!(trace.to_h)
end
render json: result
end
end
end
end
......@@ -86,10 +86,12 @@ class Projects::BuildsController < Projects::ApplicationController
end
def raw
if @build.has_trace_file?
send_file @build.trace_file_path, type: 'text/plain; charset=utf-8', disposition: 'inline'
else
render_404
build.trace.read do |stream|
if stream.file?
send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
else
render_404
end
end
end
......
app/models/ci/build.rb
......@@ -171,19 +171,6 @@ module Ci
latest_builds.where('stage_idx < ?', stage_idx)
end
def trace_html(**args)
trace_with_state(**args)[:html] || ''
end
def trace_with_state(state: nil, last_lines: nil)
trace_ansi = trace(last_lines: last_lines)
if trace_ansi.present?
Ci::Ansi2html.convert(trace_ansi, state)
else
{}
end
end
def timeout
project.build_timeout
end
......@@ -244,136 +231,35 @@ module Ci
end
def update_coverage
coverage = extract_coverage(trace, coverage_regex)
coverage = trace.extract_coverage(coverage_regex)
update_attributes(coverage: coverage) if coverage.present?
end
def extract_coverage(text, regex)
return unless regex
matches = text.scan(Regexp.new(regex)).last
matches = matches.last if matches.is_a?(Array)
coverage = matches.gsub(/\d+(\.\d+)?/).first
if coverage.present?
coverage.to_f
end
rescue
# if bad regex or something goes wrong we dont want to interrupt transition
# so we just silentrly ignore error for now
end
def has_trace_file?
File.exist?(path_to_trace) || has_old_trace_file?
def trace
Gitlab::Ci::Trace.new(self)
end
def has_trace?
raw_trace.present?
trace.exist?
end
def raw_trace(last_lines: nil)
if File.exist?(trace_file_path)
Gitlab::Ci::TraceReader.new(trace_file_path).
read(last_lines: last_lines)
else
# backward compatibility
read_attribute :trace
end
def trace=(data)
raise NotImplementedError
end
##
# Deprecated
#
# This is a hotfix for CI build data integrity, see #4246
def has_old_trace_file?
project.ci_id && File.exist?(old_path_to_trace)
end
def trace(last_lines: nil)
hide_secrets(raw_trace(last_lines: last_lines))
def old_trace
read_attribute(:trace)
end
def trace_length
if raw_trace
raw_trace.bytesize
else
0
end
end
def trace=(trace)
recreate_trace_dir
trace = hide_secrets(trace)
File.write(path_to_trace, trace)
end
def recreate_trace_dir
unless Dir.exist?(dir_to_trace)
FileUtils.mkdir_p(dir_to_trace)
end
end
private :recreate_trace_dir
def append_trace(trace_part, offset)
recreate_trace_dir
touch if needs_touch?
trace_part = hide_secrets(trace_part)
File.truncate(path_to_trace, offset) if File.exist?(path_to_trace)
File.open(path_to_trace, 'ab') do |f|
f.write(trace_part)
end
def erase_old_trace!
write_attribute(:trace, nil)
save
end
def needs_touch?
Time.now - updated_at > 15.minutes.to_i
end
def trace_file_path
if has_old_trace_file?
old_path_to_trace
else
path_to_trace
end
end
def dir_to_trace
File.join(
Settings.gitlab_ci.builds_path,
created_at.utc.strftime("%Y_%m"),
project.id.to_s
)
end
def path_to_trace
"#{dir_to_trace}/#{id}.log"
end
##
# Deprecated
#
# This is a hotfix for CI build data integrity, see #4246
# Should be removed in 8.4, after CI files migration has been done.
#
def old_dir_to_trace
File.join(
Settings.gitlab_ci.builds_path,
created_at.utc.strftime("%Y_%m"),
project.ci_id.to_s
)
end
##
# Deprecated
#
# This is a hotfix for CI build data integrity, see #4246
# Should be removed in 8.4, after CI files migration has been done.
#
def old_path_to_trace
"#{old_dir_to_trace}/#{id}.log"
end
##
# Deprecated
#
......@@ -555,6 +441,15 @@ module Ci
options[:dependencies]&.empty?
end
def hide_secrets(trace)
return unless trace
trace = trace.dup
Ci::MaskSecret.mask!(trace, project.runners_token) if project
Ci::MaskSecret.mask!(trace, token)
trace
end
private
def update_artifacts_size
......@@ -566,7 +461,7 @@ module Ci
end
def erase_trace!
self.trace = nil
trace.erase!
end
def update_erased!(user = nil)
......@@ -628,15 +523,6 @@ module Ci
pipeline.config_processor.build_attributes(name)
end
def hide_secrets(trace)
return unless trace
trace = trace.dup
Ci::MaskSecret.mask!(trace, project.runners_token) if project
Ci::MaskSecret.mask!(trace, token)
trace
end
def update_project_statistics
return unless project
......
app/views/notify/pipeline_failed_email.html.haml
......@@ -137,6 +137,6 @@
- if build.has_trace?
%td{ colspan: "2", style: "font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;padding:0 0 15px;" }
%pre{ style: "font-family:Monaco,'Lucida Console','Courier New',Courier,monospace;background-color:#fafafa;border-radius:3px;overflow:hidden;white-space:pre-wrap;word-break:break-all;font-size:13px;line-height:1.4;padding:12px;color:#333333;margin:0;" }
= build.trace_html(last_lines: 10).html_safe
= build.trace.html(last_lines: 10).html_safe
- else
%td{ colspan: "2" }
app/views/notify/pipeline_failed_email.text.erb
......@@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
Stage: <%= build.stage %>
Name: <%= build.name %>
<% if build.has_trace? -%>
Trace: <%= build.trace_with_state(last_lines: 10)[:text] %>
Trace: <%= build.trace.raw(last_lines: 10) %>
<% end -%>
<% end -%>
app/views/projects/builds/_sidebar.html.haml
......@@ -68,7 +68,7 @@
- elsif @build.runner
\##{@build.runner.id}
.btn-group.btn-group-justified{ role: :group }
- if @build.has_trace_file?
- if @build.has_trace?
= link_to 'Raw', raw_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default'
- if @build.active?
= link_to "Cancel", cancel_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post
......
db/fixtures/development/14_pipelines.rb
......@@ -130,7 +130,7 @@ class Gitlab::Seeder::Pipelines
def setup_build_log(build)
if %w(running success failed).include?(build.status)
build.trace = FFaker::Lorem.paragraphs(6).join("\n\n")
build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n"))
end
end
......
features/steps/project/builds/summary.rb
......@@ -22,9 +22,9 @@ class Spinach::Features::ProjectBuildsSummary < Spinach::FeatureSteps
end
step 'recent build has been erased' do
expect(@build).not_to have_trace
expect(@build.artifacts_file.exists?).to be_falsy
expect(@build.artifacts_metadata.exists?).to be_falsy
expect(@build.trace).to be_empty
end
step 'recent build summary does not have artifacts widget' do
......
features/steps/shared/builds.rb
......@@ -47,7 +47,7 @@ module SharedBuilds
end
step 'recent build has a build trace' do
@build.trace = 'job trace'
@build.trace.set('job trace')
end
step 'download of build artifacts archive starts' do
......
lib/api/jobs.rb
......@@ -118,7 +118,7 @@ module API
content_type 'text/plain'
env['api.format'] = :binary
trace = build.trace
trace = build.trace.raw
body trace
end
......
lib/api/runner.rb
......@@ -115,7 +115,7 @@ module API
put '/:id' do
job = authenticate_job!
job.update_attributes(trace: params[:trace]) if params[:trace]
job.trace.set(params[:trace]) if params[:trace]
Gitlab::Metrics.add_event(:update_build,
project: job.project.path_with_namespace)
......@@ -145,16 +145,14 @@ module API
content_range = request.headers['Content-Range']
content_range = content_range.split('-')
current_length = job.trace_length
unless current_length == content_range[0].to_i
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
stream_size = job.trace.append(request.body.read, content_range[0].to_i)
if stream_size < 0
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
end
job.append_trace(request.body.read, content_range[0].to_i)
status 202
header 'Job-Status', job.status
header 'Range', "0-#{job.trace_length}"
header 'Range', "0-#{stream_size}"
end
desc 'Authorize artifacts uploading for job' do
......
lib/api/v3/builds.rb
......@@ -120,7 +120,7 @@ module API
content_type 'text/plain'
env['api.format'] = :binary
trace = build.trace
trace = build.trace.raw
body trace
end
......
lib/ci/ansi2html.rb
......@@ -132,34 +132,54 @@ module Ci
STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
def convert(raw, new_state)
def convert(stream, new_state)
reset_state
restore_state(raw, new_state) if new_state.present?
start = @offset
ansi = raw[@offset..-1]
restore_state(new_state, stream) if new_state.present?
append = false
truncated = false
cur_offset = stream.tell
if cur_offset > @offset
@offset = cur_offset
truncated = true
else
stream.seek(@offset)
append = @offset > 0
end
start_offset = @offset
open_new_tag
s = StringScanner.new(ansi)
until s.eos?
if s.scan(/\e([@-_])(.*?)([@-~])/)
handle_sequence(s)
elsif s.scan(/\e(([@-_])(.*?)?)?$/)
break
elsif s.scan(/</)
@out << '&lt;'
elsif s.scan(/\r?\n/)
@out << '<br>'
else
@out << s.scan(/./m)
stream.each_line do |line|
s = StringScanner.new(line)
until s.eos?
if s.scan(/\e([@-_])(.*?)([@-~])/)
handle_sequence(s)
elsif s.scan(/\e(([@-_])(.*?)?)?$/)
break
elsif s.scan(/</)
@out << '&lt;'
elsif s.scan(/\r?\n/)
@out << '<br>'
else
@out << s.scan(/./m)
end
@offset += s.matched_size
end
@offset += s.matched_size
end
close_open_tags()
{ state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 }
OpenStruct.new(
html: @out,
state: state,
append: append,
truncated: truncated,
offset: start_offset,
size: stream.tell - start_offset,
total: stream.size
)
end
def handle_sequence(s)
......@@ -240,10 +260,10 @@ module Ci
Base64.urlsafe_encode64(state.to_json)
end
def restore_state(raw, new_state)
def restore_state(new_state, stream)
state = Base64.urlsafe_decode64(new_state)
state = JSON.parse(state, symbolize_names: true)
return if state[:offset].to_i > raw.length
return if state[:offset].to_i > stream.size
STATE_PARAMS.each do |param|
send("#{param}=".to_sym, state[param])
......
lib/ci/api/builds.rb
......@@ -61,7 +61,7 @@ module Ci
update_runner_info
build.update_attributes(trace: params[:trace]) if params[:trace]
build.trace.set(params[:trace]) if params[:trace]
Gitlab::Metrics.add_event(:update_build,
project: build.project.path_with_namespace)
......@@ -92,16 +92,14 @@ module Ci
content_range = request.headers['Content-Range']
content_range = content_range.split('-')
current_length = build.trace_length
unless current_length == content_range[0].to_i
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
stream_size = build.trace.append(request.body.read, content_range[0].to_i)
if stream_size < 0
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
end
build.append_trace(request.body.read, content_range[0].to_i)
status 202
header 'Build-Status', build.status
header 'Range', "0-#{build.trace_length}"
header 'Range', "0-#{stream_size}"
end
# Authorize artifacts uploading for build - Runners only
......
lib/gitlab/ci/trace.rb 0 → 100644
module Gitlab
module Ci
class Trace
attr_reader :job
delegate :old_trace, to: :job
def initialize(job)
@job = job
end
def html(last_lines: nil)
read do |stream|
stream.html(last_lines: last_lines)
end
end
def raw(last_lines: nil)
read do |stream|
stream.raw(last_lines: last_lines)
end
end
def extract_coverage(regex)
read do |stream|
stream.extract_coverage(regex)
end
end
def set(data)
write do |stream|
data = job.hide_secrets(data)
stream.set(data)
end
end
def append(data, offset)
write do |stream|
current_length = stream.size
return -current_length unless current_length == offset
data = job.hide_secrets(data)
stream.append(data, offset)
stream.size
end
end
def exist?
current_path.present? || old_trace.present?
end
def read
stream = Gitlab::Ci::Trace::Stream.new do
if current_path
File.open(current_path, "rb")
elsif old_trace
StringIO.new(old_trace)
end
end
yield stream
ensure
stream&.close
end
def write
stream = Gitlab::Ci::Trace::Stream.new do
File.open(ensure_path, "a+b")
end
yield(stream).tap do
job.touch if job.needs_touch?
end
ensure
stream&.close
end
def erase!
paths.each do |trace_path|
FileUtils.rm(trace_path, force: true)
end
job.erase_old_trace!
end
private
def ensure_path
return current_path if current_path
ensure_directory
default_path
end
def ensure_directory
unless Dir.exist?(default_directory)
FileUtils.mkdir_p(default_directory)
end
end
def current_path
@current_path ||= paths.find do |trace_path|
File.exist?(trace_path)
end
end
def paths
[
default_path,
deprecated_path
].compact
end
def default_directory
File.join(
Settings.gitlab_ci.builds_path,
job.created_at.utc.strftime("%Y_%m"),
job.project_id.to_s
)
end
def default_path
File.join(default_directory, "#{job.id}.log")
end
def deprecated_path
File.join(
Settings.gitlab_ci.builds_path,
job.created_at.utc.strftime("%Y_%m"),
job.project.ci_id.to_s,
"#{job.id}.log"
) if job.project&.ci_id
end
end
end
end
lib/gitlab/ci/trace/stream.rb 0 → 100644
module Gitlab
module Ci
class Trace
# This was inspired from: http://stackoverflow.com/a/10219411/1520132
class Stream
BUFFER_SIZE = 4096
LIMIT_SIZE = 50.kilobytes
attr_reader :stream
delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
delegate :valid?, to: :stream, as: :present?, allow_nil: true
def initialize
@stream = yield
end
def valid?
self.stream.present?
end
def file?
self.path.present?
end
def limit(last_bytes = LIMIT_SIZE)
stream_size = size
if stream_size < last_bytes
last_bytes = stream_size
end
stream.seek(-last_bytes, IO::SEEK_END)
end
def append(data, offset)
stream.truncate(offset)
stream.seek(0, IO::SEEK_END)
stream.write(data)
stream.flush()
end
def set(data)
truncate(0)
stream.write(data)
stream.flush()
end
def raw(last_lines: nil)
return unless valid?
if last_lines.to_i > 0
read_last_lines(last_lines)
else
stream.read
end
end
def html_with_state(state = nil)
::Ci::Ansi2html.convert(stream, state)
end
def html(last_lines: nil)
text = raw(last_lines: last_lines)
stream = StringIO.new(text)
::Ci::Ansi2html.convert(stream).html
end
def extract_coverage(regex)
return unless valid?
return unless regex
regex = Regexp.new(regex)
match = ""
stream.each_line do |line|
matches = line.scan(regex)
next unless matches.is_a?(Array)
match = matches.flatten.last
coverage = match.gsub(/\d+(\.\d+)?/).first
return coverage.to_f if coverage.present?
end
rescue
# if bad regex or something goes wrong we dont want to interrupt transition
# so we just silentrly ignore error for now
end
private
def read_last_lines(last_lines)
chunks = []
pos = lines = 0
max = stream.size
# We want an extra line to make sure fist line has full contents
while lines <= last_lines && pos < max
pos += BUFFER_SIZE
buf =
if pos <= max
stream.seek(-pos, IO::SEEK_END)
stream.read(BUFFER_SIZE)
else # Reached the head, read only left
stream.seek(0)
stream.read(BUFFER_SIZE - (pos - max))
end
lines += buf.count("\n")
chunks.unshift(buf)
end
chunks.join.lines.last(last_lines).join
.force_encoding(Encoding.default_external)
end
end
end
end
end
lib/gitlab/ci/trace_reader.rb deleted 100644 → 0
module Gitlab
module Ci
# This was inspired from: http://stackoverflow.com/a/10219411/1520132
class TraceReader
BUFFER_SIZE = 4096
attr_accessor :path, :buffer_size
def initialize(new_path, buffer_size: BUFFER_SIZE)
self.path = new_path
self.buffer_size = Integer(buffer_size)
end
def read(last_lines: nil)
if last_lines
read_last_lines(last_lines)
else
File.read(path)
end
end
def read_last_lines(max_lines)
File.open(path) do |file|
chunks = []
pos = lines = 0
max = file.size
# We want an extra line to make sure fist line has full contents
while lines <= max_lines && pos < max
pos += buffer_size
buf = if pos <= max
file.seek(-pos, IO::SEEK_END)
file.read(buffer_size)
else # Reached the head, read only left
file.seek(0)
file.read(buffer_size - (pos - max))
end
lines += buf.count("\n")
chunks.unshift(buf)
end
chunks.join.lines.last(max_lines).join
.force_encoding(Encoding.default_external)
end
end
end
end
end
spec/factories/ci/builds.rb
......@@ -111,7 +111,7 @@ FactoryGirl.define do
trait :trace do
after(:create) do |build, evaluator|
build.trace = 'BUILD TRACE'
build.trace.set('BUILD TRACE')
end
end
......
spec/features/projects/builds_spec.rb
......@@ -205,21 +205,13 @@ feature 'Builds', :feature do
it 'loads job trace' do
expect(page).to have_content 'BUILD TRACE'
build.append_trace(' and more trace', 11)
build.trace.write do |stream|
stream.append(' and more trace', 11)
end
expect(page).to have_content 'BUILD TRACE and more trace'
end
end
context 'when build does not have an initial trace' do
let(:build) { create(:ci_build, pipeline: pipeline) }
it 'loads new trace' do
build.append_trace('build trace', 0)
expect(page).to have_content 'build trace'
end
end
end
feature 'Variables' do
......@@ -390,7 +382,7 @@ feature 'Builds', :feature do
it 'sends the right headers' do
expect(page.status_code).to eq(200)
expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
expect(page.response_headers['X-Sendfile']).to eq(build.path_to_trace)
expect(page.response_headers['X-Sendfile']).to eq(build.trace.send(:current_path))
end
end
......@@ -409,43 +401,24 @@ feature 'Builds', :feature do
context 'storage form' do
let(:existing_file) { Tempfile.new('existing-trace-file').path }
let(:non_existing_file) do
file = Tempfile.new('non-existing-trace-file')
path = file.path
file.unlink
path
end
context 'when build has trace in file' do
before do
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
build.run!
visit namespace_project_build_path(project.namespace, project, build)
before do
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
allow_any_instance_of(Project).to receive(:ci_id).and_return(nil)
allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(existing_file)
allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file)
build.run!
page.within('.js-build-sidebar') { click_link 'Raw' }
end
allow_any_instance_of(Gitlab::Ci::Trace).to receive(:paths)
.and_return(paths)
it 'sends the right headers' do
expect(page.status_code).to eq(200)
expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
expect(page.response_headers['X-Sendfile']).to eq(existing_file)
end
visit namespace_project_build_path(project.namespace, project, build)
end
context 'when build has trace in old file' do
before do
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
build.run!
visit namespace_project_build_path(project.namespace, project, build)
allow_any_instance_of(Project).to receive(:ci_id).and_return(999)
allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file)
allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(existing_file)
context 'when build has trace in file' do
let(:paths) do
[existing_file]
end
before do
page.within('.js-build-sidebar') { click_link 'Raw' }
end
......@@ -457,20 +430,10 @@ feature 'Builds', :feature do
end
context 'when build has trace in DB' do
before do
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
build.run!
visit namespace_project_build_path(project.namespace, project, build)
allow_any_instance_of(Project).to receive(:ci_id).and_return(nil)
allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file)
allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file)
page.within('.js-build-sidebar') { click_link 'Raw' }
end
let(:paths) { [] }
it 'sends the right headers' do
expect(page.status_code).to eq(404)
expect(page.status_code).not_to have_link('Raw')
end
end
end
......
spec/javascripts/build_spec.js
......@@ -72,12 +72,14 @@ describe('Build', () => {
it('displays the initial build trace', () => {
expect($.ajax.calls.count()).toBe(1);
const [{ url, dataType, success, context }] = $.ajax.calls.argsFor(0);
expect(url).toBe(`${BUILD_URL}.json`);
expect(url).toBe(
`${BUILD_URL}/trace.json`,
);
expect(dataType).toBe('json');
expect(success).toEqual(jasmine.any(Function));
spyOn(gl.utils, 'setCiStatusFavicon').and.callFake(() => {});
success.call(context, { trace_html: '<span>Example</span>', status: 'running' });
success.call(context, { html: '<span>Example</span>', status: 'running' });
expect($('#build-trace .js-build-output').text()).toMatch(/Example/);
});
......
spec/lib/gitlab/ci/trace/stream_spec.rb 0 → 100644
require 'spec_helper'
describe Gitlab::Ci::Trace::Stream do
describe 'delegates' do
subject { described_class.new { nil } }
it { is_expected.to delegate_method(:close).to(:stream) }
it { is_expected.to delegate_method(:tell).to(:stream) }
it { is_expected.to delegate_method(:seek).to(:stream) }
it { is_expected.to delegate_method(:size).to(:stream) }
it { is_expected.to delegate_method(:path).to(:stream) }
it { is_expected.to delegate_method(:truncate).to(:stream) }
it { is_expected.to delegate_method(:valid?).to(:stream).as(:present?) }
it { is_expected.to delegate_method(:file?).to(:path).as(:present?) }
end
describe '#limit' do
let(:stream) do
described_class.new do
StringIO.new("12345678")
end
end
it 'if size is larger we start from beggining' do
stream.limit(10)
expect(stream.tell).to eq(0)
end
it 'if size is smaller we start from the end' do
stream.limit(2)
expect(stream.tell).to eq(6)
end
end
describe '#append' do
let(:stream) do
described_class.new do
StringIO.new("12345678")
end
end
it "truncates and append content" do
stream.append("89", 4)
stream.seek(0)
expect(stream.size).to eq(6)
expect(stream.raw).to eq("123489")
end
end
describe '#set' do
let(:stream) do
described_class.new do
StringIO.new("12345678")
end
end
before do
stream.set("8901")
end
it "overwrite content" do
stream.seek(0)
expect(stream.size).to eq(4)
expect(stream.raw).to eq("8901")
end
end
describe '#raw' do
let(:path) { __FILE__ }
let(:lines) { File.readlines(path) }
let(:stream) do
described_class.new do
File.open(path)
end
end
it 'returns all contents if last_lines is not specified' do
result = stream.raw
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
context 'limit max lines' do
before do
# specifying BUFFER_SIZE forces to seek backwards
allow(described_class).to receive(:BUFFER_SIZE)
.and_return(2)
end
it 'returns last few lines' do
result = stream.raw(last_lines: 2)
expect(result).to eq(lines.last(2).join)
expect(result.encoding).to eq(Encoding.default_external)
end
it 'returns everything if trying to get too many lines' do
result = stream.raw(last_lines: lines.size * 2)
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
end
end
describe '#html_with_state' do
let(:stream) do
described_class.new do
StringIO.new("1234")
end
end
it 'returns html content with state' do
result = stream.html_with_state
expect(result.html).to eq("1234")
end
context 'follow-up state' do
let!(:last_result) { stream.html_with_state }
before do
stream.append("5678", 4)
stream.seek(0)
end
it "returns appended trace" do
result = stream.html_with_state(last_result.state)
expect(result.append).to be_truthy
expect(result.html).to eq("5678")
end
end
end
describe '#html' do
let(:stream) do
described_class.new do
StringIO.new("12\n34\n56")
end
end
it "returns html" do
expect(stream.html).to eq("12<br>34<br>56")
end
it "returns html for last line only" do
expect(stream.html(last_lines: 1)).to eq("56")
end
end
describe '#extract_coverage' do
let(:stream) do
described_class.new do
StringIO.new(data)
end
end
subject { stream.extract_coverage(regex) }
context 'valid content & regex' do
let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' }
let(:regex) { '\(\d+.\d+\%\) covered' }
it { is_expected.to eq(98.29) }
end
context 'valid content & bad regex' do
let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
let(:regex) { 'very covered' }
it { is_expected.to be_nil }
end
context 'no coverage content & regex' do
let(:data) { 'No coverage for today :sad:' }
let(:regex) { '\(\d+.\d+\%\) covered' }
it { is_expected.to be_nil }
end
context 'multiple results in content & regex' do
let(:data) { ' (98.39%) covered. (98.29%) covered' }
let(:regex) { '\(\d+.\d+\%\) covered' }
it { is_expected.to eq(98.29) }
end
context 'using a regex capture' do
let(:data) { 'TOTAL 9926 3489 65%' }
let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' }
it { is_expected.to eq(65) }
end
end
end
spec/lib/gitlab/ci/trace_reader_spec.rb deleted 100644 → 0
require 'spec_helper'
describe Gitlab::Ci::TraceReader do
let(:path) { __FILE__ }
let(:lines) { File.readlines(path) }
let(:bytesize) { lines.sum(&:bytesize) }
it 'returns last few lines' do
10.times do
subject = build_subject
last_lines = random_lines
expected = lines.last(last_lines).join
result = subject.read(last_lines: last_lines)
expect(result).to eq(expected)
expect(result.encoding).to eq(Encoding.default_external)
end
end
it 'returns everything if trying to get too many lines' do
result = build_subject.read(last_lines: lines.size * 2)
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
it 'returns all contents if last_lines is not specified' do
result = build_subject.read
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
it 'raises an error if not passing an integer for last_lines' do
expect do
build_subject.read(last_lines: lines)
end.to raise_error(ArgumentError)
end
def random_lines
Random.rand(lines.size) + 1
end
def random_buffer
Random.rand(bytesize) + 1
end
def build_subject
described_class.new(__FILE__, buffer_size: random_buffer)
end
end
spec/lib/gitlab/ci/trace_spec.rb 0 → 100644
require 'spec_helper'
describe Gitlab::Ci::Trace do
let(:build) { create(:ci_build) }
let(:trace) { described_class.new(build) }
describe "associations" do
it { expect(trace).to respond_to(:job) }
it { expect(trace).to delegate_method(:old_trace).to(:job) }
end
describe '#html' do
before do
trace.set("12\n34")
end
it "returns formatted html" do
expect(trace.html).to eq("12<br>34")
end
it "returns last line of formatted html" do
expect(trace.html(last_lines: 1)).to eq("34")
end
end
describe '#raw' do
before do
trace.set("12\n34")
end
it "returns raw output" do
expect(trace.raw).to eq("12\n34")
end
it "returns last line of raw output" do
expect(trace.raw(last_lines: 1)).to eq("34")
end
end
describe '#extract_coverage' do
let(:regex) { '\(\d+.\d+\%\) covered' }
before do
trace.set('Coverage 1033 / 1051 LOC (98.29%) covered')
end
it "returns valid coverage" do
expect(trace.extract_coverage(regex)).to eq(98.29)
end
end
describe '#set' do
before do
trace.set("12")
end
it "returns trace" do
expect(trace.raw).to eq("12")
end
context 'overwrite trace' do
before do
trace.set("34")
end
it "returns new trace" do
expect(trace.raw).to eq("34")
end
end
context 'runners token' do
let(:token) { 'my_secret_token' }
before do
build.project.update(runners_token: token)
trace.set(token)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
context 'hides build token' do
let(:token) { 'my_secret_token' }
before do
build.update(token: token)
trace.set(token)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
end
describe '#append' do
before do
trace.set("1234")
end
it "returns correct trace" do
expect(trace.append("56", 4)).to eq(6)
expect(trace.raw).to eq("123456")
end
context 'tries to append trace at different offset' do
it "fails with append" do
expect(trace.append("56", 2)).to eq(-4)
expect(trace.raw).to eq("1234")
end
end
context 'runners token' do
let(:token) { 'my_secret_token' }
before do
build.project.update(runners_token: token)
trace.append(token, 0)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
context 'build token' do
let(:token) { 'my_secret_token' }
before do
build.update(token: token)
trace.append(token, 0)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
end
describe 'trace handling' do
context 'trace does not exist' do
it { expect(trace.exist?).to be(false) }
end
context 'new trace path is used' do
before do
trace.send(:ensure_directory)
File.open(trace.send(:default_path), "w") do |file|
file.write("data")
end
end
it "trace exist" do
expect(trace.exist?).to be(true)
end
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
end
end
context 'deprecated path' do
let(:path) { trace.send(:deprecated_path) }
context 'with valid ci_id' do
before do
build.project.update(ci_id: 1000)
FileUtils.mkdir_p(File.dirname(path))
File.open(path, "w") do |file|
file.write("data")
end
end
it "trace exist" do
expect(trace.exist?).to be(true)
end
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
end
end
context 'without valid ci_id' do
it "does not return deprecated path" do
expect(path).to be_nil
end
end
end
context 'stored in database' do
before do
build.send(:write_attribute, :trace, "data")
end
it "trace exist" do
expect(trace.exist?).to be(true)
end
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
end
it "returns database data" do
expect(trace.raw).to eq("data")
end
end
end
end
spec/requests/api/jobs_spec.rb
......@@ -320,7 +320,7 @@ describe API::Jobs, api: true do
context 'authorized user' do
it 'returns specific job trace' do
expect(response).to have_http_status(200)
expect(response.body).to eq(build.trace)
expect(response.body).to eq(build.trace.raw)
end
end
......@@ -408,7 +408,7 @@ describe API::Jobs, api: true do
it 'erases job content' do
expect(response).to have_http_status(201)
expect(build.trace).to be_empty
expect(build).not_to have_trace
expect(build.artifacts_file.exists?).to be_falsy
expect(build.artifacts_metadata.exists?).to be_falsy
end
......
spec/requests/api/runner_spec.rb
......@@ -592,7 +592,7 @@ describe API::Runner do
update_job(trace: 'BUILD TRACE UPDATED')
expect(response).to have_http_status(200)
expect(job.reload.trace).to eq 'BUILD TRACE UPDATED'
expect(job.reload.trace.raw).to eq 'BUILD TRACE UPDATED'
end
end
......@@ -600,7 +600,7 @@ describe API::Runner do
it 'does not override trace information' do
update_job
expect(job.reload.trace).to eq 'BUILD TRACE'
expect(job.reload.trace.raw).to eq 'BUILD TRACE'
end
end
......@@ -631,7 +631,7 @@ describe API::Runner do
context 'when request is valid' do
it 'gets correct response' do
expect(response.status).to eq 202
expect(job.reload.trace).to eq 'BUILD TRACE appended'
expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Job-Status'
end
......@@ -642,7 +642,7 @@ describe API::Runner do
it "changes the job's trace" do
patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended appended'
expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end
context 'when Runner makes a force-patch' do
......@@ -651,7 +651,7 @@ describe API::Runner do
it "doesn't change the build.trace" do
force_patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended'
expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
end
end
end
......@@ -664,7 +664,7 @@ describe API::Runner do
it 'changes the job.trace' do
patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended appended'
expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end
context 'when Runner makes a force-patch' do
......@@ -673,7 +673,7 @@ describe API::Runner do
it "doesn't change the job.trace" do
force_patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended'
expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
end
end
end
......@@ -698,7 +698,7 @@ describe API::Runner do
it 'gets correct response' do
expect(response.status).to eq 202
expect(job.reload.trace).to eq 'BUILD TRACE appended'
expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Job-Status'
end
......@@ -738,9 +738,11 @@ describe API::Runner do
def patch_the_trace(content = ' appended', request_headers = nil)
unless request_headers
offset = job.trace_length
limit = offset + content.length - 1
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
job.trace.read do |stream|
offset = stream.size
limit = offset + content.length - 1
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
end
end
Timecop.travel(job.updated_at + update_interval) do
......
spec/requests/api/v3/builds_spec.rb
......@@ -330,7 +330,7 @@ describe API::V3::Builds, api: true do
context 'authorized user' do
it 'returns specific job trace' do
expect(response).to have_http_status(200)
expect(response.body).to eq(build.trace)
expect(response.body).to eq(build.trace.raw)
end
end
......@@ -418,7 +418,7 @@ describe API::V3::Builds, api: true do
it 'erases job content' do
expect(response.status).to eq 201
expect(build.trace).to be_empty
expect(build).not_to have_trace
expect(build.artifacts_file.exists?).to be_falsy
expect(build.artifacts_metadata.exists?).to be_falsy
end
......
spec/requests/ci/api/builds_spec.rb
......@@ -285,7 +285,7 @@ describe Ci::API::Builds do
end
it 'does not override trace information when no trace is given' do
expect(build.reload.trace).to eq 'BUILD TRACE'
expect(build.reload.trace.raw).to eq 'BUILD TRACE'
end
context 'job has been erased' do
......@@ -309,9 +309,11 @@ describe Ci::API::Builds do
def patch_the_trace(content = ' appended', request_headers = nil)
unless request_headers
offset = build.trace_length
limit = offset + content.length - 1
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
build.trace.read do |stream|
offset = stream.size
limit = offset + content.length - 1
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
end
end
Timecop.travel(build.updated_at + update_interval) do
......@@ -335,7 +337,7 @@ describe Ci::API::Builds do
context 'when request is valid' do
it 'gets correct response' do
expect(response.status).to eq 202
expect(build.reload.trace).to eq 'BUILD TRACE appended'
expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Build-Status'
end
......@@ -346,7 +348,7 @@ describe Ci::API::Builds do
it 'changes the build trace' do
patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended appended'
expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end
context 'when Runner makes a force-patch' do
......@@ -355,7 +357,7 @@ describe Ci::API::Builds do
it "doesn't change the build.trace" do
force_patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended'
expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
end
end
end
......@@ -368,7 +370,7 @@ describe Ci::API::Builds do
it 'changes the build.trace' do
patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended appended'
expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end
context 'when Runner makes a force-patch' do
......@@ -377,7 +379,7 @@ describe Ci::API::Builds do
it "doesn't change the build.trace" do
force_patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended'
expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
end
end
end
......@@ -403,7 +405,7 @@ describe Ci::API::Builds do
it 'gets correct response' do
expect(response.status).to eq 202
expect(build.reload.trace).to eq 'BUILD TRACE appended'
expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Build-Status'
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment