BigW Consortium Gitlab
Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
G
gitlab-ce
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Commits
Issue Boards
Open sidebar
Forest Godfrey
gitlab-ce
Commits
6db7c232
Unverified
Commit
6db7c232
authored
Feb 09, 2017
by
Tomasz Maczukin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add minor updates
parent
48cb391c
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
33 additions
and
9 deletions
+33
-9
stuck_ci_builds_worker.rb
app/workers/stuck_ci_builds_worker.rb
+13
-5
stuck_ci_builds_worker_spec.rb
spec/workers/stuck_ci_builds_worker_spec.rb
+20
-4
No files found.
app/workers/stuck_ci_builds_worker.rb
View file @
6db7c232
...
@@ -2,6 +2,8 @@ class StuckCiBuildsWorker
...
@@ -2,6 +2,8 @@ class StuckCiBuildsWorker
include
Sidekiq
::
Worker
include
Sidekiq
::
Worker
include
CronjobQueue
include
CronjobQueue
EXCLUSIVE_LEASE_KEY
=
'stuck_ci_builds_worker_lease'
BUILD_RUNNING_OUTDATED_TIMEOUT
=
1
.
hour
BUILD_RUNNING_OUTDATED_TIMEOUT
=
1
.
hour
BUILD_PENDING_OUTDATED_TIMEOUT
=
1
.
day
BUILD_PENDING_OUTDATED_TIMEOUT
=
1
.
day
BUILD_PENDING_STUCK_TIMEOUT
=
1
.
hour
BUILD_PENDING_STUCK_TIMEOUT
=
1
.
hour
...
@@ -11,20 +13,26 @@ class StuckCiBuildsWorker
...
@@ -11,20 +13,26 @@ class StuckCiBuildsWorker
Rails
.
logger
.
info
"
#{
self
.
class
}
: Cleaning stuck builds"
Rails
.
logger
.
info
"
#{
self
.
class
}
: Cleaning stuck builds"
drop
:running
,
BUILD_RUNNING_OUTDATED_TIMEOUT
drop
:running
,
BUILD_RUNNING_OUTDATED_TIMEOUT
drop
:pending
,
BUILD_PENDING_OUTDATED_TIMEOUT
drop
:pending
,
BUILD_PENDING_OUTDATED_TIMEOUT
drop_stuck
:pending
,
BUILD_PENDING_STUCK_TIMEOUT
drop_stuck
:pending
,
BUILD_PENDING_STUCK_TIMEOUT
remove_lease
end
end
private
private
def
try_obtain_lease
def
try_obtain_lease
Gitlab
::
ExclusiveLease
.
new
(
@uuid
=
Gitlab
::
ExclusiveLease
.
new
(
'stuck_ci_builds_worker_lease'
,
EXCLUSIVE_LEASE_KEY
,
timeout:
30
.
minutes
timeout:
30
.
minutes
).
try_obtain
).
try_obtain
end
end
def
remove_lease
Gitlab
::
ExclusiveLease
.
cancel
(
EXCLUSIVE_LEASE_KEY
,
@uuid
)
end
def
drop
(
status
,
timeout
)
def
drop
(
status
,
timeout
)
search
(
status
,
timeout
)
do
|
build
|
search
(
status
,
timeout
)
do
|
build
|
drop_build
:outdated
,
build
,
status
,
timeout
drop_build
:outdated
,
build
,
status
,
timeout
...
...
spec/workers/stuck_ci_builds_worker_spec.rb
View file @
6db7c232
...
@@ -4,6 +4,7 @@ describe StuckCiBuildsWorker do
...
@@ -4,6 +4,7 @@ describe StuckCiBuildsWorker do
let!
(
:runner
)
{
create
:ci_runner
}
let!
(
:runner
)
{
create
:ci_runner
}
let!
(
:build
)
{
create
:ci_build
,
runner:
runner
}
let!
(
:build
)
{
create
:ci_build
,
runner:
runner
}
let
(
:worker
)
{
described_class
.
new
}
let
(
:worker
)
{
described_class
.
new
}
let
(
:exclusive_lease_uuid
)
{
SecureRandom
.
uuid
}
subject
do
subject
do
build
.
reload
build
.
reload
...
@@ -12,7 +13,7 @@ describe StuckCiBuildsWorker do
...
@@ -12,7 +13,7 @@ describe StuckCiBuildsWorker do
before
do
before
do
build
.
update!
(
status:
status
,
updated_at:
updated_at
)
build
.
update!
(
status:
status
,
updated_at:
updated_at
)
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
true
)
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
exclusive_lease_uuid
)
end
end
shared_examples
'build is dropped'
do
shared_examples
'build is dropped'
do
...
@@ -103,12 +104,26 @@ describe StuckCiBuildsWorker do
...
@@ -103,12 +104,26 @@ describe StuckCiBuildsWorker do
describe
'exclusive lease'
do
describe
'exclusive lease'
do
let
(
:status
)
{
'running'
}
let
(
:status
)
{
'running'
}
let
(
:updated_at
)
{
2
.
days
.
ago
}
let
(
:updated_at
)
{
2
.
days
.
ago
}
let
(
:worker2
)
{
described_class
.
new
}
it
'is guard by exclusive lease'
do
it
'is guard by exclusive lease when executed concurrently'
do
expect
(
worker
).
to
receive
(
:drop
).
at_least
(
:once
)
expect
(
worker2
).
not_to
receive
(
:drop
)
worker
.
perform
worker
.
perform
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
false
)
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
false
)
expect
(
worker
).
not_to
receive
(
:drop
)
worker2
.
perform
end
it
'can be executed in sequence'
do
expect
(
worker
).
to
receive
(
:drop
).
at_least
(
:once
)
expect
(
worker2
).
to
receive
(
:drop
).
at_least
(
:once
)
worker
.
perform
worker2
.
perform
end
it
'cancels exclusive lease after worker perform'
do
expect
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:cancel
).
with
(
described_class
::
EXCLUSIVE_LEASE_KEY
,
exclusive_lease_uuid
)
worker
.
perform
worker
.
perform
end
end
end
end
end
end
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment