BigW Consortium Gitlab
Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
G
gitlab-ce
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Commits
Issue Boards
Open sidebar
Forest Godfrey
gitlab-ce
Commits
59d3e84f
Commit
59d3e84f
authored
May 08, 2018
by
Micaël Bergeron
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
remove the `Upload` redefinition in `MigrateUploadsWorker`
parent
9618e49b
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
95 additions
and
149 deletions
+95
-149
migrate_uploads_worker.rb
app/workers/object_storage/migrate_uploads_worker.rb
+0
-79
migrate_uploads_worker_spec.rb
...ers/workers/object_storage/migrate_uploads_worker_spec.rb
+95
-70
No files found.
app/workers/object_storage/migrate_uploads_worker.rb
View file @
59d3e84f
...
...
@@ -9,85 +9,6 @@ module ObjectStorage
SanityCheckError
=
Class
.
new
(
StandardError
)
class
Upload
<
ActiveRecord
::
Base
# Upper limit for foreground checksum processing
CHECKSUM_THRESHOLD
=
100
.
megabytes
belongs_to
:model
,
polymorphic:
true
# rubocop:disable Cop/PolymorphicAssociations
validates
:size
,
presence:
true
validates
:path
,
presence:
true
validates
:model
,
presence:
true
validates
:uploader
,
presence:
true
before_save
:calculate_checksum!
,
if: :foreground_checksummable?
after_commit
:schedule_checksum
,
if: :checksummable?
scope
:stored_locally
,
->
{
where
(
store:
[
nil
,
ObjectStorage
::
Store
::
LOCAL
])
}
scope
:stored_remotely
,
->
{
where
(
store:
ObjectStorage
::
Store
::
REMOTE
)
}
def
self
.
hexdigest
(
path
)
Digest
::
SHA256
.
file
(
path
).
hexdigest
end
def
absolute_path
raise
ObjectStorage
::
RemoteStoreError
,
"Remote object has no absolute path."
unless
local?
return
path
unless
relative_path?
uploader_class
.
absolute_path
(
self
)
end
def
calculate_checksum!
self
.
checksum
=
nil
return
unless
checksummable?
self
.
checksum
=
self
.
class
.
hexdigest
(
absolute_path
)
end
def
build_uploader
(
mounted_as
=
nil
)
uploader_class
.
new
(
model
,
mounted_as
).
tap
do
|
uploader
|
uploader
.
upload
=
self
uploader
.
retrieve_from_store!
(
identifier
)
end
end
def
exist?
File
.
exist?
(
absolute_path
)
end
def
local?
return
true
if
store
.
nil?
store
==
ObjectStorage
::
Store
::
LOCAL
end
private
def
checksummable?
checksum
.
nil?
&&
local?
&&
exist?
end
def
foreground_checksummable?
checksummable?
&&
size
<=
CHECKSUM_THRESHOLD
end
def
schedule_checksum
UploadChecksumWorker
.
perform_async
(
id
)
end
def
relative_path?
!
path
.
start_with?
(
'/'
)
end
def
identifier
File
.
basename
(
path
)
end
def
uploader_class
Object
.
const_get
(
uploader
)
end
end
class
MigrationResult
attr_reader
:upload
attr_accessor
:error
...
...
spec/uploaders/workers/object_storage/migrate_uploads_worker_spec.rb
View file @
59d3e84f
...
...
@@ -7,113 +7,138 @@ describe ObjectStorage::MigrateUploadsWorker, :sidekiq do
end
end
let!
(
:projects
)
{
create_list
(
:project
,
10
,
:with_avatar
)
}
let
(
:uploads
)
{
Upload
.
all
}
let
(
:model_class
)
{
Project
}
let
(
:
mounted_as
)
{
:avatar
}
let
(
:
uploads
)
{
Upload
.
all
}
let
(
:to_store
)
{
ObjectStorage
::
Store
::
REMOTE
}
before
do
stub_uploads_object_storage
(
AvatarUploader
)
end
describe
'.enqueue!'
do
def
enqueue!
described_class
.
enqueue!
(
uploads
,
Project
,
mounted_as
,
to_store
)
end
shared_examples
"correctly"
do
describe
'.enqueue!'
do
def
enqueue!
described_class
.
enqueue!
(
uploads
,
Project
,
mounted_as
,
to_store
)
end
it
'is guarded by .sanity_check!'
do
expect
(
described_class
).
to
receive
(
:perform_async
)
expect
(
described_class
).
to
receive
(
:sanity_check!
)
it
'is guarded by .sanity_check!'
do
expect
(
described_class
).
to
receive
(
:perform_async
)
expect
(
described_class
).
to
receive
(
:sanity_check!
)
enqueue!
end
enqueue!
end
context
'sanity_check! fails'
do
include_context
'sanity_check! fails'
context
'sanity_check! fails'
do
include_context
'sanity_check! fails'
it
'does not enqueue a job'
do
expect
(
described_class
).
not_to
receive
(
:perform_async
)
it
'does not enqueue a job'
do
expect
(
described_class
).
not_to
receive
(
:perform_async
)
expect
{
enqueue!
}.
to
raise_error
(
described_class
::
SanityCheckError
)
expect
{
enqueue!
}.
to
raise_error
(
described_class
::
SanityCheckError
)
end
end
end
end
describe
'.sanity_check!'
do
shared_examples
'raises a SanityCheckError'
do
let
(
:mount_point
)
{
nil
}
describe
'.sanity_check!'
do
shared_examples
'raises a SanityCheckError'
do
let
(
:mount_point
)
{
nil
}
it
do
expect
{
described_class
.
sanity_check!
(
uploads
,
model_class
,
mount_point
)
}
.
to
raise_error
(
described_class
::
SanityCheckError
)
it
do
expect
{
described_class
.
sanity_check!
(
uploads
,
model_class
,
mount_point
)
}
.
to
raise_error
(
described_class
::
SanityCheckError
)
end
end
end
context
'uploader types mismatch'
do
let!
(
:outlier
)
{
create
(
:upload
,
uploader:
'FileUploader'
)
}
before
do
stub_const
(
"WrongModel"
,
Class
.
new
)
end
include_examples
'raises a SanityCheckError'
end
context
'uploader types mismatch'
do
let!
(
:outlier
)
{
create
(
:upload
,
uploader:
'GitlabUploader'
)
}
context
'model types mismatch'
do
let!
(
:outlier
)
{
create
(
:upload
,
model_type:
'Potato'
)
}
include_examples
'raises a SanityCheckError'
end
include_examples
'raises a SanityCheckError'
end
context
'model types mismatch'
do
let!
(
:outlier
)
{
create
(
:upload
,
model_type:
'WrongModel'
)
}
context
'mount point not found'
do
include_examples
'raises a SanityCheckError'
do
let
(
:mount_point
)
{
:potato
}
include_examples
'raises a SanityCheckError'
end
end
end
describe
'#perform
'
do
def
perform
described_class
.
new
.
perform
(
uploads
.
ids
,
model_class
.
to_s
,
mounted_as
,
to_store
)
rescue
ObjectStorage
::
MigrateUploadsWorker
::
Report
::
MigrationFailures
# swallow
context
'mount point not found
'
do
include_examples
'raises a SanityCheckError'
do
let
(
:mount_point
)
{
:potato
}
end
end
end
shared_examples
'outputs correctly'
do
|
success:
0
,
failures:
0
|
total
=
success
+
failures
describe
'#perform'
do
def
perform
described_class
.
new
.
perform
(
uploads
.
ids
,
model_class
.
to_s
,
mounted_as
,
to_store
)
rescue
ObjectStorage
::
MigrateUploadsWorker
::
Report
::
MigrationFailures
# swallow
end
shared_examples
'outputs correctly'
do
|
success:
0
,
failures:
0
|
total
=
success
+
failures
if
success
>
0
it
'outputs the reports'
do
expect
(
Rails
.
logger
).
to
receive
(
:info
).
with
(
%r{Migrated
#{
success
}
/
#{
total
}
files}
)
if
success
>
0
it
'outputs the reports'
do
expect
(
Rails
.
logger
).
to
receive
(
:info
).
with
(
%r{Migrated
#{
success
}
/
#{
total
}
files}
)
perform
perform
end
end
end
if
failures
>
0
it
'outputs upload failures'
do
expect
(
Rails
.
logger
).
to
receive
(
:warn
).
with
(
/Error .* I am a teapot/
)
if
failures
>
0
it
'outputs upload failures'
do
expect
(
Rails
.
logger
).
to
receive
(
:warn
).
with
(
/Error .* I am a teapot/
)
perform
perform
end
end
end
end
it_behaves_like
'outputs correctly'
,
success:
10
it_behaves_like
'outputs correctly'
,
success:
10
it
'migrates files'
do
perform
it
'migrates files'
do
perform
expect
(
Upload
.
where
(
store:
ObjectStorage
::
Store
::
LOCAL
).
count
).
to
eq
(
0
)
end
aggregate_failures
do
projects
.
each
do
|
project
|
expect
(
project
.
reload
.
avatar
.
upload
.
local?
).
to
be_falsey
context
'migration is unsuccessful'
do
before
do
allow_any_instance_of
(
ObjectStorage
::
Concern
)
.
to
receive
(
:migrate!
).
and_raise
(
CarrierWave
::
UploadError
,
"I am a teapot."
)
end
it_behaves_like
'outputs correctly'
,
failures:
10
end
end
end
context
'migration is unsuccessful'
do
before
do
allow_any_instance_of
(
ObjectStorage
::
Concern
).
to
receive
(
:migrate!
).
and_raise
(
CarrierWave
::
UploadError
,
"I am a teapot."
)
end
context
"for AvatarUploader"
do
let!
(
:projects
)
{
create_list
(
:project
,
10
,
:with_avatar
)
}
let
(
:mounted_as
)
{
:avatar
}
it_behaves_like
'outputs correctly'
,
failures:
10
before
do
stub_uploads_object_storage
(
AvatarUploader
)
end
it_behaves_like
"correctly"
end
context
"for FileUploader"
do
let!
(
:projects
)
{
create_list
(
:project
,
10
)
}
let
(
:secret
)
{
SecureRandom
.
hex
}
let
(
:mounted_as
)
{
nil
}
before
do
stub_uploads_object_storage
(
FileUploader
)
projects
.
map
do
|
project
|
uploader
=
FileUploader
.
new
(
project
)
uploader
.
store!
(
fixture_file_upload
(
'spec/fixtures/doc_sample.txt'
))
end
end
it_behaves_like
"correctly"
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment