BigW Consortium Gitlab

Commit 59d3e84f by Micaël Bergeron

remove the `Upload` redefinition in `MigrateUploadsWorker`

parent 9618e49b
...@@ -9,85 +9,6 @@ module ObjectStorage ...@@ -9,85 +9,6 @@ module ObjectStorage
SanityCheckError = Class.new(StandardError) SanityCheckError = Class.new(StandardError)
class Upload < ActiveRecord::Base
# Upper limit for foreground checksum processing
CHECKSUM_THRESHOLD = 100.megabytes
belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
validates :size, presence: true
validates :path, presence: true
validates :model, presence: true
validates :uploader, presence: true
before_save :calculate_checksum!, if: :foreground_checksummable?
after_commit :schedule_checksum, if: :checksummable?
scope :stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) }
scope :stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) }
def self.hexdigest(path)
Digest::SHA256.file(path).hexdigest
end
def absolute_path
raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local?
return path unless relative_path?
uploader_class.absolute_path(self)
end
def calculate_checksum!
self.checksum = nil
return unless checksummable?
self.checksum = self.class.hexdigest(absolute_path)
end
def build_uploader(mounted_as = nil)
uploader_class.new(model, mounted_as).tap do |uploader|
uploader.upload = self
uploader.retrieve_from_store!(identifier)
end
end
def exist?
File.exist?(absolute_path)
end
def local?
return true if store.nil?
store == ObjectStorage::Store::LOCAL
end
private
def checksummable?
checksum.nil? && local? && exist?
end
def foreground_checksummable?
checksummable? && size <= CHECKSUM_THRESHOLD
end
def schedule_checksum
UploadChecksumWorker.perform_async(id)
end
def relative_path?
!path.start_with?('/')
end
def identifier
File.basename(path)
end
def uploader_class
Object.const_get(uploader)
end
end
class MigrationResult class MigrationResult
attr_reader :upload attr_reader :upload
attr_accessor :error attr_accessor :error
......
...@@ -7,113 +7,138 @@ describe ObjectStorage::MigrateUploadsWorker, :sidekiq do ...@@ -7,113 +7,138 @@ describe ObjectStorage::MigrateUploadsWorker, :sidekiq do
end end
end end
let!(:projects) { create_list(:project, 10, :with_avatar) }
let(:uploads) { Upload.all }
let(:model_class) { Project } let(:model_class) { Project }
let(:mounted_as) { :avatar } let(:uploads) { Upload.all }
let(:to_store) { ObjectStorage::Store::REMOTE } let(:to_store) { ObjectStorage::Store::REMOTE }
before do shared_examples "correctly" do
stub_uploads_object_storage(AvatarUploader) describe '.enqueue!' do
end def enqueue!
described_class.enqueue!(uploads, Project, mounted_as, to_store)
describe '.enqueue!' do end
def enqueue!
described_class.enqueue!(uploads, Project, mounted_as, to_store)
end
it 'is guarded by .sanity_check!' do it 'is guarded by .sanity_check!' do
expect(described_class).to receive(:perform_async) expect(described_class).to receive(:perform_async)
expect(described_class).to receive(:sanity_check!) expect(described_class).to receive(:sanity_check!)
enqueue! enqueue!
end end
context 'sanity_check! fails' do context 'sanity_check! fails' do
include_context 'sanity_check! fails' include_context 'sanity_check! fails'
it 'does not enqueue a job' do it 'does not enqueue a job' do
expect(described_class).not_to receive(:perform_async) expect(described_class).not_to receive(:perform_async)
expect { enqueue! }.to raise_error(described_class::SanityCheckError) expect { enqueue! }.to raise_error(described_class::SanityCheckError)
end
end end
end end
end
describe '.sanity_check!' do describe '.sanity_check!' do
shared_examples 'raises a SanityCheckError' do shared_examples 'raises a SanityCheckError' do
let(:mount_point) { nil } let(:mount_point) { nil }
it do it do
expect { described_class.sanity_check!(uploads, model_class, mount_point) } expect { described_class.sanity_check!(uploads, model_class, mount_point) }
.to raise_error(described_class::SanityCheckError) .to raise_error(described_class::SanityCheckError)
end
end end
end
context 'uploader types mismatch' do before do
let!(:outlier) { create(:upload, uploader: 'FileUploader') } stub_const("WrongModel", Class.new)
end
include_examples 'raises a SanityCheckError' context 'uploader types mismatch' do
end let!(:outlier) { create(:upload, uploader: 'GitlabUploader') }
context 'model types mismatch' do include_examples 'raises a SanityCheckError'
let!(:outlier) { create(:upload, model_type: 'Potato') } end
include_examples 'raises a SanityCheckError' context 'model types mismatch' do
end let!(:outlier) { create(:upload, model_type: 'WrongModel') }
context 'mount point not found' do include_examples 'raises a SanityCheckError'
include_examples 'raises a SanityCheckError' do
let(:mount_point) { :potato }
end end
end
end
describe '#perform' do context 'mount point not found' do
def perform include_examples 'raises a SanityCheckError' do
described_class.new.perform(uploads.ids, model_class.to_s, mounted_as, to_store) let(:mount_point) { :potato }
rescue ObjectStorage::MigrateUploadsWorker::Report::MigrationFailures end
# swallow end
end end
shared_examples 'outputs correctly' do |success: 0, failures: 0| describe '#perform' do
total = success + failures def perform
described_class.new.perform(uploads.ids, model_class.to_s, mounted_as, to_store)
rescue ObjectStorage::MigrateUploadsWorker::Report::MigrationFailures
# swallow
end
shared_examples 'outputs correctly' do |success: 0, failures: 0|
total = success + failures
if success > 0 if success > 0
it 'outputs the reports' do it 'outputs the reports' do
expect(Rails.logger).to receive(:info).with(%r{Migrated #{success}/#{total} files}) expect(Rails.logger).to receive(:info).with(%r{Migrated #{success}/#{total} files})
perform perform
end
end end
end
if failures > 0 if failures > 0
it 'outputs upload failures' do it 'outputs upload failures' do
expect(Rails.logger).to receive(:warn).with(/Error .* I am a teapot/) expect(Rails.logger).to receive(:warn).with(/Error .* I am a teapot/)
perform perform
end
end end
end end
end
it_behaves_like 'outputs correctly', success: 10 it_behaves_like 'outputs correctly', success: 10
it 'migrates files' do
perform
it 'migrates files' do expect(Upload.where(store: ObjectStorage::Store::LOCAL).count).to eq(0)
perform end
aggregate_failures do context 'migration is unsuccessful' do
projects.each do |project| before do
expect(project.reload.avatar.upload.local?).to be_falsey allow_any_instance_of(ObjectStorage::Concern)
.to receive(:migrate!).and_raise(CarrierWave::UploadError, "I am a teapot.")
end end
it_behaves_like 'outputs correctly', failures: 10
end end
end end
end
context 'migration is unsuccessful' do context "for AvatarUploader" do
before do let!(:projects) { create_list(:project, 10, :with_avatar) }
allow_any_instance_of(ObjectStorage::Concern).to receive(:migrate!).and_raise(CarrierWave::UploadError, "I am a teapot.") let(:mounted_as) { :avatar }
end
it_behaves_like 'outputs correctly', failures: 10 before do
stub_uploads_object_storage(AvatarUploader)
end
it_behaves_like "correctly"
end
context "for FileUploader" do
let!(:projects) { create_list(:project, 10) }
let(:secret) { SecureRandom.hex }
let(:mounted_as) { nil }
before do
stub_uploads_object_storage(FileUploader)
projects.map do |project|
uploader = FileUploader.new(project)
uploader.store!(fixture_file_upload('spec/fixtures/doc_sample.txt'))
end
end end
it_behaves_like "correctly"
end end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment