BigW Consortium Gitlab

Commit 22b05a1f by Francisco Javier López Committed by Douwe Maan

Extend API for exporting a project with direct upload URL

parent 7c36e856
......@@ -1544,8 +1544,8 @@ class Project < ActiveRecord::Base
@errors = original_errors
end
def add_export_job(current_user:, params: {})
job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params)
def add_export_job(current_user:, after_export_strategy: nil, params: {})
job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params)
if job_id
Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
......@@ -1571,6 +1571,8 @@ class Project < ActiveRecord::Base
def export_status
if export_in_progress?
:started
elsif after_export_in_progress?
:after_export_action
elsif export_project_path
:finished
else
......@@ -1582,12 +1584,22 @@ class Project < ActiveRecord::Base
import_export_shared.active_export_count > 0
end
def after_export_in_progress?
import_export_shared.after_export_in_progress?
end
def remove_exports
return nil unless export_path.present?
FileUtils.rm_rf(export_path)
end
def remove_exported_project_file
return unless export_project_path.present?
FileUtils.rm_f(export_project_path)
end
def full_path_slug
Gitlab::Utils.slugify(full_path.to_s)
end
......
module Projects
module ImportExport
class ExportService < BaseService
def execute(_options = {})
def execute(after_export_strategy = nil, options = {})
@shared = project.import_export_shared
save_all
save_all!
execute_after_export_action(after_export_strategy)
end
private
def save_all
if [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
def execute_after_export_action(after_export_strategy)
return unless after_export_strategy
unless after_export_strategy.execute(current_user, project)
cleanup_and_notify_error
end
end
def save_all!
if save_services
Gitlab::ImportExport::Saver.save(project: project, shared: @shared)
notify_success
else
cleanup_and_notify
cleanup_and_notify_error!
end
end
def save_services
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
end
def version_saver
Gitlab::ImportExport::VersionSaver.new(shared: @shared)
end
......@@ -41,19 +55,22 @@ module Projects
Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
end
def cleanup_and_notify
def cleanup_and_notify_error
Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}")
FileUtils.rm_rf(@shared.export_path)
notify_error
end
def cleanup_and_notify_error!
cleanup_and_notify_error
raise Gitlab::ImportExport::Error.new(@shared.errors.join(', '))
end
def notify_success
Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported")
notification_service.project_exported(@project, @current_user)
end
def notify_error
......
......@@ -4,11 +4,19 @@ class ProjectExportWorker
sidekiq_options retry: 3
def perform(current_user_id, project_id, params = {})
params = params.with_indifferent_access
def perform(current_user_id, project_id, after_export_strategy = {}, params = {})
current_user = User.find(current_user_id)
project = Project.find(project_id)
after_export = build!(after_export_strategy)
::Projects::ImportExport::ExportService.new(project, current_user, params).execute
::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export)
end
private
def build!(after_export_strategy)
strategy_klass = after_export_strategy&.delete('klass')
Gitlab::ImportExport::AfterExportStrategyBuilder.build!(strategy_klass, after_export_strategy)
end
end
---
title: Extend API for exporting a project with direct upload URL
merge_request: 17686
author:
type: added
......@@ -8,6 +8,14 @@
Start a new export.
The endpoint also accepts an `upload` param. This param is a hash that contains
all the necessary information to upload the exported project to a web server or
to any S3-compatible platform. At the moment we only support binary
data file uploads to the final server.
If the `upload` params is present, `upload[url]` param is required.
(**Note:** This feature was introduced in GitLab 10.7)
```http
POST /projects/:id/export
```
......@@ -16,9 +24,12 @@ POST /projects/:id/export
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
| `description` | string | no | Overrides the project description |
| `upload` | hash | no | Hash that contains the information to upload the exported project to a web server |
| `upload[url]` | string | yes | The URL to upload the project |
| `upload[http_method]` | string | no | The HTTP method to upload the exported project. Only `PUT` and `POST` methods allowed. Default is `PUT` |
```console
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "description=Foo Bar" https://gitlab.example.com/api/v4/projects/1/export
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export --data "description=FooBar&upload[http_method]=PUT&upload[url]=https://example-bucket.s3.eu-west-3.amazonaws.com/backup?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMBJHN2O62W8IELQ%2F20180312%2Feu-west-3%2Fs3%2Faws4_request&X-Amz-Date=20180312T110328Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=8413facb20ff33a49a147a0b4abcff4c8487cc33ee1f7e450c46e8f695569dbd"
```
```json
......@@ -43,7 +54,11 @@ GET /projects/:id/export
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export
```
Status can be one of `none`, `started`, or `finished`.
Status can be one of `none`, `started`, `after_export_action` or `finished`. The
`after_export_action` state represents that the export process has been completed successfully and
the platform is performing some actions on the resulted file. For example, sending
an email notifying the user to download the file, uploading the exported file
to a web server, etc.
`_links` are only present when export has finished.
......
......@@ -33,11 +33,28 @@ module API
end
params do
optional :description, type: String, desc: 'Override the project description'
optional :upload, type: Hash do
optional :url, type: String, desc: 'The URL to upload the project'
optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project'
end
end
post ':id/export' do
project_export_params = declared_params(include_missing: false)
after_export_params = project_export_params.delete(:upload) || {}
user_project.add_export_job(current_user: current_user, params: project_export_params)
export_strategy = if after_export_params[:url].present?
params = after_export_params.slice(:url, :http_method).symbolize_keys
Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params)
end
if export_strategy&.invalid?
render_validation_error!(export_strategy)
else
user_project.add_export_job(current_user: current_user,
after_export_strategy: export_strategy,
params: project_export_params)
end
accepted!
end
......
module Gitlab
module ImportExport
module AfterExportStrategies
class BaseAfterExportStrategy
include ActiveModel::Validations
extend Forwardable
StrategyError = Class.new(StandardError)
AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze
private
attr_reader :project, :current_user
public
def initialize(attributes = {})
@options = OpenStruct.new(attributes)
self.class.instance_eval do
def_delegators :@options, *attributes.keys
end
end
def execute(current_user, project)
return unless project&.export_project_path
@project = project
@current_user = current_user
if invalid?
log_validation_errors
return
end
create_or_update_after_export_lock
strategy_execute
true
rescue => e
project.import_export_shared.error(e)
false
ensure
delete_after_export_lock
end
def to_json(options = {})
@options.to_h.merge!(klass: self.class.name).to_json
end
def self.lock_file_path(project)
return unless project&.export_path
File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME)
end
protected
def strategy_execute
raise NotImplementedError
end
private
def create_or_update_after_export_lock
FileUtils.touch(self.class.lock_file_path(project))
end
def delete_after_export_lock
lock_file = self.class.lock_file_path(project)
FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file)
end
def log_validation_errors
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end
end
end
end
end
module Gitlab
module ImportExport
module AfterExportStrategies
class DownloadNotificationStrategy < BaseAfterExportStrategy
private
def strategy_execute
notification_service.project_exported(project, current_user)
end
def notification_service
@notification_service ||= NotificationService.new
end
end
end
end
end
module Gitlab
module ImportExport
module AfterExportStrategies
class WebUploadStrategy < BaseAfterExportStrategy
PUT_METHOD = 'PUT'.freeze
POST_METHOD = 'POST'.freeze
INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze
validates :url, url: true
validate do
unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase)
errors.add(:http_method, INVALID_HTTP_METHOD)
end
end
def initialize(url:, http_method: PUT_METHOD)
super
end
protected
def strategy_execute
handle_response_error(send_file)
project.remove_exported_project_file
end
def handle_response_error(response)
unless response.success?
error_code = response.dig('Error', 'Code') || response.code
error_message = response.dig('Error', 'Message') || response.message
raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}")
end
end
private
def send_file
export_file = File.open(project.export_project_path)
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend
ensure
export_file.close if export_file
end
def send_file_options(export_file)
{
body_stream: export_file,
headers: headers
}
end
def headers
{ 'Content-Length' => File.size(project.export_project_path).to_s }
end
end
end
end
end
module Gitlab
module ImportExport
class AfterExportStrategyBuilder
StrategyNotFoundError = Class.new(StandardError)
def self.build!(strategy_klass, attributes = {})
return default_strategy.new unless strategy_klass
attributes ||= {}
klass = strategy_klass.constantize rescue nil
unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy
raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found")
end
klass.new(**attributes.symbolize_keys)
end
def self.default_strategy
AfterExportStrategies::DownloadNotificationStrategy
end
end
end
end
......@@ -22,7 +22,7 @@ module Gitlab
def error(error)
error_out(error.message, caller[0].dup)
@errors << error.message
add_error_message(error.message)
# Debug:
if error.backtrace
......@@ -32,6 +32,14 @@ module Gitlab
end
end
def add_error_message(error_message)
@errors << error_message
end
def after_export_in_progress?
File.exist?(after_export_lock_file)
end
private
def relative_path
......@@ -45,6 +53,10 @@ module Gitlab
def error_out(message, caller)
Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
end
def after_export_lock_file
AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project)
end
end
end
end
{
"type": "object",
"allOf": [
{ "$ref": "identity.json" },
{
"$ref": "identity.json"
},
{
"required": [
"export_status"
......@@ -9,7 +11,12 @@
"properties": {
"export_status": {
"type": "string",
"enum": ["none", "started", "finished"]
"enum": [
"none",
"started",
"finished",
"after_export_action"
]
}
}
}
......
require 'spec_helper'
describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
let!(:service) { described_class.new }
let!(:project) { create(:project, :with_export) }
let(:shared) { project.import_export_shared }
let!(:user) { create(:user) }
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
end
it 'returns if project exported file is not found' do
allow(project).to receive(:export_project_path).and_return(nil)
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'creates a lock file in the export dir' do
allow(service).to receive(:delete_after_export_lock)
service.execute(user, project)
expect(lock_path_exist?).to be_truthy
end
context 'when the method succeeds' do
it 'removes the lock file' do
service.execute(user, project)
expect(lock_path_exist?).to be_falsey
end
end
context 'when the method fails' do
before do
allow(service).to receive(:strategy_execute).and_call_original
end
context 'when validation fails' do
before do
allow(service).to receive(:invalid?).and_return(true)
end
it 'does not create the lock file' do
expect(service).not_to receive(:create_or_update_after_export_lock)
service.execute(user, project)
end
it 'does not execute main logic' do
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'logs validation errors in shared context' do
expect(service).to receive(:log_validation_errors)
service.execute(user, project)
end
end
context 'when an exception is raised' do
it 'removes the lock' do
expect { service.execute(user, project) }.to raise_error(NotImplementedError)
expect(lock_path_exist?).to be_falsey
end
end
end
end
describe '#log_validation_errors' do
it 'add the message to the shared context' do
errors = %w(test_message test_message2)
allow(service).to receive(:invalid?).and_return(true)
allow(service.errors).to receive(:full_messages).and_return(errors)
expect(shared).to receive(:add_error_message).twice.and_call_original
service.execute(user, project)
expect(shared.errors).to eq errors
end
end
describe '#to_json' do
it 'adds the current strategy class to the serialized attributes' do
params = { param1: 1 }
result = params.merge(klass: described_class.to_s).to_json
expect(described_class.new(params).to_json).to eq result
end
end
def lock_path_exist?
File.exist?(described_class.lock_file_path(project))
end
end
require 'spec_helper'
describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
let(:example_url) { 'http://www.example.com' }
let(:strategy) { subject.new(url: example_url, http_method: 'post') }
let!(:project) { create(:project, :with_export) }
let!(:user) { build(:user) }
subject { described_class }
describe 'validations' do
it 'only POST and PUT method allowed' do
%w(POST post PUT put).each do |method|
expect(subject.new(url: example_url, http_method: method)).to be_valid
end
expect(subject.new(url: example_url, http_method: 'whatever')).not_to be_valid
end
it 'onyl allow urls as upload urls' do
expect(subject.new(url: example_url)).to be_valid
expect(subject.new(url: 'whatever')).not_to be_valid
end
end
describe '#execute' do
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
expect(project).to receive(:remove_exported_project_file)
strategy.execute(user, project)
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::AfterExportStrategyBuilder do
let!(:strategies_namespace) { 'Gitlab::ImportExport::AfterExportStrategies' }
describe '.build!' do
context 'when klass param is' do
it 'null it returns the default strategy' do
expect(described_class.build!(nil).class).to eq described_class.default_strategy
end
it 'not a valid class it raises StrategyNotFoundError exception' do
expect { described_class.build!('Whatever') }.to raise_error(described_class::StrategyNotFoundError)
end
it 'not a descendant of AfterExportStrategy' do
expect { described_class.build!('User') }.to raise_error(described_class::StrategyNotFoundError)
end
end
it 'initializes strategy with attributes param' do
params = { param1: 1, param2: 2, param3: 3 }
strategy = described_class.build!("#{strategies_namespace}::DownloadNotificationStrategy", params)
params.each { |k, v| expect(strategy.public_send(k)).to eq v }
end
end
end
......@@ -2560,7 +2560,7 @@ describe Project do
end
end
describe '#remove_exports' do
describe '#remove_export' do
let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
......@@ -2608,6 +2608,23 @@ describe Project do
end
end
describe '#remove_exported_project_file' do
let(:project) { create(:project, :with_export) }
it 'removes the exported project file' do
exported_file = project.export_project_path
expect(File.exist?(exported_file)).to be_truthy
allow(FileUtils).to receive(:rm_f).and_call_original
expect(FileUtils).to receive(:rm_f).with(exported_file).and_call_original
project.remove_exported_project_file
expect(File.exist?(exported_file)).to be_falsy
end
end
describe '#forks_count' do
it 'returns the number of forks' do
project = build(:project)
......
......@@ -5,6 +5,7 @@ describe API::ProjectExport do
set(:project_none) { create(:project) }
set(:project_started) { create(:project) }
set(:project_finished) { create(:project) }
set(:project_after_export) { create(:project) }
set(:user) { create(:user) }
set(:admin) { create(:admin) }
......@@ -12,11 +13,13 @@ describe API::ProjectExport do
let(:path_none) { "/projects/#{project_none.id}/export" }
let(:path_started) { "/projects/#{project_started.id}/export" }
let(:path_finished) { "/projects/#{project_finished.id}/export" }
let(:path_after_export) { "/projects/#{project_after_export.id}/export" }
let(:download_path) { "/projects/#{project.id}/export/download" }
let(:download_path_none) { "/projects/#{project_none.id}/export/download" }
let(:download_path_started) { "/projects/#{project_started.id}/export/download" }
let(:download_path_finished) { "/projects/#{project_finished.id}/export/download" }
let(:download_path_export_action) { "/projects/#{project_after_export.id}/export/download" }
let(:export_path) { "#{Dir.tmpdir}/project_export_spec" }
......@@ -29,6 +32,11 @@ describe API::ProjectExport do
# simulate exported
FileUtils.mkdir_p project_finished.export_path
FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
# simulate in after export action
FileUtils.mkdir_p project_after_export.export_path
FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz')
FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export)
end
after do
......@@ -73,6 +81,14 @@ describe API::ProjectExport do
expect(json_response['export_status']).to eq('started')
end
it 'is after_export' do
get api(path_after_export, user)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/project/export_status')
expect(json_response['export_status']).to eq('after_export_action')
end
it 'is finished' do
get api(path_finished, user)
......@@ -99,6 +115,7 @@ describe API::ProjectExport do
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
project_after_export.add_master(user)
end
it_behaves_like 'get project export status ok'
......@@ -163,6 +180,36 @@ describe API::ProjectExport do
end
end
shared_examples_for 'get project export upload after action' do
context 'and is uploading' do
it 'downloads' do
get api(download_path_export_action, user)
expect(response).to have_gitlab_http_status(200)
end
end
context 'when upload complete' do
before do
FileUtils.rm_rf(project_after_export.export_path)
end
it_behaves_like '404 response' do
let(:request) { get api(download_path_export_action, user) }
end
end
end
shared_examples_for 'get project download by strategy' do
context 'when upload strategy set' do
it_behaves_like 'get project export upload after action'
end
context 'when download strategy set' do
it_behaves_like 'get project export download'
end
end
it_behaves_like 'when project export is disabled' do
let(:request) { get api(download_path, admin) }
end
......@@ -171,7 +218,7 @@ describe API::ProjectExport do
context 'when user is an admin' do
let(:user) { admin }
it_behaves_like 'get project export download'
it_behaves_like 'get project download by strategy'
end
context 'when user is a master' do
......@@ -180,9 +227,10 @@ describe API::ProjectExport do
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
project_after_export.add_master(user)
end
it_behaves_like 'get project export download'
it_behaves_like 'get project download by strategy'
end
context 'when user is a developer' do
......@@ -229,10 +277,30 @@ describe API::ProjectExport do
end
shared_examples_for 'post project export start' do
it 'starts' do
post api(path, user)
context 'with upload strategy' do
context 'when params invalid' do
it_behaves_like '400 response' do
let(:request) { post(api(path, user), 'upload[url]' => 'whatever') }
end
end
it 'starts' do
allow_any_instance_of(Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy).to receive(:send_file)
post(api(path, user), 'upload[url]' => 'http://gitlab.com')
expect(response).to have_gitlab_http_status(202)
expect(response).to have_gitlab_http_status(202)
end
end
context 'with download strategy' do
it 'starts' do
expect_any_instance_of(Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy).not_to receive(:send_file)
post api(path, user)
expect(response).to have_gitlab_http_status(202)
end
end
end
......@@ -253,6 +321,7 @@ describe API::ProjectExport do
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
project_after_export.add_master(user)
end
it_behaves_like 'post project export start'
......
require 'spec_helper'
describe Projects::ImportExport::ExportService do
describe '#execute' do
let!(:user) { create(:user) }
let(:project) { create(:project) }
let(:shared) { project.import_export_shared }
let(:service) { described_class.new(project, user) }
let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new }
context 'when all saver services succeed' do
before do
allow(service).to receive(:save_services).and_return(true)
end
it 'saves the project in the file system' do
expect(Gitlab::ImportExport::Saver).to receive(:save).with(project: project, shared: shared)
service.execute
end
it 'calls the after export strategy' do
expect(after_export_strategy).to receive(:execute)
service.execute(after_export_strategy)
end
context 'when after export strategy fails' do
before do
allow(after_export_strategy).to receive(:execute).and_return(false)
end
after do
service.execute(after_export_strategy)
end
it 'removes the remaining exported data' do
allow(shared).to receive(:export_path).and_return('whatever')
allow(FileUtils).to receive(:rm_rf)
expect(FileUtils).to receive(:rm_rf).with(shared.export_path)
end
it 'notifies the user' do
expect_any_instance_of(NotificationService).to receive(:project_not_exported)
end
it 'notifies logger' do
allow(Rails.logger).to receive(:error)
expect(Rails.logger).to receive(:error)
end
end
end
context 'when saver services fail' do
before do
allow(service).to receive(:save_services).and_return(false)
end
after do
expect { service.execute }.to raise_error(Gitlab::ImportExport::Error)
end
it 'removes the remaining exported data' do
allow(shared).to receive(:export_path).and_return('whatever')
allow(FileUtils).to receive(:rm_rf)
expect(FileUtils).to receive(:rm_rf).with(shared.export_path)
end
it 'notifies the user' do
expect_any_instance_of(NotificationService).to receive(:project_not_exported)
end
it 'notifies logger' do
expect(Rails.logger).to receive(:error)
end
it 'the after export strategy is not called' do
expect(service).not_to receive(:execute_after_export_action)
end
end
end
end
require 'spec_helper'
describe ProjectExportWorker do
let!(:user) { create(:user) }
let!(:project) { create(:project) }
subject { described_class.new }
describe '#perform' do
context 'when it succeeds' do
it 'calls the ExportService' do
expect_any_instance_of(::Projects::ImportExport::ExportService).to receive(:execute)
subject.perform(user.id, project.id, { 'klass' => 'Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy' })
end
end
context 'when it fails' do
it 'raises an exception when params are invalid' do
expect_any_instance_of(::Projects::ImportExport::ExportService).not_to receive(:execute)
expect { subject.perform(1234, project.id, {}) }.to raise_exception(ActiveRecord::RecordNotFound)
expect { subject.perform(user.id, 1234, {}) }.to raise_exception(ActiveRecord::RecordNotFound)
expect { subject.perform(user.id, project.id, { 'klass' => 'Whatever' }) }.to raise_exception(Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError)
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment