BigW Consortium Gitlab

Commit dbcbbf26 by Ahmad Sherif

Speed up label-applying process for GitHub importing

* No need to re-fetch issues from GH to read their labels, the labels are already there from the index request. * No need to look up labels on the database for every application, so we cache them.
parent 395a9301
...@@ -10,6 +10,7 @@ module Gitlab ...@@ -10,6 +10,7 @@ module Gitlab
@repo = project.import_source @repo = project.import_source
@repo_url = project.import_url @repo_url = project.import_url
@errors = [] @errors = []
@labels = {}
if credentials if credentials
@client = Client.new(credentials[:user]) @client = Client.new(credentials[:user])
...@@ -49,7 +50,8 @@ module Gitlab ...@@ -49,7 +50,8 @@ module Gitlab
client.labels(repo, per_page: 100) do |labels| client.labels(repo, per_page: 100) do |labels|
labels.each do |raw| labels.each do |raw|
begin begin
LabelFormatter.new(project, raw).create! label = LabelFormatter.new(project, raw).create!
@labels[label.title] = label.id
rescue => e rescue => e
errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
end end
...@@ -77,7 +79,7 @@ module Gitlab ...@@ -77,7 +79,7 @@ module Gitlab
if gh_issue.valid? if gh_issue.valid?
begin begin
issue = gh_issue.create! issue = gh_issue.create!
apply_labels(issue) apply_labels(issue, raw)
import_comments(issue) if gh_issue.has_comments? import_comments(issue) if gh_issue.has_comments?
rescue => e rescue => e
errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
...@@ -98,7 +100,7 @@ module Gitlab ...@@ -98,7 +100,7 @@ module Gitlab
restore_target_branch(pull_request) unless pull_request.target_branch_exists? restore_target_branch(pull_request) unless pull_request.target_branch_exists?
merge_request = pull_request.create! merge_request = pull_request.create!
apply_labels(merge_request) apply_labels(merge_request, raw)
import_comments(merge_request) import_comments(merge_request)
import_comments_on_diff(merge_request) import_comments_on_diff(merge_request)
rescue => e rescue => e
...@@ -131,12 +133,10 @@ module Gitlab ...@@ -131,12 +133,10 @@ module Gitlab
project.repository.after_remove_branch project.repository.after_remove_branch
end end
def apply_labels(issuable) def apply_labels(issuable, raw_issuable)
issue = client.issue(repo, issuable.iid) if raw_issuable.labels.count > 0
label_ids = raw_issuable.labels
if issue.labels.count > 0 .map { |attrs| @labels[attrs.name] }
label_ids = issue.labels
.map { |attrs| project.labels.find_by(title: attrs.name).try(:id) }
.compact .compact
issuable.update_attribute(:label_ids, label_ids) issuable.update_attribute(:label_ids, label_ids)
......
...@@ -57,7 +57,8 @@ describe Gitlab::GithubImport::Importer, lib: true do ...@@ -57,7 +57,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
created_at: created_at, created_at: created_at,
updated_at: updated_at, updated_at: updated_at,
closed_at: nil, closed_at: nil,
url: 'https://api.github.com/repos/octocat/Hello-World/issues/1347' url: 'https://api.github.com/repos/octocat/Hello-World/issues/1347',
labels: [double(name: 'Label #1')],
) )
end end
...@@ -75,7 +76,8 @@ describe Gitlab::GithubImport::Importer, lib: true do ...@@ -75,7 +76,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
created_at: created_at, created_at: created_at,
updated_at: updated_at, updated_at: updated_at,
closed_at: nil, closed_at: nil,
url: 'https://api.github.com/repos/octocat/Hello-World/issues/1348' url: 'https://api.github.com/repos/octocat/Hello-World/issues/1348',
labels: [double(name: 'Label #2')],
) )
end end
...@@ -94,7 +96,8 @@ describe Gitlab::GithubImport::Importer, lib: true do ...@@ -94,7 +96,8 @@ describe Gitlab::GithubImport::Importer, lib: true do
updated_at: updated_at, updated_at: updated_at,
closed_at: nil, closed_at: nil,
merged_at: nil, merged_at: nil,
url: 'https://api.github.com/repos/octocat/Hello-World/pulls/1347' url: 'https://api.github.com/repos/octocat/Hello-World/pulls/1347',
labels: [double(name: 'Label #3')],
) )
end end
...@@ -148,9 +151,7 @@ describe Gitlab::GithubImport::Importer, lib: true do ...@@ -148,9 +151,7 @@ describe Gitlab::GithubImport::Importer, lib: true do
errors: [ errors: [
{ type: :label, url: "https://api.github.com/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" }, { type: :label, url: "https://api.github.com/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" },
{ type: :milestone, url: "https://api.github.com/repos/octocat/Hello-World/milestones/1", errors: "Validation failed: Title has already been taken" }, { type: :milestone, url: "https://api.github.com/repos/octocat/Hello-World/milestones/1", errors: "Validation failed: Title has already been taken" },
{ type: :issue, url: "https://api.github.com/repos/octocat/Hello-World/issues/1347", errors: "Invalid Repository. Use user/repo format." },
{ type: :issue, url: "https://api.github.com/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank, Title is too short (minimum is 0 characters)" }, { type: :issue, url: "https://api.github.com/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank, Title is too short (minimum is 0 characters)" },
{ type: :pull_request, url: "https://api.github.com/repos/octocat/Hello-World/pulls/1347", errors: "Invalid Repository. Use user/repo format." },
{ type: :pull_request, url: "https://api.github.com/repos/octocat/Hello-World/pulls/1347", errors: "Validation failed: Validate branches Cannot Create: This merge request already exists: [\"New feature\"]" }, { type: :pull_request, url: "https://api.github.com/repos/octocat/Hello-World/pulls/1347", errors: "Validation failed: Validate branches Cannot Create: This merge request already exists: [\"New feature\"]" },
{ type: :wiki, errors: "Gitlab::Shell::Error" }, { type: :wiki, errors: "Gitlab::Shell::Error" },
{ type: :release, url: 'https://api.github.com/repos/octocat/Hello-World/releases/2', errors: "Validation failed: Description can't be blank" } { type: :release, url: 'https://api.github.com/repos/octocat/Hello-World/releases/2', errors: "Validation failed: Description can't be blank" }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment