Fix bug where only the first 15 Bitbucket issues would be imported.

CHANGELOG

@@ -2,6 +2,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 
 v 8.0.0 (unreleased)
   - Prevent anchors from being hidden by header (Stan Hu)
+  - Fix bug where only the first 15 Bitbucket issues would be imported (Stan Hu)
   - Sort issues by creation date in Bitbucket importer (Stan Hu)
   - Upgrade gitlab_git to 7.2.15 to fix `git blame` errors with ISO-encoded files (Stan Hu)
   - Prevent too many redirects upon login when home page URL is set to external_url (Stan Hu)

lib/gitlab/bitbucket_import/client.rb

@@ -52,11 +52,26 @@ module Gitlab
       end
 
       def issues(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on").body)
+        all_issues = []
+        offset = 0
+        per_page = 50  # Maximum number allowed by Bitbucket
+        index = 0
+
+        begin
+          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
+          # Find out how many total issues are present
+          total = issues["count"] if index == 0
+          all_issues.concat(issues["issues"])
+          offset += issues["issues"].count
+          index += 1
+        end while all_issues.count < total
+
+        all_issues
       end
 
       def issue_comments(project_identifier, issue_id)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments.sort_by { |comment| comment["utc_created_on"] }
       end
 
       def project(project_identifier)
@@ -100,6 +115,10 @@ module Gitlab
         response
       end
 
+      def issue_api_endpoint(project_identifier, per_page, offset)
+        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
       end

lib/gitlab/bitbucket_import/importer.rb

@@ -20,8 +20,18 @@ module Gitlab
         #Issues && Comments
         issues = client.issues(project_identifier)
 
-        issues["issues"].each do |issue|
-          body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
+        issues.each do |issue|
+          body = ''
+          reporter = nil
+          author = 'Anonymous'
+
+          if issue["reported_by"] && issue["reported_by"]["username"]
+            reporter = issue["reported_by"]["username"]
+            author = reporter
+          end
+
+          body = @formatter.author_line(author)
+          body += issue["content"]
 
           comments = client.issue_comments(project_identifier, issue["local_id"])
 
@@ -30,14 +40,20 @@ module Gitlab
           end
 
           comments.each do |comment|
-            body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"])
+            author = 'Anonymous'
+
+            if comment["author_info"] && comment["author_info"]["username"]
+              author = comment["author_info"]["username"]
+            end
+
+            body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
           end
 
           project.issues.create!(
             description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gl_user_id(project, issue["reported_by"]["username"])
+            author_id: gl_user_id(project, reporter)
           )
         end
 
@@ -47,9 +63,13 @@ module Gitlab
       private
 
       def gl_user_id(project, bitbucket_id)
-        user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
-        (user && user.id) || project.creator_id
-      end
+        if bitbucket_id
+          user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
+          (user && user.id) || project.creator_id
+        else
+          project.creator_id
+        end
+       end
     end
   end
 end

lib/gitlab/github_import/importer.rb

@@ -18,7 +18,8 @@ module Gitlab
                                                   direction: :asc).each do |issue|
           if issue.pull_request.nil?
 
-            body = @formatter.author_line(issue.user.login, issue.body)
+            body = @formatter.author_line(issue.user.login)
+            body += issue.body
 
             if issue.comments > 0
               body += @formatter.comments_header

lib/gitlab/gitlab_import/importer.rb

@@ -18,7 +18,8 @@ module Gitlab
         issues = client.issues(project_identifier)
 
         issues.each do |issue|
-          body = @formatter.author_line(issue["author"]["name"], issue["description"])
+          body = @formatter.author_line(issue["author"]["name"])
+          body += issue["description"]
 
           comments = client.issue_comments(project_identifier, issue["id"])
 

lib/gitlab/import_formatter.rb

@@ -8,8 +8,8 @@ module Gitlab
       "\n\n\n**Imported comments:**\n"
     end
 
-    def author_line(author, body)
-      "*Created by: #{author}*\n\n#{body}"
+    def author_line(author)
+      "*Created by: #{author}*\n\n"
     end
   end
 end

spec/lib/gitlab/bitbucket_import/client_spec.rb

@@ -14,4 +14,38 @@ describe Gitlab::BitbucketImport::Client do
       expect(key).to be_kind_of(Symbol)
     end
   end
+
+  context 'issues' do
+    let(:per_page) { 50 }
+    let(:count) { 95 }
+    let(:sample_issues) do
+      issues = []
+
+      count.times do |i|
+        issues << { local_id: i }
+      end
+
+      issues
+    end
+    let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } }
+    let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } }
+    let(:project_id) { 'namespace/repo' }
+
+    it 'retrieves issues over a number of pages' do
+      stub_request(:get,
+                   "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0").
+          to_return(status: 200,
+                    body: first_sample_data.to_json,
+                    headers: {})
+
+      stub_request(:get,
+                   "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50").
+          to_return(status: 200,
+                    body: second_sample_data.to_json,
+                    headers: {})
+
+      issues = client.issues(project_id)
+      expect(issues.count).to eq(95)
+    end
+  end
 end