Merge branch 'bitbucket-oauth2' into 'master'

Gemfile

@@ -22,7 +22,6 @@ gem 'doorkeeper',             '~> 4.2.0'
 gem 'omniauth',               '~> 1.3.1'
 gem 'omniauth-auth0',         '~> 1.4.1'
 gem 'omniauth-azure-oauth2',  '~> 0.0.6'
-gem 'omniauth-bitbucket',     '~> 0.0.2'
 gem 'omniauth-cas3',          '~> 1.1.2'
 gem 'omniauth-facebook',      '~> 4.0.0'
 gem 'omniauth-github',        '~> 1.1.1'

Gemfile.lock

@@ -432,10 +432,6 @@ GEM
       jwt (~> 1.0)
       omniauth (~> 1.0)
       omniauth-oauth2 (~> 1.1)
-    omniauth-bitbucket (0.0.2)
-      multi_json (~> 1.7)
-      omniauth (~> 1.1)
-      omniauth-oauth (~> 1.0)
     omniauth-cas3 (1.1.3)
       addressable (~> 2.3)
       nokogiri (~> 1.6.6)
@@ -902,7 +898,6 @@ DEPENDENCIES
   omniauth (~> 1.3.1)
   omniauth-auth0 (~> 1.4.1)
   omniauth-azure-oauth2 (~> 0.0.6)
-  omniauth-bitbucket (~> 0.0.2)
   omniauth-cas3 (~> 1.1.2)
   omniauth-facebook (~> 4.0.0)
   omniauth-github (~> 1.1.1)

app/controllers/application_controller.rb

@@ -262,7 +262,7 @@ class ApplicationController < ActionController::Base
   end
 
   def bitbucket_import_configured?
-    Gitlab::OAuth::Provider.enabled?(:bitbucket) && Gitlab::BitbucketImport.public_key.present?
+    Gitlab::OAuth::Provider.enabled?(:bitbucket)
   end
 
   def google_code_import_enabled?

app/controllers/import/bitbucket_controller.rb

@@ -2,50 +2,57 @@ class Import::BitbucketController < Import::BaseController
   before_action :verify_bitbucket_import_enabled
   before_action :bitbucket_auth, except: :callback
 
-  rescue_from OAuth::Error, with: :bitbucket_unauthorized
-  rescue_from Gitlab::BitbucketImport::Client::Unauthorized, with: :bitbucket_unauthorized
+  rescue_from OAuth2::Error, with: :bitbucket_unauthorized
+  rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized
 
   def callback
-    request_token = session.delete(:oauth_request_token)
-    raise "Session expired!" if request_token.nil?
+    response = client.auth_code.get_token(params[:code], redirect_uri: callback_import_bitbucket_url)
 
-    request_token.symbolize_keys!
-
-    access_token = client.get_token(request_token, params[:oauth_verifier], callback_import_bitbucket_url)
-
-    session[:bitbucket_access_token] = access_token.token
-    session[:bitbucket_access_token_secret] = access_token.secret
+    session[:bitbucket_token]         = response.token
+    session[:bitbucket_expires_at]    = response.expires_at
+    session[:bitbucket_expires_in]    = response.expires_in
+    session[:bitbucket_refresh_token] = response.refresh_token
 
     redirect_to status_import_bitbucket_url
   end
 
   def status
-    @repos = client.projects
-    @incompatible_repos = client.incompatible_projects
+    bitbucket_client = Bitbucket::Client.new(credentials)
+    repos = bitbucket_client.repos
+
+    @repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
 
-    @already_added_projects = current_user.created_projects.where(import_type: "bitbucket")
+    @already_added_projects = current_user.created_projects.where(import_type: 'bitbucket')
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
-    @repos.to_a.reject!{ |repo| already_added_projects_names.include? "#{repo["owner"]}/#{repo["slug"]}" }
+    @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) }
   end
 
   def jobs
-    jobs = current_user.created_projects.where(import_type: "bitbucket").to_json(only: [:id, :import_status])
-    render json: jobs
+    render json: current_user.created_projects
+                             .where(import_type: 'bitbucket')
+                             .to_json(only: [:id, :import_status])
   end
 
   def create
+    bitbucket_client = Bitbucket::Client.new(credentials)
+
     @repo_id = params[:repo_id].to_s
-    repo = client.project(@repo_id.gsub('___', '/'))
-    @project_name = repo['slug']
-    @target_namespace = find_or_create_namespace(repo['owner'], client.user['user']['username'])
+    name = @repo_id.gsub('___', '/')
+    repo = bitbucket_client.repo(name)
+    @project_name = params[:new_name].presence || repo.name
 
-    unless Gitlab::BitbucketImport::KeyAdder.new(repo, current_user, access_params).execute
-      render 'deploy_key' and return
-    end
+    repo_owner = repo.owner
+    repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
+    @target_namespace = params[:new_namespace].presence || repo_owner
+
+    namespace = find_or_create_namespace(@target_namespace, current_user)
 
-    if current_user.can?(:create_projects, @target_namespace)
-      @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @target_namespace, current_user, access_params).execute
+    if current_user.can?(:create_projects, namespace)
+      # The token in a session can be expired, we need to get most recent one because
+      # Bitbucket::Connection class refreshes it.
+      session[:bitbucket_token] = bitbucket_client.connection.token
+      @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @project_name, namespace, current_user, credentials).execute
     else
       render 'unauthorized'
     end
@@ -54,8 +61,15 @@ class Import::BitbucketController < Import::BaseController
   private
 
   def client
-    @client ||= Gitlab::BitbucketImport::Client.new(session[:bitbucket_access_token],
-                                                    session[:bitbucket_access_token_secret])
+    @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+  end
+
+  def provider
+    Gitlab::OAuth::Provider.config_for('bitbucket')
+  end
+
+  def options
+    OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
   end
 
   def verify_bitbucket_import_enabled
@@ -63,26 +77,23 @@ class Import::BitbucketController < Import::BaseController
   end
 
   def bitbucket_auth
-    if session[:bitbucket_access_token].blank?
-      go_to_bitbucket_for_permissions
-    end
+    go_to_bitbucket_for_permissions if session[:bitbucket_token].blank?
   end
 
   def go_to_bitbucket_for_permissions
-    request_token = client.request_token(callback_import_bitbucket_url)
-    session[:oauth_request_token] = request_token
-
-    redirect_to client.authorize_url(request_token, callback_import_bitbucket_url)
+    redirect_to client.auth_code.authorize_url(redirect_uri: callback_import_bitbucket_url)
   end
 
   def bitbucket_unauthorized
     go_to_bitbucket_for_permissions
   end
 
-  def access_params
+  def credentials
     {
-      bitbucket_access_token: session[:bitbucket_access_token],
-      bitbucket_access_token_secret: session[:bitbucket_access_token_secret]
+      token: session[:bitbucket_token],
+      expires_at: session[:bitbucket_expires_at],
+      expires_in: session[:bitbucket_expires_in],
+      refresh_token: session[:bitbucket_refresh_token]
     }
   end
 end

app/views/import/bitbucket/status.html.haml

-- page_title "Bitbucket import"
-- header_title "Projects", root_path
+- page_title 'Bitbucket import'
+- header_title 'Projects', root_path
+
 %h3.page-title
   %i.fa.fa-bitbucket
   Import projects from Bitbucket
@@ -10,13 +11,13 @@
   %hr
   %p
   - if @incompatible_repos.any?
-    = button_tag class: "btn btn-import btn-success js-import-all" do
+    = button_tag class: 'btn btn-import btn-success js-import-all' do
       Import all compatible projects
-      = icon("spinner spin", class: "loading-icon")
+      = icon('spinner spin', class: 'loading-icon')
   - else
-    = button_tag class: "btn btn-success js-import-all" do
+    = button_tag class: 'btn btn-import btn-success js-import-all' do
       Import all projects
-      = icon("spinner spin", class: "loading-icon")
+      = icon('spinner spin', class: 'loading-icon')
 
 .table-responsive
   %table.table.import-jobs
@@ -32,7 +33,7 @@
       - @already_added_projects.each do |project|
         %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
           %td
-            = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: "_blank"
+            = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: '_blank'
           %td
             = link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
           %td.job-status
@@ -47,31 +48,41 @@
               = project.human_import_status_name
 
       - @repos.each do |repo|
-        %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"}
+        %tr{id: "repo_#{repo.owner}___#{repo.slug}"}
           %td
-            = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank"
+            = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: "_blank"
           %td.import-target
-            = import_project_target(repo['owner'], repo['slug'])
+            %fieldset.row
+            .input-group
+              .project-path.input-group-btn
+                - if current_user.can_select_namespace?
+                  - selected = params[:namespace_id] || :current_user
+                  - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner, path: repo.owner) } : {}
+                  = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 }
+                - else
+                  = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true
+              %span.input-group-addon /
+              = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
           %td.import-actions.job-status
-            = button_tag class: "btn btn-import js-add-to-import" do
+            = button_tag class: 'btn btn-import js-add-to-import' do
               Import
-              = icon("spinner spin", class: "loading-icon")
+              = icon('spinner spin', class: 'loading-icon')
       - @incompatible_repos.each do |repo|
-        %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"}
+        %tr{id: "repo_#{repo.owner}___#{repo.slug}"}
           %td
-            = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank"
+            = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: '_blank'
           %td.import-target
           %td.import-actions-job-status
-            = label_tag "Incompatible Project", nil, class: "label label-danger"
+            = label_tag 'Incompatible Project', nil, class: 'label label-danger'
 
 - if @incompatible_repos.any?
   %p
     One or more of your Bitbucket projects cannot be imported into GitLab
     directly because they use Subversion or Mercurial for version control,
     rather than Git. Please convert
-    = link_to "them to Git,", "https://www.atlassian.com/git/tutorials/migrating-overview"
+    = link_to 'them to Git,', 'https://www.atlassian.com/git/tutorials/migrating-overview'
     and go through the
-    = link_to "import flow", status_import_bitbucket_path, "data-no-turbolink" => "true"
+    = link_to 'import flow', status_import_bitbucket_path, 'data-no-turbolink' => 'true'
     again.
 
 .js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_path}", import_path: "#{import_bitbucket_path}" } }

changelogs/unreleased/bitbucket-oauth2.yml

+---
+title: Refactor Bitbucket importer to use BitBucket API Version 2
+merge_request: 
+author: 

config/initializers/omniauth.rb

@@ -26,3 +26,9 @@ if Gitlab.config.omniauth.enabled
     end
   end
 end
+
+module OmniAuth
+  module Strategies
+    autoload :Bitbucket, Rails.root.join('lib', 'omniauth', 'strategies', 'bitbucket')
+  end
+end

config/initializers/public_key.rb

-path = File.expand_path("~/.ssh/bitbucket_rsa.pub")
-Gitlab::BitbucketImport.public_key = File.read(path) if File.exist?(path)

doc/README.md

@@ -8,7 +8,7 @@
 - [GitLab as OAuth2 authentication service provider](integration/oauth_provider.md). It allows you to login to other applications from GitLab.
 - [Container Registry](user/project/container_registry.md) Learn how to use GitLab Container Registry.
 - [GitLab basics](gitlab-basics/README.md) Find step by step how to start working on your commandline and on GitLab.
-- [Importing to GitLab](workflow/importing/README.md).
+- [Importing to GitLab](workflow/importing/README.md) Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab.
 - [Importing and exporting projects between instances](user/project/settings/import_export.md).
 - [Markdown](user/markdown.md) GitLab's advanced formatting system.
 - [Migrating from SVN](workflow/importing/migrating_from_svn.md) Convert a SVN repository to Git and GitLab.

doc/integration/bitbucket.md

@@ -18,8 +18,10 @@ Bitbucket.org.
 ## Bitbucket OmniAuth provider
 
 > **Note:**
-Make sure to first follow the [Initial OmniAuth configuration][init-oauth]
-before proceeding with setting up the Bitbucket integration.
+GitLab 8.15 significantly simplified the way to integrate Bitbucket.org with
+GitLab. You are encouraged to upgrade your GitLab instance if you haven't done
+already. If you're using GitLab 8.14 and below, [use the previous integration
+docs][bb-old].
 
 To enable the Bitbucket OmniAuth provider you must register your application
 with Bitbucket.org. Bitbucket will generate an application ID and secret key for
@@ -44,14 +46,12 @@ you to use.
     And grant at least the following permissions:
 
     ```
-    Account: Email
-    Repositories: Read, Admin
+    Account: Email, Read
+    Repositories: Read
+    Pull Requests: Read
+    Issues: Read
     ```
 
-    >**Note:**
-    It may seem a little odd to giving GitLab admin permissions to repositories,
-    but this is needed in order for GitLab to be able to clone the repositories.
-
     ![Bitbucket OAuth settings page](img/bitbucket_oauth_settings_page.png)
 
 1.  Select **Save**.
@@ -93,7 +93,8 @@ you to use.
     ```yaml
     - { name: 'bitbucket',
         app_id: 'BITBUCKET_APP_KEY',
-        app_secret: 'BITBUCKET_APP_SECRET' }
+        app_secret: 'BITBUCKET_APP_SECRET',
+        url: 'https://bitbucket.org/' }
     ```
 
     ---
@@ -112,100 +113,12 @@ well, the user will be returned to GitLab and will be signed in.
 
 ## Bitbucket project import
 
-To allow projects to be imported directly into GitLab, Bitbucket requires two
-extra setup steps compared to [GitHub](github.md) and [GitLab.com](gitlab.md).
-
-Bitbucket doesn't allow OAuth applications to clone repositories over HTTPS, and
-instead requires GitLab to use SSH and identify itself using your GitLab
-server's SSH key.
-
-To be able to access repositories on Bitbucket, GitLab will automatically
-register your public key with Bitbucket as a deploy key for the repositories to
-be imported. Your public key needs to be at `~/.ssh/bitbucket_rsa` which
-translates to `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages and to
-`/home/git/.ssh/bitbucket_rsa` for installations from source.
-
----
-
-Below are the steps that will allow GitLab to be able to import your projects
-from Bitbucket.
-
-1. Make sure you [have enabled the Bitbucket OAuth support](#bitbucket-omniauth-provider).
-1. Create a new SSH key with an **empty passphrase**:
-
-    ```sh
-    sudo -u git -H ssh-keygen
-    ```
-
-    When asked to 'Enter file in which to save the key' enter:
-    `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages or
-    `/home/git/.ssh/bitbucket_rsa` for installations from source. The name is
-    important so make sure to get it right.
-
-    > **Warning:**
-    This key must NOT be associated with ANY existing Bitbucket accounts. If it
-    is, the import will fail with an `Access denied! Please verify you can add
-    deploy keys to this repository.` error.
-
-1. Next, you need to to configure the SSH client to use your new key. Open the
-   SSH configuration file of the `git` user:
-
-    ```
-    # For Omnibus packages
-    sudo editor /var/opt/gitlab/.ssh/config
-
-    # For installations from source
-    sudo editor /home/git/.ssh/config
-    ```
-
-1. Add a host configuration for `bitbucket.org`:
-
-    ```sh
-    Host bitbucket.org
-      IdentityFile ~/.ssh/bitbucket_rsa
-      User git
-    ```
-
-1. Save the file and exit.
-1. Manually connect to `bitbucket.org` over SSH, while logged in as the `git`
-   user that GitLab will use:
-
-    ```sh
-    sudo -u git -H ssh bitbucket.org
-    ```
-
-    That step is performed because GitLab needs to connect to Bitbucket over SSH,
-    in order to add `bitbucket.org` to your GitLab server's known SSH hosts.
-
-1.  Verify the RSA key fingerprint you'll see in the response matches the one
-    in the [Bitbucket documentation][bitbucket-docs] (the specific IP address
-    doesn't matter):
-
-    ```sh
-    The authenticity of host 'bitbucket.org (104.192.143.1)' can't be established.
-    RSA key fingerprint is SHA256:zzXQOXSRBEiUtuE8AikJYKwbHaxvSc0ojez9YXaGp1A.
-    Are you sure you want to continue connecting (yes/no)?
-    ```
-
-1. If the fingerprint matches, type `yes` to continue connecting and have
-   `bitbucket.org` be added to your known SSH hosts. After confirming you should
-   see a permission denied message. If you see an authentication successful
-   message you have done something wrong. The key you are using has already been
-   added to a Bitbucket account and will cause the import script to fail. Ensure
-   the key you are using CANNOT authenticate with Bitbucket.
-1. Restart GitLab to allow it to find the new public key.
-
-Your GitLab server is now able to connect to Bitbucket over SSH. You should be
-able to see the "Import projects from Bitbucket" option on the New Project page
-enabled.
-
-## Acknowledgements
-
-Special thanks to the writer behind the following article:
-
-- http://stratus3d.com/blog/2015/09/06/migrating-from-bitbucket-to-local-gitlab-server/
+Once the above configuration is set up, you can use Bitbucket to sign into
+GitLab and [start importing your projects][bb-import].
 
 [init-oauth]: omniauth.md#initial-omniauth-configuration
+[bb-import]: ../workflow/importing/import_projects_from_bitbucket.md
+[bb-old]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-14-stable/doc/integration/bitbucket.md
 [bitbucket-docs]: https://confluence.atlassian.com/bitbucket/use-the-ssh-protocol-with-bitbucket-cloud-221449711.html#UsetheSSHprotocolwithBitbucketCloud-KnownhostorBitbucket%27spublickeyfingerprints
 [reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
 [restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/workflow/importing/import_projects_from_bitbucket.md

 # Import your project from Bitbucket to GitLab
 
-It takes just a few steps to import your existing Bitbucket projects to GitLab. But keep in mind that it is possible only if Bitbucket support is enabled on your GitLab instance. You can read more about Bitbucket support [here](../../integration/bitbucket.md).
+Import your projects from Bitbucket to GitLab with minimal effort.
 
-* Sign in to GitLab.com and go to your dashboard
+## Overview
 
-* Click on "New project"
+>**Note:**
+The [Bitbucket integration][bb-import] must be first enabled in order to be
+able to import your projects from Bitbucket. Ask your GitLab administrator
+to enable this if not already.
 
-![New project in GitLab](bitbucket_importer/bitbucket_import_new_project.png)
+- At its current state, the Bitbucket importer can import:
+  - the repository description (GitLab 7.7+)
+  - the Git repository data (GitLab 7.7+)
+  - the issues (GitLab 7.7+)
+  - the issue comments (GitLab 8.15+)
+  - the pull requests (GitLab 8.4+)
+  - the pull request comments (GitLab 8.15+)
+  - the milestones (GitLab 8.15+)
+- References to pull requests and issues are preserved (GitLab 8.7+)
+- Repository public access is retained. If a repository is private in Bitbucket
+  it will be created as private in GitLab as well.
 
-* Click on the "Bitbucket" button
 
-![Bitbucket](bitbucket_importer/bitbucket_import_select_bitbucket.png)
+## How it works
 
-* Grant GitLab access to your Bitbucket account
+When issues/pull requests are being imported, the Bitbucket importer tries to find
+the Bitbucket author/assignee in GitLab's database using the Bitbucket ID. For this
+to work, the Bitbucket author/assignee should have signed in beforehand in GitLab
+and [**associated their Bitbucket account**][social sign-in]. If the user is not
+found in GitLab's database, the project creator (most of the times the current
+user that started the import process) is set as the author, but a reference on
+the issue about the original Bitbucket author is kept.
 
-![Grant access](bitbucket_importer/bitbucket_import_grant_access.png)
+The importer will create any new namespaces (groups) if they don't exist or in
+the case the namespace is taken, the repository will be imported under the user's
+namespace that started the import process.
 
-* Click on the projects that you'd like to import or "Import all projects"
+## Importing your Bitbucket repositories
 
-![Import projects](bitbucket_importer/bitbucket_import_select_project.png)
+1. Sign in to GitLab and go to your dashboard.
+1. Click on **New project**.
 
-A new GitLab project will be created with your imported data.
+    ![New project in GitLab](img/bitbucket_import_new_project.png)
 
-### Note
-Milestones and wiki pages are not imported from Bitbucket.
+1. Click on the "Bitbucket" button
+
+    ![Bitbucket](img/import_projects_from_new_project_page.png)
+
+1. Grant GitLab access to your Bitbucket account
+
+    ![Grant access](img/bitbucket_import_grant_access.png)
+
+1. Click on the projects that you'd like to import or **Import all projects**.
+   You can also select the namespace under which each project will be
+   imported.
+
+    ![Import projects](img/bitbucket_import_select_project.png)
+
+[bb-import]: ../../integration/bitbucket.md
+[social sign-in]: ../../user/profile/account/social_sign_in.md

doc/workflow/importing/import_projects_from_github.md

@@ -40,7 +40,7 @@ namespace that started the import process.
 
 The importer page is visible when you create a new project.
 
-![New project page on GitLab](img/import_projects_from_github_new_project_page.png)
+![New project page on GitLab](img/import_projects_from_new_project_page.png)
 
 Click on the **GitHub** link and the import authorization process will start.
 There are two ways to authorize access to your GitHub repositories:

lib/bitbucket/client.rb

+module Bitbucket
+  class Client
+    attr_reader :connection
+
+    def initialize(options = {})
+      @connection = Connection.new(options)
+    end
+
+    def issues(repo)
+      path = "/repositories/#{repo}/issues"
+      get_collection(path, :issue)
+    end
+
+    def issue_comments(repo, issue_id)
+      path = "/repositories/#{repo}/issues/#{issue_id}/comments"
+      get_collection(path, :comment)
+    end
+
+    def pull_requests(repo)
+      path = "/repositories/#{repo}/pullrequests?state=ALL"
+      get_collection(path, :pull_request)
+    end
+
+    def pull_request_comments(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments"
+      get_collection(path, :pull_request_comment)
+    end
+
+    def pull_request_diff(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff"
+      connection.get(path)
+    end
+
+    def repo(name)
+      parsed_response = connection.get("/repositories/#{name}")
+      Representation::Repo.new(parsed_response)
+    end
+
+    def repos
+      path = "/repositories?role=member"
+      get_collection(path, :repo)
+    end
+
+    def user
+      @user ||= begin
+        parsed_response = connection.get('/user')
+        Representation::User.new(parsed_response)
+      end
+    end
+
+    private
+
+    def get_collection(path, type)
+      paginator = Paginator.new(connection, path, type)
+      Collection.new(paginator)
+    end
+  end
+end

lib/bitbucket/collection.rb

+module Bitbucket
+  class Collection < Enumerator
+    def initialize(paginator)
+      super() do |yielder|
+        loop do
+          paginator.items.each { |item| yielder << item }
+        end
+      end
+
+      lazy
+    end
+
+    def method_missing(method, *args)
+      return super unless self.respond_to?(method)
+
+      self.send(method, *args) do |item|
+        block_given? ? yield(item) : item
+      end
+    end
+  end
+end

lib/bitbucket/connection.rb

+module Bitbucket
+  class Connection
+    DEFAULT_API_VERSION = '2.0'
+    DEFAULT_BASE_URI    = 'https://api.bitbucket.org/'
+    DEFAULT_QUERY       = {}
+
+    attr_reader :expires_at, :expires_in, :refresh_token, :token
+
+    def initialize(options = {})
+      @api_version   = options.fetch(:api_version, DEFAULT_API_VERSION)
+      @base_uri      = options.fetch(:base_uri, DEFAULT_BASE_URI)
+      @default_query = options.fetch(:query, DEFAULT_QUERY)
+
+      @token         = options[:token]
+      @expires_at    = options[:expires_at]
+      @expires_in    = options[:expires_in]
+      @refresh_token = options[:refresh_token]
+    end
+
+    def get(path, extra_query = {})
+      refresh! if expired?
+
+      response = connection.get(build_url(path), params: @default_query.merge(extra_query))
+      response.parsed
+    end
+
+    def expired?
+      connection.expired?
+    end
+
+    def refresh!
+      response = connection.refresh!
+
+      @token         = response.token
+      @expires_at    = response.expires_at
+      @expires_in    = response.expires_in
+      @refresh_token = response.refresh_token
+      @connection    = nil
+    end
+
+    private
+
+    def client
+      @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+    end
+
+    def connection
+      @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in)
+    end
+
+    def build_url(path)
+      return path if path.starts_with?(root_url)
+
+      "#{root_url}#{path}"
+    end
+
+    def root_url
+      @root_url ||= "#{@base_uri}#{@api_version}"
+    end
+
+    def provider
+      Gitlab::OAuth::Provider.config_for('bitbucket')
+    end
+
+    def options
+      OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
+    end
+  end
+end

lib/bitbucket/error/unauthorized.rb

+module Bitbucket
+  module Error
+    class Unauthorized < StandardError
+    end
+  end
+end

lib/bitbucket/page.rb

+module Bitbucket
+  class Page
+    attr_reader :attrs, :items
+
+    def initialize(raw, type)
+      @attrs = parse_attrs(raw)
+      @items = parse_values(raw, representation_class(type))
+    end
+
+    def next?
+      attrs.fetch(:next, false)
+    end
+
+    def next
+      attrs.fetch(:next)
+    end
+
+    private
+
+    def parse_attrs(raw)
+      raw.slice(*%w(size page pagelen next previous)).symbolize_keys
+    end
+
+    def parse_values(raw, bitbucket_rep_class)
+      return [] unless raw['values'] && raw['values'].is_a?(Array)
+
+      bitbucket_rep_class.decorate(raw['values'])
+    end
+
+    def representation_class(type)
+      Bitbucket::Representation.const_get(type.to_s.camelize)
+    end
+  end
+end

lib/bitbucket/paginator.rb

+module Bitbucket
+  class Paginator
+    PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100.
+
+    def initialize(connection, url, type)
+      @connection = connection
+      @type = type
+      @url = url
+      @page = nil
+    end
+
+    def items
+      raise StopIteration unless has_next_page?
+
+      @page = fetch_next_page
+      @page.items
+    end
+
+    private
+
+    attr_reader :connection, :page, :url, :type
+
+    def has_next_page?
+      page.nil? || page.next?
+    end
+
+    def next_url
+      page.nil? ? url : page.next
+    end
+
+    def fetch_next_page
+      parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
+      Page.new(parsed_response, type)
+    end
+  end
+end

lib/bitbucket/representation/base.rb

+module Bitbucket
+  module Representation
+    class Base
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def self.decorate(entries)
+        entries.map { |entry| new(entry)}
+      end
+
+      private
+
+      attr_reader :raw
+    end
+  end
+end

lib/bitbucket/representation/comment.rb

+module Bitbucket
+  module Representation
+    class Comment < Representation::Base
+      def author
+        user['username']
+      end
+
+      def note
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on'] || raw['created_on']
+      end
+
+      private
+
+      def user
+        raw.fetch('user', {})
+      end
+    end
+  end
+end

lib/bitbucket/representation/issue.rb

+module Bitbucket
+  module Representation
+    class Issue < Representation::Base
+      CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze
+
+      def iid
+        raw['id']
+      end
+
+      def kind
+        raw['kind']
+      end
+
+      def author
+        raw.fetch('reporter', {}).fetch('username', nil)
+      end
+
+      def description
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def state
+        closed? ? 'closed' : 'opened'
+      end
+
+      def title
+        raw['title']
+      end
+
+      def milestone
+        raw['milestone']['name'] if raw['milestone'].present?
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['edited_on']
+      end
+
+      def to_s
+        iid
+      end
+
+      private
+
+      def closed?
+        CLOSED_STATUS.include?(raw['state'])
+      end
+    end
+  end
+end

lib/bitbucket/representation/pull_request.rb

+module Bitbucket
+  module Representation
+    class PullRequest < Representation::Base
+      def author
+        raw.fetch('author', {}).fetch('username', nil)
+      end
+
+      def description
+        raw['description']
+      end
+
+      def iid
+        raw['id']
+      end
+
+      def state
+        if raw['state'] == 'MERGED'
+          'merged'
+        elsif raw['state'] == 'DECLINED'
+          'closed'
+        else
+          'opened'
+        end
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def source_branch_name
+        source_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def source_branch_sha
+        source_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      def target_branch_name
+        target_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def target_branch_sha
+        target_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      private
+
+      def source_branch
+        raw['source']
+      end
+
+      def target_branch
+        raw['destination']
+      end
+    end
+  end
+end

lib/bitbucket/representation/pull_request_comment.rb

+module Bitbucket
+  module Representation
+    class PullRequestComment < Comment
+      def iid
+        raw['id']
+      end
+
+      def file_path
+        inline.fetch('path')
+      end
+
+      def old_pos
+        inline.fetch('from')
+      end
+
+      def new_pos
+        inline.fetch('to')
+      end
+
+      def parent_id
+        raw.fetch('parent', {}).fetch('id', nil)
+      end
+
+      def inline?
+        raw.has_key?('inline')
+      end
+
+      def has_parent?
+        raw.has_key?('parent')
+      end
+
+      private
+
+      def inline
+        raw.fetch('inline', {})
+      end
+    end
+  end
+end

lib/bitbucket/representation/repo.rb

+module Bitbucket
+  module Representation
+    class Repo < Representation::Base
+      attr_reader :owner, :slug
+
+      def initialize(raw)
+        super(raw)
+      end
+
+      def owner_and_slug
+        @owner_and_slug ||= full_name.split('/', 2)
+      end
+
+      def owner
+        owner_and_slug.first
+      end
+
+      def slug
+        owner_and_slug.last
+      end
+
+      def clone_url(token = nil)
+        url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href')
+
+        if token.present?
+          clone_url = URI::parse(url)
+          clone_url.user = "x-token-auth:#{token}"
+          clone_url.to_s
+        else
+          url
+        end
+      end
+
+      def description
+        raw['description']
+      end
+
+      def full_name
+        raw['full_name']
+      end
+
+      def issues_enabled?
+        raw['has_issues']
+      end
+
+      def name
+        raw['name']
+      end
+
+      def valid?
+        raw['scm'] == 'git'
+      end
+
+      def visibility_level
+        if raw['is_private']
+          Gitlab::VisibilityLevel::PRIVATE
+        else
+          Gitlab::VisibilityLevel::PUBLIC
+        end
+      end
+
+      def to_s
+        full_name
+      end
+    end
+  end
+end

lib/bitbucket/representation/user.rb

+module Bitbucket
+  module Representation
+    class User < Representation::Base
+      def username
+        raw['username']
+      end
+    end
+  end
+end

lib/gitlab/bitbucket_import.rb

-module Gitlab
-  module BitbucketImport
-    mattr_accessor :public_key
-    @public_key = nil
-  end
-end

lib/gitlab/bitbucket_import/client.rb

-module Gitlab
-  module BitbucketImport
-    class Client
-      class Unauthorized < StandardError; end
-
-      attr_reader :consumer, :api
-
-      def self.from_project(project)
-        import_data_credentials = project.import_data.credentials if project.import_data
-        if import_data_credentials && import_data_credentials[:bb_session]
-          token = import_data_credentials[:bb_session][:bitbucket_access_token]
-          token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret]
-          new(token, token_secret)
-        else
-          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}"
-        end
-      end
-
-      def initialize(access_token = nil, access_token_secret = nil)
-        @consumer = ::OAuth::Consumer.new(
-          config.app_id,
-          config.app_secret,
-          bitbucket_options
-        )
-
-        if access_token && access_token_secret
-          @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret)
-        end
-      end
-
-      def request_token(redirect_uri)
-        request_token = consumer.get_request_token(oauth_callback: redirect_uri)
-
-        {
-          oauth_token:              request_token.token,
-          oauth_token_secret:       request_token.secret,
-          oauth_callback_confirmed: request_token.callback_confirmed?.to_s
-        }
-      end
-
-      def authorize_url(request_token, redirect_uri)
-        request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
-        if request_token.callback_confirmed?
-          request_token.authorize_url
-        else
-          request_token.authorize_url(oauth_callback: redirect_uri)
-        end
-      end
-
-      def get_token(request_token, oauth_verifier, redirect_uri)
-        request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
-        if request_token.callback_confirmed?
-          request_token.get_access_token(oauth_verifier: oauth_verifier)
-        else
-          request_token.get_access_token(oauth_callback: redirect_uri)
-        end
-      end
-
-      def user
-        JSON.parse(get("/api/1.0/user").body)
-      end
-
-      def issues(project_identifier)
-        all_issues = []
-        offset = 0
-        per_page = 50 # Maximum number allowed by Bitbucket
-        index = 0
-
-        begin
-          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
-          # Find out how many total issues are present
-          total = issues["count"] if index == 0
-          all_issues.concat(issues["issues"])
-          offset += issues["issues"].count
-          index += 1
-        end while all_issues.count < total
-
-        all_issues
-      end
-
-      def issue_comments(project_identifier, issue_id)
-        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
-        comments.sort_by { |comment| comment["utc_created_on"] }
-      end
-
-      def project(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body)
-      end
-
-      def find_deploy_key(project_identifier, key)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key|
-          deploy_key["key"].chomp == key.chomp
-        end
-      end
-
-      def add_deploy_key(project_identifier, key)
-        deploy_key = find_deploy_key(project_identifier, key)
-        return if deploy_key
-
-        JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body)
-      end
-
-      def delete_deploy_key(project_identifier, key)
-        deploy_key = find_deploy_key(project_identifier, key)
-        return unless deploy_key
-
-        api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204"
-      end
-
-      def projects
-        JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" }
-      end
-
-      def incompatible_projects
-        JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" }
-      end
-
-      private
-
-      def get(url)
-        response = api.get(url)
-        raise Unauthorized if (400..499).cover?(response.code.to_i)
-
-        response
-      end
-
-      def issue_api_endpoint(project_identifier, per_page, offset)
-        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
-      end
-
-      def config
-        Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" }
-      end
-
-      def bitbucket_options
-        OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys
-      end
-    end
-  end
-end

lib/gitlab/bitbucket_import/key_adder.rb

-module Gitlab
-  module BitbucketImport
-    class KeyAdder
-      attr_reader :repo, :current_user, :client
-
-      def initialize(repo, current_user, access_params)
-        @repo, @current_user = repo, current_user
-        @client = Client.new(access_params[:bitbucket_access_token],
-                             access_params[:bitbucket_access_token_secret])
-      end
-
-      def execute
-        return false unless BitbucketImport.public_key.present?
-
-        project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
-        client.add_deploy_key(project_identifier, BitbucketImport.public_key)
-
-        true
-      rescue
-        false
-      end
-    end
-  end
-end

lib/gitlab/bitbucket_import/key_deleter.rb

-module Gitlab
-  module BitbucketImport
-    class KeyDeleter
-      attr_reader :project, :current_user, :client
-
-      def initialize(project)
-        @project = project
-        @current_user = project.creator
-        @client = Client.from_project(@project)
-      end
-
-      def execute
-        return false unless BitbucketImport.public_key.present?
-
-        client.delete_deploy_key(project.import_source, BitbucketImport.public_key)
-
-        true
-      rescue
-        false
-      end
-    end
-  end
-end

lib/gitlab/bitbucket_import/project_creator.rb

 module Gitlab
   module BitbucketImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user, session_data)
+      def initialize(repo, name, namespace, current_user, session_data)
         @repo = repo
+        @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
@@ -13,15 +14,15 @@ module Gitlab
       def execute
         ::Projects::CreateService.new(
           current_user,
-          name: repo["name"],
-          path: repo["slug"],
-          description: repo["description"],
+          name: name,
+          path: name,
+          description: repo.description,
           namespace_id: namespace.id,
-          visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
-          import_type: "bitbucket",
-          import_source: "#{repo["owner"]}/#{repo["slug"]}",
-          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git",
-          import_data: { credentials: { bb_session: session_data } }
+          visibility_level: repo.visibility_level,
+          import_type: 'bitbucket',
+          import_source: repo.full_name,
+          import_url: repo.clone_url(session_data[:token]),
+          import_data: { credentials: session_data }
         ).execute
       end
     end

lib/omniauth/strategies/bitbucket.rb

+require 'omniauth-oauth2'
+
+module OmniAuth
+  module Strategies
+    class Bitbucket < OmniAuth::Strategies::OAuth2
+      option :name, 'bitbucket'
+
+      option :client_options, {
+        site: 'https://bitbucket.org',
+        authorize_url: 'https://bitbucket.org/site/oauth2/authorize',
+        token_url: 'https://bitbucket.org/site/oauth2/access_token'
+      }
+
+      uid do
+        raw_info['username']
+      end
+
+      info do
+        {
+          name: raw_info['display_name'],
+          avatar: raw_info['links']['avatar']['href'],
+          email: primary_email
+        }
+      end
+
+      def raw_info
+        @raw_info ||= access_token.get('api/2.0/user').parsed
+      end
+
+      def primary_email
+        primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] }
+        primary && primary['email'] || nil
+      end
+
+      def emails
+        email_response = access_token.get('api/2.0/user/emails').parsed
+        @emails ||= email_response && email_response['values'] || nil
+      end
+    end
+  end
+end

spec/controllers/import/bitbucket_controller_spec.rb

@@ -6,11 +6,11 @@ describe Import::BitbucketController do
   let(:user) { create(:user) }
   let(:token) { "asdasd12345" }
   let(:secret) { "sekrettt" }
-  let(:access_params) { { bitbucket_access_token: token, bitbucket_access_token_secret: secret } }
+  let(:refresh_token) { SecureRandom.hex(15) }
+  let(:access_params) { { token: token, expires_at: nil, expires_in: nil, refresh_token: nil } }
 
   def assign_session_tokens
-    session[:bitbucket_access_token] = token
-    session[:bitbucket_access_token_secret] = secret
+    session[:bitbucket_token] = token
   end
 
   before do
@@ -24,29 +24,36 @@ describe Import::BitbucketController do
     end
 
     it "updates access token" do
-      access_token = double(token: token, secret: secret)
-      allow_any_instance_of(Gitlab::BitbucketImport::Client).
+      expires_at = Time.now + 1.day
+      expires_in = 1.day
+      access_token = double(token: token,
+                            secret: secret,
+                            expires_at: expires_at,
+                            expires_in: expires_in,
+                            refresh_token: refresh_token)
+      allow_any_instance_of(OAuth2::Client).
         to receive(:get_token).and_return(access_token)
       stub_omniauth_provider('bitbucket')
 
       get :callback
 
-      expect(session[:bitbucket_access_token]).to eq(token)
-      expect(session[:bitbucket_access_token_secret]).to eq(secret)
+      expect(session[:bitbucket_token]).to eq(token)
+      expect(session[:bitbucket_refresh_token]).to eq(refresh_token)
+      expect(session[:bitbucket_expires_at]).to eq(expires_at)
+      expect(session[:bitbucket_expires_in]).to eq(expires_in)
       expect(controller).to redirect_to(status_import_bitbucket_url)
     end
   end
 
   describe "GET status" do
     before do
-      @repo = OpenStruct.new(slug: 'vim', owner: 'asd')
+      @repo = double(slug: 'vim', owner: 'asd', full_name: 'asd/vim', "valid?" => true)
       assign_session_tokens
     end
 
     it "assigns variables" do
       @project = create(:project, import_type: 'bitbucket', creator_id: user.id)
-      client = stub_client(projects: [@repo])
-      allow(client).to receive(:incompatible_projects).and_return([])
+      allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo])
 
       get :status
 
@@ -57,7 +64,7 @@ describe Import::BitbucketController do
 
     it "does not show already added project" do
       @project = create(:project, import_type: 'bitbucket', creator_id: user.id, import_source: 'asd/vim')
-      stub_client(projects: [@repo])
+      allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo])
 
       get :status
 
@@ -70,19 +77,16 @@ describe Import::BitbucketController do
     let(:bitbucket_username) { user.username }
 
     let(:bitbucket_user) do
-      { user: { username: bitbucket_username } }.with_indifferent_access
+      double(username: bitbucket_username)
     end
 
     let(:bitbucket_repo) do
-      { slug: "vim", owner: bitbucket_username }.with_indifferent_access
+      double(slug: "vim", owner: bitbucket_username, name: 'vim')
     end
 
     before do
-      allow(Gitlab::BitbucketImport::KeyAdder).
-        to receive(:new).with(bitbucket_repo, user, access_params).
-        and_return(double(execute: true))
-
-      stub_client(user: bitbucket_user, project: bitbucket_repo)
+      allow_any_instance_of(Bitbucket::Client).to receive(:repo).and_return(bitbucket_repo)
+      allow_any_instance_of(Bitbucket::Client).to receive(:user).and_return(bitbucket_user)
       assign_session_tokens
     end
 
@@ -90,7 +94,7 @@ describe Import::BitbucketController do
       context "when the Bitbucket user and GitLab user's usernames match" do
         it "takes the current user's namespace" do
           expect(Gitlab::BitbucketImport::ProjectCreator).
-            to receive(:new).with(bitbucket_repo, user.namespace, user, access_params).
+            to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params).
             and_return(double(execute: true))
 
           post :create, format: :js
@@ -102,7 +106,7 @@ describe Import::BitbucketController do
 
         it "takes the current user's namespace" do
           expect(Gitlab::BitbucketImport::ProjectCreator).
-            to receive(:new).with(bitbucket_repo, user.namespace, user, access_params).
+            to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params).
             and_return(double(execute: true))
 
           post :create, format: :js
@@ -114,7 +118,7 @@ describe Import::BitbucketController do
       let(:other_username) { "someone_else" }
 
       before do
-        bitbucket_repo["owner"] = other_username
+        allow(bitbucket_repo).to receive(:owner).and_return(other_username)
       end
 
       context "when a namespace with the Bitbucket user's username already exists" do
@@ -123,7 +127,7 @@ describe Import::BitbucketController do
         context "when the namespace is owned by the GitLab user" do
           it "takes the existing namespace" do
             expect(Gitlab::BitbucketImport::ProjectCreator).
-              to receive(:new).with(bitbucket_repo, existing_namespace, user, access_params).
+              to receive(:new).with(bitbucket_repo, bitbucket_repo.name, existing_namespace, user, access_params).
               and_return(double(execute: true))
 
             post :create, format: :js
@@ -156,7 +160,7 @@ describe Import::BitbucketController do
 
           it "takes the new namespace" do
             expect(Gitlab::BitbucketImport::ProjectCreator).
-              to receive(:new).with(bitbucket_repo, an_instance_of(Group), user, access_params).
+              to receive(:new).with(bitbucket_repo, bitbucket_repo.name, an_instance_of(Group), user, access_params).
               and_return(double(execute: true))
 
             post :create, format: :js
@@ -177,7 +181,7 @@ describe Import::BitbucketController do
 
           it "takes the current user's namespace" do
             expect(Gitlab::BitbucketImport::ProjectCreator).
-              to receive(:new).with(bitbucket_repo, user.namespace, user, access_params).
+              to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params).
               and_return(double(execute: true))
 
             post :create, format: :js

spec/lib/bitbucket/collection_spec.rb

+require 'spec_helper'
+
+# Emulates paginator. It returns 2 pages with results
+class TestPaginator
+  def initialize
+    @current_page = 0
+  end
+
+  def items
+    @current_page += 1
+
+    raise StopIteration if @current_page > 2
+
+    ["result_1_page_#{@current_page}", "result_2_page_#{@current_page}"]
+  end
+end
+
+describe Bitbucket::Collection do
+  it "iterates paginator" do
+    collection = described_class.new(TestPaginator.new)
+
+    expect(collection.to_a).to match(["result_1_page_1", "result_2_page_1", "result_1_page_2", "result_2_page_2"])
+  end
+end

spec/lib/bitbucket/connection_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Connection do
+  before do
+    allow_any_instance_of(described_class).to receive(:provider).and_return(double(app_id: '', app_secret: ''))
+  end
+
+  describe '#get' do
+    it 'calls OAuth2::AccessToken::get' do
+      expect_any_instance_of(OAuth2::AccessToken).to receive(:get).and_return(double(parsed: true))
+
+      connection = described_class.new({})
+
+      connection.get('/users')
+    end
+  end
+
+  describe '#expired?' do
+    it 'calls connection.expired?' do
+      expect_any_instance_of(OAuth2::AccessToken).to receive(:expired?).and_return(true)
+
+      expect(described_class.new({}).expired?).to be_truthy
+    end
+  end
+
+  describe '#refresh!' do
+    it 'calls connection.refresh!' do
+      response = double(token: nil, expires_at: nil, expires_in: nil, refresh_token: nil)
+
+      expect_any_instance_of(OAuth2::AccessToken).to receive(:refresh!).and_return(response)
+
+      described_class.new({}).refresh!
+    end
+  end
+end

spec/lib/bitbucket/page_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Page do
+  let(:response) { { 'values' => [{ 'username' => 'Ben' }], 'pagelen' => 2, 'next' => '' } }
+
+  before do
+    # Autoloading hack
+    Bitbucket::Representation::User.new({})
+  end
+
+  describe '#items' do
+    it 'returns collection of needed objects' do
+      page = described_class.new(response, :user)
+
+      expect(page.items.first).to be_a(Bitbucket::Representation::User)
+      expect(page.items.count).to eq(1)
+    end
+  end
+
+  describe '#attrs' do
+    it 'returns attributes' do
+      page = described_class.new(response, :user)
+
+      expect(page.attrs.keys).to include(:pagelen, :next)
+    end
+  end
+
+  describe '#next?' do
+    it 'returns true' do
+      page = described_class.new(response, :user)
+
+      expect(page.next?).to be_truthy
+    end
+
+    it 'returns false' do
+      response['next'] = nil
+      page = described_class.new(response, :user)
+
+      expect(page.next?).to be_falsey
+    end
+  end
+
+  describe '#next' do
+    it 'returns next attribute' do
+      page = described_class.new(response, :user)
+
+      expect(page.next).to eq('')
+    end
+  end
+end

spec/lib/bitbucket/paginator_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Paginator do
+  let(:last_page) { double(:page, next?: false, items: ['item_2']) }
+  let(:first_page) { double(:page, next?: true, next: last_page, items: ['item_1']) }
+
+  describe 'items' do
+    it 'return items and raises StopIteration in the end' do
+      paginator = described_class.new(nil, nil, nil)
+
+      allow(paginator).to receive(:fetch_next_page).and_return(first_page)
+      expect(paginator.items).to match(['item_1'])
+
+      allow(paginator).to receive(:fetch_next_page).and_return(last_page)
+      expect(paginator.items).to match(['item_2'])
+
+      allow(paginator).to receive(:fetch_next_page).and_return(nil)
+      expect{ paginator.items }.to raise_error(StopIteration)
+    end
+  end
+end

spec/lib/bitbucket/representation/comment_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Representation::Comment do
+  describe '#author' do
+    it { expect(described_class.new('user' => { 'username' => 'Ben' }).author).to eq('Ben') }
+    it { expect(described_class.new({}).author).to be_nil }
+  end
+
+  describe '#note' do
+    it { expect(described_class.new('content' => { 'raw' => 'Text' }).note).to eq('Text') }
+    it { expect(described_class.new({}).note).to be_nil }
+  end
+
+  describe '#created_at' do
+    it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) }
+  end
+
+  describe '#updated_at' do
+    it { expect(described_class.new('updated_on' => Date.today).updated_at).to eq(Date.today) }
+    it { expect(described_class.new('created_on' => Date.today).updated_at).to eq(Date.today) }
+  end
+end

spec/lib/bitbucket/representation/issue_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Representation::Issue do
+  describe '#iid' do
+    it { expect(described_class.new('id' => 1).iid).to eq(1) }
+  end
+
+  describe '#kind' do
+    it { expect(described_class.new('kind' => 'bug').kind).to eq('bug') }
+  end
+
+  describe '#milestone' do
+    it { expect(described_class.new({ 'milestone' => { 'name' => '1.0' } }).milestone).to eq('1.0') }
+    it { expect(described_class.new({}).milestone).to be_nil }
+  end
+
+  describe '#author' do
+    it { expect(described_class.new({ 'reporter' => { 'username' => 'Ben' } }).author).to eq('Ben') }
+    it { expect(described_class.new({}).author).to be_nil }
+  end
+
+  describe '#description' do
+    it { expect(described_class.new({ 'content' => { 'raw' => 'Text' } }).description).to eq('Text') }
+    it { expect(described_class.new({}).description).to be_nil }
+  end
+
+  describe '#state' do
+    it { expect(described_class.new({ 'state' => 'invalid' }).state).to eq('closed') }
+    it { expect(described_class.new({ 'state' => 'wontfix' }).state).to eq('closed') }
+    it { expect(described_class.new({ 'state' => 'resolved' }).state).to eq('closed') }
+    it { expect(described_class.new({ 'state' => 'duplicate' }).state).to eq('closed') }
+    it { expect(described_class.new({ 'state' => 'closed' }).state).to eq('closed') }
+    it { expect(described_class.new({ 'state' => 'opened' }).state).to eq('opened') }
+  end
+
+  describe '#title' do
+    it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') }
+  end
+
+  describe '#created_at' do
+    it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) }
+  end
+
+  describe '#updated_at' do
+    it { expect(described_class.new('edited_on' => Date.today).updated_at).to eq(Date.today) }
+  end
+end

spec/lib/bitbucket/representation/pull_request_comment_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Representation::PullRequestComment do
+  describe '#iid' do
+    it { expect(described_class.new('id' => 1).iid).to eq(1) }
+  end
+
+  describe '#file_path' do
+    it { expect(described_class.new('inline' => { 'path' => '/path' }).file_path).to eq('/path') }
+  end
+
+  describe '#old_pos' do
+    it { expect(described_class.new('inline' => { 'from' => 3 }).old_pos).to eq(3) }
+  end
+
+  describe '#new_pos' do
+    it { expect(described_class.new('inline' => { 'to' => 3 }).new_pos).to eq(3) }
+  end
+
+  describe '#parent_id' do
+    it { expect(described_class.new({ 'parent' => { 'id' => 2 } }).parent_id).to eq(2) }
+    it { expect(described_class.new({}).parent_id).to be_nil }
+  end
+
+  describe '#inline?' do
+    it { expect(described_class.new('inline' => {}).inline?).to be_truthy }
+    it { expect(described_class.new({}).inline?).to be_falsey }
+  end
+
+  describe '#has_parent?' do
+    it { expect(described_class.new('parent' => {}).has_parent?).to be_truthy }
+    it { expect(described_class.new({}).has_parent?).to be_falsey }
+  end
+end

spec/lib/bitbucket/representation/pull_request_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Representation::PullRequest do
+  describe '#iid' do
+    it { expect(described_class.new('id' => 1).iid).to eq(1) }
+  end
+
+  describe '#author' do
+    it { expect(described_class.new({ 'author' => { 'username' => 'Ben' } }).author).to eq('Ben') }
+    it { expect(described_class.new({}).author).to be_nil }
+  end
+
+  describe '#description' do
+    it { expect(described_class.new({ 'description' => 'Text' }).description).to eq('Text') }
+    it { expect(described_class.new({}).description).to be_nil }
+  end
+
+  describe '#state' do
+    it { expect(described_class.new({ 'state' => 'MERGED' }).state).to eq('merged') }
+    it { expect(described_class.new({ 'state' => 'DECLINED' }).state).to eq('closed') }
+    it { expect(described_class.new({}).state).to eq('opened') }
+  end
+
+  describe '#title' do
+    it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') }
+  end
+
+  describe '#source_branch_name' do
+    it { expect(described_class.new({ source: { branch: { name: 'feature' } } }.with_indifferent_access).source_branch_name).to eq('feature') }
+    it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_name).to be_nil }
+  end
+
+  describe '#source_branch_sha' do
+    it { expect(described_class.new({ source: { commit: { hash: 'abcd123' } } }.with_indifferent_access).source_branch_sha).to eq('abcd123') }
+    it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_sha).to be_nil }
+  end
+
+  describe '#target_branch_name' do
+    it { expect(described_class.new({ destination: { branch: { name: 'master' } } }.with_indifferent_access).target_branch_name).to eq('master') }
+    it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_name).to be_nil }
+  end
+
+  describe '#target_branch_sha' do
+    it { expect(described_class.new({ destination: { commit: { hash: 'abcd123' } } }.with_indifferent_access).target_branch_sha).to eq('abcd123') }
+    it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_sha).to be_nil }
+  end
+end

spec/lib/bitbucket/representation/user_spec.rb

+require 'spec_helper'
+
+describe Bitbucket::Representation::User do
+  describe '#username' do
+    it 'returns correct value' do
+      user = described_class.new('username' => 'Ben')
+
+      expect(user.username).to eq('Ben')
+    end
+  end
+end

spec/lib/gitlab/bitbucket_import/client_spec.rb

-require 'spec_helper'
-
-describe Gitlab::BitbucketImport::Client, lib: true do
-  include ImportSpecHelper
-
-  let(:token) { '123456' }
-  let(:secret) { 'secret' }
-  let(:client) { Gitlab::BitbucketImport::Client.new(token, secret) }
-
-  before do
-    stub_omniauth_provider('bitbucket')
-  end
-
-  it 'all OAuth client options are symbols' do
-    client.consumer.options.keys.each do |key|
-      expect(key).to be_kind_of(Symbol)
-    end
-  end
-
-  context 'issues' do
-    let(:per_page) { 50 }
-    let(:count) { 95 }
-    let(:sample_issues) do
-      issues = []
-
-      count.times do |i|
-        issues << { local_id: i }
-      end
-
-      issues
-    end
-    let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } }
-    let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } }
-    let(:project_id) { 'namespace/repo' }
-
-    it 'retrieves issues over a number of pages' do
-      stub_request(:get,
-                   "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0").
-        to_return(status: 200,
-                  body: first_sample_data.to_json,
-                  headers: {})
-
-      stub_request(:get,
-                   "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50").
-        to_return(status: 200,
-                  body: second_sample_data.to_json,
-                  headers: {})
-
-      issues = client.issues(project_id)
-      expect(issues.count).to eq(95)
-    end
-  end
-
-  context 'project import' do
-    it 'calls .from_project with no errors' do
-      project = create(:empty_project)
-      project.import_url = "ssh://git@bitbucket.org/test/test.git"
-      project.create_or_update_import_data(credentials:
-                                             { user: "git",
-                                               password: nil,
-                                               bb_session: { bitbucket_access_token: "test",
-                                                             bitbucket_access_token_secret: "test" } })
-
-      expect { described_class.from_project(project) }.not_to raise_error
-    end
-  end
-end

spec/lib/gitlab/bitbucket_import/importer_spec.rb

@@ -18,15 +18,21 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
       "closed"  # undocumented status
     ]
   end
+
   let(:sample_issues_statuses) do
     issues = []
 
     statuses.map.with_index do |status, index|
       issues << {
-        local_id: index,
-        status: status,
+        id: index,
+        state: status,
         title: "Issue #{index}",
-        content: "Some content to issue #{index}"
+        kind: 'bug',
+        content: {
+            raw: "Some content to issue #{index}",
+            markup: "markdown",
+            html: "Some content to issue #{index}"
+        }
       }
     end
 
@@ -34,14 +40,16 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
   end
 
   let(:project_identifier) { 'namespace/repo' }
+
   let(:data) do
     {
       'bb_session' => {
-        'bitbucket_access_token' => "123456",
-        'bitbucket_access_token_secret' => "secret"
+        'bitbucket_token' => "123456",
+        'bitbucket_refresh_token' => "secret"
       }
     }
   end
+
   let(:project) do
     create(
       :project,
@@ -49,11 +57,13 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
       import_data: ProjectImportData.new(credentials: data)
     )
   end
+
   let(:importer) { Gitlab::BitbucketImport::Importer.new(project) }
+
   let(:issues_statuses_sample_data) do
     {
       count: sample_issues_statuses.count,
-      issues: sample_issues_statuses
+      values: sample_issues_statuses
     }
   end
 
@@ -61,26 +71,46 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
     before do
       stub_request(
         :get,
-        "https://bitbucket.org/api/1.0/repositories/#{project_identifier}"
-      ).to_return(status: 200, body: { has_issues: true }.to_json)
+        "https://api.bitbucket.org/2.0/repositories/#{project_identifier}"
+      ).to_return(status: 200,
+                  headers: { "Content-Type" => "application/json" },
+                  body: { has_issues: true, full_name: project_identifier }.to_json)
 
       stub_request(
         :get,
-        "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues?limit=50&sort=utc_created_on&start=0"
-      ).to_return(status: 200, body: issues_statuses_sample_data.to_json)
+        "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues?pagelen=50&sort=created_on"
+      ).to_return(status: 200,
+                  headers: { "Content-Type" => "application/json" },
+                  body: issues_statuses_sample_data.to_json)
+
+      stub_request(:get, "https://api.bitbucket.org/2.0/repositories/namespace/repo?pagelen=50&sort=created_on").
+         with(headers: { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization' => 'Bearer', 'User-Agent' => 'Faraday v0.9.2' }).
+         to_return(status: 200,
+                   body: "",
+                   headers: {})
 
       sample_issues_statuses.each_with_index do |issue, index|
         stub_request(
           :get,
-          "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues/#{issue[:local_id]}/comments"
+          "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues/#{issue[:id]}/comments?pagelen=50&sort=created_on"
         ).to_return(
           status: 200,
-          body: [{ author_info: { username: "username" }, utc_created_on: index }].to_json
+          headers: { "Content-Type" => "application/json" },
+          body: { author_info: { username: "username" }, utc_created_on: index }.to_json
         )
       end
+
+      stub_request(
+        :get,
+        "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/pullrequests?pagelen=50&sort=created_on&state=ALL"
+      ).to_return(status: 200,
+                  headers: { "Content-Type" => "application/json" },
+                  body: {}.to_json)
     end
 
     it 'map statuses to open or closed' do
+      # HACK: Bitbucket::Representation.const_get('Issue') seems to return ::Issue without this
+      Bitbucket::Representation::Issue.new({})
       importer.execute
 
       expect(project.issues.where(state: "closed").size).to eq(5)

spec/lib/gitlab/bitbucket_import/project_creator_spec.rb

@@ -2,14 +2,18 @@ require 'spec_helper'
 
 describe Gitlab::BitbucketImport::ProjectCreator, lib: true do
   let(:user) { create(:user) }
+
   let(:repo) do
-    {
-      name: 'Vim',
-      slug: 'vim',
-      is_private: true,
-      owner: "asd"
-    }.with_indifferent_access
+    double(name: 'Vim',
+           slug: 'vim',
+           description: 'Test repo',
+           is_private: true,
+           owner: "asd",
+           full_name: 'Vim repo',
+           visibility_level: Gitlab::VisibilityLevel::PRIVATE,
+           clone_url: 'ssh://git@bitbucket.org/asd/vim.git')
   end
+
   let(:namespace){ create(:group, owner: user) }
   let(:token) { "asdasd12345" }
   let(:secret) { "sekrettt" }
@@ -22,7 +26,7 @@ describe Gitlab::BitbucketImport::ProjectCreator, lib: true do
   it 'creates project' do
     allow_any_instance_of(Project).to receive(:add_import_job)
 
-    project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, user, access_params)
+    project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, 'vim', namespace, user, access_params)
     project = project_creator.execute
 
     expect(project.import_url).to eq("ssh://git@bitbucket.org/asd/vim.git")