Gogs Importer

app/assets/images/gogs-logo.svg

+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g fill-rule="evenodd" transform="translate(0 1)"><path d="m14 15.01h1v-8.02c0-3.862-3.134-6.991-7-6.991-3.858 0-7 3.13-7 6.991v8.02h1v-8.02c0-3.306 2.691-5.991 6-5.991 3.314 0 6 2.682 6 5.991v8.02m-10.52-13.354c-.366-.402-.894-.655-1.48-.655-1.105 0-2 .895-2 2 0 .868.552 1.606 1.325 1.883.102-.321.226-.631.371-.93-.403-.129-.695-.507-.695-.953 0-.552.448-1 1-1 .306 0 .58.138.764.354.222-.25.461-.483.717-.699m9.04-.002c.366-.401.893-.653 1.479-.653 1.105 0 2 .895 2 2 0 .867-.552 1.606-1.324 1.883-.101-.321-.225-.632-.37-.931.403-.129.694-.507.694-.952 0-.552-.448-1-1-1-.305 0-.579.137-.762.353-.222-.25-.461-.483-.717-.699"/><path d="m5.726 7.04h1.557v.124c0 .283-.033.534-.1.752-.065.202-.175.391-.33.566-.35.394-.795.591-1.335.591-.527 0-.979-.19-1.355-.571-.376-.382-.564-.841-.564-1.377 0-.547.191-1.01.574-1.391.382-.382.848-.574 1.396-.574.295 0 .57.06.825.181.244.12.484.316.72.586l-.405.388c-.309-.412-.686-.618-1.13-.618-.399 0-.733.138-1 .413-.27.27-.405.609-.405 1.015 0 .42.151.766.452 1.037.282.252.587.378.915.378.28 0 .531-.094.754-.283.223-.19.347-.418.373-.683h-.94v-.535m2.884.061c0-.53.194-.986.583-1.367.387-.381.853-.571 1.396-.571.537 0 .998.192 1.382.576.386.384.578.845.578 1.384 0 .542-.194 1-.581 1.379-.389.379-.858.569-1.408.569-.487 0-.923-.168-1.311-.505-.426-.373-.64-.861-.64-1.465m.574.007c0 .417.14.759.42 1.028.278.269.6.403.964.403.395 0 .729-.137 1-.41.272-.277.408-.613.408-1.01 0-.402-.134-.739-.403-1.01-.267-.273-.597-.41-.991-.41-.392 0-.723.137-.993.41-.27.27-.405.604-.405 1m-.184 3.918c.525.026.812.063.812.063.271.025.324-.096.116-.273 0 0-.775-.813-1.933-.813-1.159 0-1.923.813-1.923.813-.211.174-.153.3.12.273 0 0 .286-.037.81-.063v.477c0 .268.224.5.5.5.268 0 .5-.223.5-.498v-.252.25c0 .268.224.5.5.5.268 0 .5-.223.5-.498v-.478m-1-1.023c.552 0 1-.224 1-.5 0-.276-.448-.5-1-.5-.552 0-1 .224-1 .5 0 .276.448.5 1 .5"/></g></svg>
\ No newline at end of file

app/controllers/application_controller.rb

@@ -25,7 +25,7 @@ class ApplicationController < ActionController::Base
   protect_from_forgery with: :exception
 
   helper_method :can?, :current_application_settings
-  helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
+  helper_method :import_sources_enabled?, :github_import_enabled?, :gogs_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
 
   rescue_from Encoding::CompatibilityError do |exception|
     log_exception(exception)
@@ -245,6 +245,10 @@ class ApplicationController < ActionController::Base
     current_application_settings.import_sources.include?('github')
   end
 
+  def gogs_import_enabled?
+    current_application_settings.import_sources.include?('gogs')
+  end
+
   def github_import_configured?
     Gitlab::OAuth::Provider.enabled?(:github)
   end

app/controllers/import/gogs_controller.rb

+class Import::GogsController < Import::BaseController
+  before_action :verify_gogs_import_enabled
+  before_action :gogs_auth, only: [:status, :jobs, :create]
+
+  rescue_from Octokit::Unauthorized, with: :gogs_unauthorized
+
+  helper_method :logged_in_with_gogs?
+
+  def new
+    if session[:gogs_access_token]
+      redirect_to status_import_gogs_url
+    end
+  end
+
+  def personal_access_token
+    session[:gogs_access_token] = params[:personal_access_token]
+    session[:gogs_host_url] = params[:gogs_host_url]
+    redirect_to status_import_gogs_url
+  end
+
+  def status
+    @repos = client.repos
+    @already_added_projects = current_user.created_projects.where(import_type: "gogs")
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @gogs_root_url = session[:gogs_host_url]
+
+    @repos.reject!{ |repo| already_added_projects_names.include? repo.full_name }
+  end
+
+  def jobs
+    jobs = current_user.created_projects.where(import_type: "gogs").to_json(only: [:id, :import_status])
+    render json: jobs
+  end
+
+  def create
+    @repo_id = params[:repo_id].to_i
+    repo = client.repo(@repo_id)
+    @project_name = params[:new_name].presence || repo.name
+    namespace_path = params[:target_namespace].presence || current_user.namespace_path
+    @target_namespace = find_or_create_namespace(namespace_path, current_user.namespace_path)
+
+    if current_user.can?(:create_projects, @target_namespace)
+      @project = Gitlab::GithubImport::ProjectCreator.new(repo, @project_name, @target_namespace, current_user, access_params, type: 'gogs').execute
+    else
+      render 'unauthorized'
+    end
+  end
+
+  private
+
+  def client
+    @client ||= Gitlab::GithubImport::Client.new(session[:gogs_access_token], host: session[:gogs_host_url], api_version: 'v1')
+  end
+
+  def verify_gogs_import_enabled
+    render_404 unless gogs_import_enabled?
+  end
+
+  def gogs_auth
+    if session[:gogs_access_token].blank? || session[:gogs_host_url].blank?
+      redirect_to new_import_gogs_url,
+        alert: 'You need to specify both an Access Token and a Host URL.'
+    end
+  end
+
+  def gogs_unauthorized
+    session[:gogs_access_token] = nil
+    redirect_to new_import_gogs_url,
+      alert: 'Access denied to your Gogs account.'
+  end
+
+  def access_params
+    { github_access_token: session[:gogs_access_token] }
+  end
+end

app/helpers/import_helper.rb

@@ -8,6 +8,10 @@ module ImportHelper
     link_to path_with_namespace, github_project_url(path_with_namespace), target: '_blank'
   end
 
+  def gogs_project_link(path_with_namespace)
+    link_to path_with_namespace, gogs_project_url(path_with_namespace), target: '_blank'
+  end
+
   private
 
   def github_project_url(path_with_namespace)
@@ -20,4 +24,8 @@ module ImportHelper
     provider = Gitlab.config.omniauth.providers.find { |p| p.name == 'github' }
     @github_url = provider.fetch('url', 'https://github.com') if provider
   end
+
+  def gogs_project_url(path_with_namespace)
+    "#{@gogs_root_url}/#{path_with_namespace}"
+  end
 end

app/services/projects/import_service.rb

@@ -5,6 +5,7 @@ module Projects
     class Error < StandardError; end
 
     ALLOWED_TYPES = [
+      'gogs',
       'bitbucket',
       'fogbugz',
       'gitlab',

app/views/import/gogs/new.html.haml

+- page_title "Gogs Import"
+- header_title "Projects", root_path
+
+%h3.page-title
+  = image_tag(image_path('gogs-logo.svg'), alt: 'Gogs', size: "16x16")
+  Gogs
+
+%p
+  To import a Gogs project, you can use a
+  = succeed '.' do
+    = link_to 'Personal Access Token', 'https://github.com/gogits/go-gogs-client/wiki#access-token'
+
+= form_tag personal_access_token_import_gogs_path, method: :post, class: 'form-inline' do
+  .form-group
+    = text_field_tag :personal_access_token, '', class: 'form-control', placeholder: "Personal Access Token", size: 40
+    = text_field_tag :gogs_host_url, '', class: 'form-control', placeholder: "Gogs Host URL", size: 128
+    = submit_tag 'List Your Gogs Repositories', class: 'btn btn-success'

app/views/import/gogs/status.html.haml

+- page_title "Gogs import"
+- header_title "Projects", root_path
+%h3.page-title
+  %i.fa.fa-github
+  Import projects from Gogs
+
+%p.light
+  Select projects you want to import.
+%hr
+%p
+  = button_tag class: "btn btn-import btn-success js-import-all" do
+    Import all projects
+    = icon("spinner spin", class: "loading-icon")
+
+.table-responsive
+  %table.table.import-jobs
+    %colgroup.import-jobs-from-col
+    %colgroup.import-jobs-to-col
+    %colgroup.import-jobs-status-col
+    %thead
+      %tr
+        %th From Gogs
+        %th To GitLab
+        %th Status
+    %tbody
+      - @already_added_projects.each do |project|
+        %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
+          %td
+            = gogs_project_link(project.import_source)
+          %td
+            = link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
+          %td.job-status
+            - if project.import_status == 'finished'
+              %span
+                %i.fa.fa-check
+                done
+            - elsif project.import_status == 'started'
+              %i.fa.fa-spinner.fa-spin
+              started
+            - else
+              = project.human_import_status_name
+
+      - @repos.each do |repo|
+        %tr{id: "repo_#{repo.id}"}
+          %td
+            = gogs_project_link(repo.full_name)
+          %td.import-target
+            %fieldset.row
+            .input-group
+              .project-path.input-group-btn
+                - if current_user.can_select_namespace?
+                  - selected = params[:namespace_id] || :current_user
+                  - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner.login, path: repo.owner.login) } : {}
+                  = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 }
+                - else
+                  = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true
+              %span.input-group-addon /
+              = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
+          %td.import-actions.job-status
+            = button_tag class: "btn btn-import js-add-to-import" do
+              Import
+              = icon("spinner spin", class: "loading-icon")
+
+.js-importer-status{ data: { jobs_import_path: "#{jobs_import_gogs_path}", import_path: "#{import_gogs_path}" } }

app/views/projects/new.html.haml

@@ -49,6 +49,11 @@
                     = link_to new_import_github_path, class: 'btn import_github' do
                       = icon('github', text: 'GitHub')
                 %div
+                  - if gogs_import_enabled?
+                    = link_to new_import_gogs_url, class: 'btn import_gogs' do
+                      = image_tag(image_path('gogs-logo.svg'), alt: 'Gogs', size: "14x14")
+                      Gogs
+                %div
                   - if bitbucket_import_enabled?
                     = link_to status_import_bitbucket_path, class: "btn import_bitbucket #{'how_to_import_link' unless bitbucket_import_configured?}", "data-no-turbolink" => "true" do
                       = icon('bitbucket', text: 'Bitbucket')

changelogs/unreleased/22348-gogs-importer.yml

+---
+title: Gogs importer
+merge_request: 6945
+author: Kim Carlbäcker

config/initializers/1_settings.rb

@@ -213,7 +213,7 @@ Settings.gitlab.default_projects_features['builds']             = true if Settin
 Settings.gitlab.default_projects_features['container_registry'] = true if Settings.gitlab.default_projects_features['container_registry'].nil?
 Settings.gitlab.default_projects_features['visibility_level']   = Settings.send(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
 Settings.gitlab['domain_whitelist'] ||= []
-Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project]
+Settings.gitlab['import_sources'] ||= %w[gogs github bitbucket gitlab google_code fogbugz git gitlab_project]
 Settings.gitlab['trusted_proxies'] ||= []
 Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config', 'no_todos_messages.yml'))
 

config/routes/import.rb

@@ -6,6 +6,12 @@ namespace :import do
     get :jobs
   end
 
+  resource :gogs, only: [:create, :new], controller: :gogs do
+    post :personal_access_token
+    get :status
+    get :jobs
+  end
+
   resource :gitlab, only: [:create], controller: :gitlab do
     get :status
     get :callback

lib/gitlab/current_settings.rb

@@ -45,7 +45,7 @@ module Gitlab
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         domain_whitelist: Settings.gitlab['domain_whitelist'],
-        import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
+        import_sources: %w[gogs github bitbucket gitlab google_code fogbugz git gitlab_project],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,

lib/gitlab/github_import/client.rb

@@ -4,24 +4,30 @@ module Gitlab
       GITHUB_SAFE_REMAINING_REQUESTS = 100
       GITHUB_SAFE_SLEEP_TIME = 500
 
-      attr_reader :access_token
+      attr_reader :access_token, :host, :api_version
 
-      def initialize(access_token)
+      def initialize(access_token, host: nil, api_version: 'v3')
         @access_token = access_token
+        @host = host
+        @api_version = api_version
 
         if access_token
           ::Octokit.auto_paginate = false
         end
       end
 
+      def api_endpoint
+        host.present? && api_version.present? ? "#{host}/api/#{api_version}" : github_options[:site]
+      end
+
       def api
         @api ||= ::Octokit::Client.new(
           access_token: access_token,
-          api_endpoint: github_options[:site],
+          api_endpoint: api_endpoint,
           # If there is no config, we're connecting to github.com and we
           # should verify ssl.
           connection_options: {
-            ssl: { verify: config ? config['verify_ssl'] : true }
+            ssl: { verify: config ? config['verify_ssl'] : false }
           }
         )
       end

lib/gitlab/github_import/importer.rb

@@ -60,7 +60,7 @@ module Gitlab
         fetch_resources(:labels, repo, per_page: 100) do |labels|
           labels.each do |raw|
             begin
-              LabelFormatter.new(project, raw).create!
+              GithubImport::LabelFormatter.new(project, raw).create!
             rescue => e
               errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
             end
@@ -74,7 +74,7 @@ module Gitlab
         fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
           milestones.each do |raw|
             begin
-              MilestoneFormatter.new(project, raw).create!
+              GithubImport::MilestoneFormatter.new(project, raw).create!
             rescue => e
               errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
             end
@@ -85,7 +85,7 @@ module Gitlab
       def import_issues
         fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
           issues.each do |raw|
-            gh_issue = IssueFormatter.new(project, raw)
+            gh_issue = GithubImport::IssueFormatter.new(project, raw)
 
             begin
               issuable =
@@ -106,7 +106,7 @@ module Gitlab
       def import_pull_requests
         fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
           pull_requests.each do |raw|
-            pull_request = PullRequestFormatter.new(project, raw)
+            pull_request = GithubImport::PullRequestFormatter.new(project, raw)
             next unless pull_request.valid?
 
             begin
@@ -179,7 +179,7 @@ module Gitlab
         ActiveRecord::Base.no_touching do
           comments.each do |raw|
             begin
-              comment         = CommentFormatter.new(project, raw)
+              comment         = GithubImport::CommentFormatter.new(project, raw)
               # GH does not return info about comment's parent, so we guess it by checking its URL!
               *_, parent, iid = URI(raw.html_url).path.split('/')
               issuable_class = parent == 'issues' ? Issue : MergeRequest
@@ -198,7 +198,7 @@ module Gitlab
         last_note_attrs = nil
 
         cut_off_index = comments.find_index do |raw|
-          comment           = CommentFormatter.new(project, raw)
+          comment           = GithubImport::CommentFormatter.new(project, raw)
           comment_attrs     = comment.attributes
           last_note_attrs ||= last_note.slice(*comment_attrs.keys)
 
@@ -214,7 +214,7 @@ module Gitlab
 
       def import_wiki
         unless project.wiki.repository_exists?
-          wiki = WikiFormatter.new(project)
+          wiki = GithubImport::WikiFormatter.new(project)
           gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
         end
       rescue Gitlab::Shell::Error => e
@@ -230,7 +230,7 @@ module Gitlab
         fetch_resources(:releases, repo, per_page: 100) do |releases|
           releases.each do |raw|
             begin
-              gh_release = ReleaseFormatter.new(project, raw)
+              gh_release = GithubImport::ReleaseFormatter.new(project, raw)
               gh_release.create! if gh_release.valid?
             rescue => e
               errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }

lib/gitlab/github_import/issue_formatter.rb

@@ -70,7 +70,7 @@ module Gitlab
 
       def milestone
         if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
+          project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr))
         end
       end
 

lib/gitlab/github_import/milestone_formatter.rb

@@ -3,7 +3,7 @@ module Gitlab
     class MilestoneFormatter < BaseFormatter
       def attributes
         {
-          iid: raw_data.number,
+          iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr),
           project: project,
           title: raw_data.title,
           description: raw_data.description,
@@ -19,7 +19,11 @@ module Gitlab
       end
 
       def find_condition
-        { iid: raw_data.number }
+        { iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr) }
+      end
+
+      def self.iid_attr
+        :number
       end
 
       private

lib/gitlab/github_import/project_creator.rb

 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :name, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data, :type
 
-      def initialize(repo, name, namespace, current_user, session_data)
+      def initialize(repo, name, namespace, current_user, session_data, type: 'github')
         @repo = repo
         @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
+        @type = type
       end
 
       def execute
@@ -19,7 +20,7 @@ module Gitlab
           description: repo.description,
           namespace_id: namespace.id,
           visibility_level: visibility_level,
-          import_type: "github",
+          import_type: type,
           import_source: repo.full_name,
           import_url: import_url,
           skip_wiki: skip_wiki
@@ -29,7 +30,7 @@ module Gitlab
       private
 
       def import_url
-        repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
+        repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@")
       end
 
       def visibility_level

lib/gitlab/github_import/pull_request_formatter.rb

@@ -98,7 +98,7 @@ module Gitlab
 
       def milestone
         if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
+          project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr))
         end
       end
 

lib/gitlab/gogs_import/importer.rb

+require 'uri'
+
+module Gitlab
+  module GogsImport
+    class Importer < Gitlab::GithubImport::Importer
+      include Gitlab::ShellAdapter
+
+      attr_reader :client, :errors, :project, :repo, :repo_url
+
+      def initialize(project)
+        @project  = project
+        @repo     = project.import_source
+        @repo_url = project.import_url
+        @errors   = []
+        @labels   = {}
+
+        if credentials
+          uri = URI.parse(project.import_url)
+          host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(/[\w-]+\/[\w-]+\.git\z/, '')
+          @client = GithubImport::Client.new(credentials[:user], host: host, api_version: 'v1')
+        else
+          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
+        end
+      end
+
+      def execute
+        import_labels
+        import_milestones
+        import_pull_requests
+        import_issues
+        import_comments(:issues)
+        import_comments(:pull_requests)
+        import_wiki
+        # NOTE: this is commented out since Gogs doesn't have release-API yet
+        # import_releases
+        handle_errors
+
+        true
+      end
+
+      def import_milestones
+        fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
+          milestones.each do |raw|
+            begin
+              GogsImport::MilestoneFormatter.new(project, raw).create!
+            rescue => e
+              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/gogs_import/milestone_formatter.rb

+module Gitlab
+  module GogsImport
+    class MilestoneFormatter < GithubImport::MilestoneFormatter
+      def self.iid_attr
+        :id
+      end
+    end
+  end
+end

lib/gitlab/import_sources.rb

@@ -14,6 +14,7 @@ module Gitlab
 
       def options
         {
+          'Gogs'          => 'gogs',
           'GitHub'        => 'github',
           'Bitbucket'     => 'bitbucket',
           'GitLab.com'    => 'gitlab',