BigW Consortium Gitlab

importer.rb 5.9 KB
Newer Older
Valery Sizov committed
1
module Gitlab
2
  module GithubImport
Valery Sizov committed
3
    class Importer
4 5
      include Gitlab::ShellAdapter

6
      attr_reader :client, :project, :repo, :repo_url
Valery Sizov committed
7 8

      def initialize(project)
9 10
        @project  = project
        @repo     = project.import_source
11 12 13 14
        @repo_url = project.import_url

        if credentials
          @client = Client.new(credentials[:user])
James Lopez committed
15 16 17 18
          @formatter = Gitlab::ImportFormatter.new
        else
          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
        end
Valery Sizov committed
19 20 21
      end

      def execute
22 23
        import_labels && import_milestones && import_issues &&
          import_pull_requests && import_wiki
24 25 26 27
      end

      private

28 29
      def credentials
        @credentials ||= project.import_data.credentials if project.import_data
James Lopez committed
30 31
      end

32
      def import_labels
33
        labels = client.labels(repo, per_page: 100)
34
        labels.each { |raw| LabelFormatter.new(project, raw).create! }
35 36 37 38 39 40

        true
      rescue ActiveRecord::RecordInvalid => e
        raise Projects::ImportService::Error, e.message
      end

41
      def import_milestones
42
        milestones = client.milestones(repo, state: :all, per_page: 100)
43
        milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
44 45 46 47 48 49

        true
      rescue ActiveRecord::RecordInvalid => e
        raise Projects::ImportService::Error, e.message
      end

50
      def import_issues
51
        issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
52

53
        issues.each do |raw|
54
          gh_issue = IssueFormatter.new(project, raw)
55

56 57 58 59
          if gh_issue.valid?
            issue = gh_issue.create!
            apply_labels(issue)
            import_comments(issue) if gh_issue.has_comments?
Valery Sizov committed
60 61
          end
        end
62 63

        true
64 65
      rescue ActiveRecord::RecordInvalid => e
        raise Projects::ImportService::Error, e.message
Valery Sizov committed
66 67
      end

68
      def import_pull_requests
69
        disable_webhooks
70

71
        pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
72
        pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
73

74 75
        source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
        target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
76 77
        branches_removed = source_branches_removed | target_branches_removed

78
        restore_branches(branches_removed)
79 80

        pull_requests.each do |pull_request|
81 82 83 84
          merge_request = pull_request.create!
          apply_labels(merge_request)
          import_comments(merge_request)
          import_comments_on_diff(merge_request)
85
        end
86 87

        true
88 89
      rescue ActiveRecord::RecordInvalid => e
        raise Projects::ImportService::Error, e.message
90
      ensure
91
        clean_up_restored_branches(branches_removed)
92
        clean_up_disabled_webhooks
93 94
      end

95
      def disable_webhooks
96 97 98
        update_webhooks(hooks, active: false)
      end

99
      def clean_up_disabled_webhooks
100 101 102 103 104 105 106
        update_webhooks(hooks, active: true)
      end

      def update_webhooks(hooks, options)
        hooks.each do |hook|
          client.edit_hook(repo, hook.id, hook.name, hook.config, options)
        end
107 108 109 110 111 112 113 114 115 116 117 118 119 120
      end

      def hooks
        @hooks ||=
          begin
            client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)

          # The GitHub Repository Webhooks API returns 404 for users
          # without admin access to the repository when listing hooks.
          # In this case we just want to return gracefully instead of
          # spitting out an error and stop the import process.
          rescue Octokit::NotFound
            []
          end
121 122
      end

123
      def restore_branches(branches)
124 125
        branches.each do |name, sha|
          client.create_ref(repo, "refs/heads/#{name}", sha)
126
        end
127 128

        project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*')
129 130
      end

131
      def clean_up_restored_branches(branches)
132 133
        branches.each do |name, _|
          client.delete_ref(repo, "heads/#{name}")
134
          project.repository.delete_branch(name) rescue Rugged::ReferenceError
135
        end
136 137

        project.repository.after_remove_branch
138 139
      end

140 141
      def apply_labels(issuable)
        issue = client.issue(repo, issuable.iid)
142 143 144 145 146 147 148 149 150 151

        if issue.labels.count > 0
          label_ids = issue.labels.map do |raw|
            Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id)
          end

          issuable.update_attribute(:label_ids, label_ids)
        end
      end

152
      def import_comments(issuable)
153
        comments = client.issue_comments(repo, issuable.iid, per_page: 100)
154
        create_comments(issuable, comments)
155 156
      end

157
      def import_comments_on_diff(merge_request)
158
        comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100)
159
        create_comments(merge_request, comments)
160
      end
Valery Sizov committed
161

162 163 164 165
      def create_comments(issuable, comments)
        comments.each do |raw|
          comment = CommentFormatter.new(project, raw)
          issuable.notes.create!(comment.attributes)
166
        end
Valery Sizov committed
167
      end
168 169 170 171

      def import_wiki
        unless project.wiki_enabled?
          wiki = WikiFormatter.new(project)
172
          gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
173 174
          project.update_attribute(:wiki_enabled, true)
        end
175 176

        true
177
      rescue Gitlab::Shell::Error => e
178 179 180 181 182
        # GitHub error message when the wiki repo has not been created,
        # this means that repo has wiki enabled, but have no pages. So,
        # we can skip the import.
        if e.message !~ /repository not exported/
          raise Projects::ImportService::Error, e.message
183
        else
184
          true
185
        end
186
      end
Valery Sizov committed
187 188 189
    end
  end
end