Merge branch 'google-code-import' into 'master'

app/assets/stylesheets/base/mixins.scss

@@ -119,6 +119,22 @@
   li {
     line-height: 1.5;
   }
+
+  a[href*="/uploads/"], a[href*="storage.googleapis.com/google-code-attachments/"] {
+    &:before {
+      margin-right: 4px;
+
+      font: normal normal normal 14px/1 FontAwesome;
+      font-size: inherit;
+      text-rendering: auto;
+      -webkit-font-smoothing: antialiased;
+      content: "\f0c6";
+    }
+
+    &:hover:before {
+      text-decoration: none;
+    }
+  }
 }
 
 @mixin str-truncated($max_width: 82%) {

app/assets/stylesheets/pages/notes.scss

@@ -62,20 +62,8 @@ ul.notes {
         word-wrap: break-word;
         @include md-typography;
 
-        a[href*="/uploads/"] {
-          &:before {
-            margin-right: 4px;
-
-            font: normal normal normal 14px/1 FontAwesome;
-            font-size: inherit;
-            text-rendering: auto;
-            -webkit-font-smoothing: antialiased;
-            content: "\f0c6";
-          }
-
-          &:hover:before {
-            text-decoration: none;
-          }
+        hr {
+          margin: 10px 0;
         }
       }
     }

app/controllers/import/google_code_controller.rb

+class Import::GoogleCodeController < Import::BaseController
+  before_filter :user_map, only: [:new_user_map, :create_user_map]
+
+  def new
+    
+  end
+
+  def callback
+    dump_file = params[:dump_file]
+
+    unless dump_file.respond_to?(:read)
+      return redirect_to :back, alert: "You need to upload a Google Takeout archive."
+    end
+
+    begin
+      dump = JSON.parse(dump_file.read)
+    rescue
+      return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout archive."
+    end
+
+    client = Gitlab::GoogleCodeImport::Client.new(dump)
+    unless client.valid?
+      return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout archive."
+    end
+
+    session[:google_code_dump] = dump
+
+    if params[:create_user_map] == "1"
+      redirect_to new_user_map_import_google_code_path
+    else
+      redirect_to status_import_google_code_path
+    end
+  end
+
+  def new_user_map
+
+  end
+
+  def create_user_map
+    user_map_json = params[:user_map]
+    user_map_json = "{}" if user_map_json.blank?
+
+    begin
+      user_map = JSON.parse(user_map_json)
+    rescue
+      flash.now[:alert] = "The entered user map is not a valid JSON user map."
+
+      render "new_user_map" and return
+    end
+
+    unless user_map.is_a?(Hash) && user_map.all? { |k, v| k.is_a?(String) && v.is_a?(String) }
+      flash.now[:alert] = "The entered user map is not a valid JSON user map."
+
+      render "new_user_map" and return
+    end
+
+    session[:google_code_user_map] = user_map
+
+    flash[:notice] = "The user map has been saved. Continue by selecting the projects you want to import."
+
+    redirect_to status_import_google_code_path
+  end
+
+  def status
+    unless client.valid?
+      return redirect_to new_import_google_path 
+    end
+
+    @repos = client.repos
+
+    @already_added_projects = current_user.created_projects.where(import_type: "google_code")
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @repos.reject! { |repo| already_added_projects_names.include? repo.name }
+  end
+
+  def jobs
+    jobs = current_user.created_projects.where(import_type: "google_code").to_json(only: [:id, :import_status])
+    render json: jobs
+  end
+
+  def create
+    @repo_id = params[:repo_id]
+    repo = client.repo(@repo_id)
+    @target_namespace = current_user.namespace
+    @project_name = repo.name
+
+    namespace = @target_namespace
+
+    user_map = session[:google_code_user_map]
+
+    @project = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, current_user, user_map).execute
+  end
+
+  private
+
+  def client
+    @client ||= Gitlab::GoogleCodeImport::Client.new(session[:google_code_dump])
+  end
+
+  def user_map
+    @user_map ||= begin
+      user_map = client.user_map
+
+      stored_user_map = session[:google_code_user_map]
+      user_map.update(stored_user_map) if stored_user_map
+
+      Hash[user_map.sort]
+    end
+  end
+end

app/models/project.rb

@@ -27,6 +27,7 @@
 #  import_type            :string(255)
 #  import_source          :string(255)
 #  avatar                 :string(255)
+#  import_data            :text
 #
 
 require 'carrierwave/orm/activerecord'
@@ -50,6 +51,8 @@ class Project < ActiveRecord::Base
   default_value_for :wall_enabled, false
   default_value_for :snippets_enabled, gitlab_config_features.snippets
 
+  serialize :import_data, JSON
+
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
   def set_last_activity_at
@@ -185,6 +188,7 @@ class Project < ActiveRecord::Base
     state :failed
 
     after_transition any => :started, do: :add_import_job
+    after_transition any => :finished, do: :clear_import_data
   end
 
   class << self
@@ -262,6 +266,11 @@ class Project < ActiveRecord::Base
     RepositoryImportWorker.perform_in(2.seconds, id)
   end
 
+  def clear_import_data
+    self.import_data = nil
+    self.save
+  end
+
   def import?
     import_url.present?
   end

app/views/import/google_code/new.html.haml

+%h3.page-title
+  %i.fa.fa-google
+  Import projects from Google Code
+%hr
+
+= form_tag callback_import_google_code_path, class: 'form-horizontal', multipart: true do
+  %p
+    Follow the steps below to export your Google Code project data. 
+    In the next step, you'll be able to select the projects you want to import.
+  %ol
+    %li
+      %p
+        Go to
+        #{link_to "Google Takeout", "https://www.google.com/settings/takeout", target: "_blank"}.
+    %li
+      %p
+        Make sure you're logged into the account that owns the projects you'd like to import.
+    %li
+      %p
+        Click the <strong>Select none</strong> button on the right, since we only need "Google Code Project Hosting".
+    %li
+      %p
+        Scroll down to <strong>Google Code Project Hosting</strong> and enable the switch on the right.
+    %li
+      %p
+        Choose <strong>Next</strong> at the bottom of the page.
+    %li
+      %p
+        Leave the "File type" and "Delivery method" options on their default values.
+    %li
+      %p
+        Choose <strong>Create archive</strong> and wait for archiving to complete.
+    %li
+      %p
+        Click the <strong>Download</strong> button and wait for downloading to complete.
+    %li
+      %p
+        Find the downloaded ZIP file and decompress it.
+    %li
+      %p
+        Find the newly extracted <code>Takeout/Google Code Project Hosting/GoogleCodeProjectHosting.json</code> file.
+    %li
+      %p
+        Upload <code>GoogleCodeProjectHosting.json</code> here:
+      %p
+        %input{type: "file", name: "dump_file", id: "dump_file"}
+    %li
+      %p
+        Do you want to customize how Google Code email addresses and usernames are imported into GitLab?
+      %p
+        = label_tag :create_user_map_0 do
+          = radio_button_tag :create_user_map, 0, true
+          No, directly import the existing email addresses and usernames.
+      %p
+        = label_tag :create_user_map_1 do
+          = radio_button_tag :create_user_map, 1, false
+          Yes, let me map Google Code users to full names or GitLab users.
+    %li
+      %p
+        = submit_tag 'Continue to the next step', class: "btn btn-create"

app/views/import/google_code/new_user_map.html.haml

+%h3.page-title
+  %i.fa.fa-google
+  Import projects from Google Code
+%hr
+
+= form_tag create_user_map_import_google_code_path, class: 'form-horizontal' do
+  %p
+    Customize how Google Code email addresses and usernames are imported into GitLab.
+    In the next step, you'll be able to select the projects you want to import.
+  %p 
+    The user map is a JSON document mapping Google Code users (as keys) to the way they will be imported into GitLab (as values). By default the username is masked to ensure users' privacy.
+  %p
+    To map a Google Code user to a full name or GitLab user, simply replace the value, e.g. <code>"johnsmith@gmail.com": "John Smith"</code> or <code>"johnsmith@gmail.com": "@johnsmith"</code>. Be sure to preserve the surrounding double quotes and other punctuation.
+
+  .form-group
+    .col-sm-12
+      = text_area_tag :user_map, JSON.pretty_generate(@user_map), class: 'form-control', rows: 15
+
+  .form-actions
+    = submit_tag 'Continue to the next step', class: "btn btn-create"

app/views/import/google_code/status.html.haml

+%h3.page-title
+  %i.fa.fa-google
+  Import projects from Google Code
+
+%p.light
+  Select projects you want to import.
+%p.light
+  Optionally, you can
+  = link_to "customize", new_user_map_import_google_code_path
+  how Google Code email addresses and usernames are imported into GitLab.
+%hr
+%p
+  = button_tag 'Import all projects', class: "btn btn-success js-import-all"
+
+%table.table.import-jobs
+  %thead
+    %tr
+      %th From Google Code
+      %th To GitLab
+      %th Status
+  %tbody
+    - @already_added_projects.each do |project|
+      %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
+        %td
+          = link_to project.import_source, "https://code.google.com/p/#{project.import_source}", target: "_blank"
+        %td
+          %strong= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
+        %td.job-status
+          - if project.import_status == 'finished'
+            %span
+              %i.fa.fa-check
+              done
+          - elsif project.import_status == 'started'
+            %i.fa.fa-spinner.fa-spin
+            started
+          - else
+            = project.human_import_status_name
+
+    - @repos.each do |repo|
+      %tr{id: "repo_#{repo.id}"}
+        %td
+          = link_to repo.name, "https://code.google.com/p/#{repo.name}", target: "_blank"
+        %td.import-target
+          = "#{current_user.username}/#{repo.name}"
+        %td.import-actions.job-status
+          = button_tag "Import", class: "btn js-add-to-import"
+
+:coffeescript
+  new ImporterStatus("#{jobs_import_google_code_path}", "#{import_google_code_path}")

app/views/projects/new.html.haml

@@ -62,6 +62,10 @@
               %i.icon-gitorious.icon-gitorious-small
               Gitorious.org
 
+            = link_to new_import_google_code_path, class: 'btn' do
+              %i.fa.fa-google
+              Google Code
+
             = link_to "#", class: 'btn js-toggle-button' do
               %i.fa.fa-git
               %span Any repo by URL

app/workers/repository_import_worker.rb

@@ -18,6 +18,8 @@ class RepositoryImportWorker
                             Gitlab::GitlabImport::Importer.new(project).execute
                           elsif project.import_type == 'bitbucket'
                             Gitlab::BitbucketImport::Importer.new(project).execute
+                          elsif project.import_type == 'google_code'
+                            Gitlab::GoogleCodeImport::Importer.new(project).execute
                           else
                             true
                           end

config/routes.rb

@@ -81,6 +81,15 @@ Gitlab::Application.routes.draw do
       get :callback
       get :jobs
     end
+
+    resource :google_code, only: [:create, :new], controller: :google_code do
+      get :status
+      post :callback
+      get :jobs
+
+      get   :new_user_map,    path: :user_map
+      post  :create_user_map, path: :user_map
+    end
   end
 
   #

db/migrate/20150327150017_add_import_data_to_project.rb

+class AddImportDataToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_data, :text
+  end
+end

db/schema.rb

@@ -343,6 +343,7 @@ ActiveRecord::Schema.define(version: 20150413192223) do
     t.integer  "star_count",             default: 0,        null: false
     t.string   "import_type"
     t.string   "import_source"
+    t.text     "import_data"
   end
 
   add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree

lib/gitlab/google_code_import/client.rb

+module Gitlab
+  module GoogleCodeImport
+    class Client
+      attr_reader :raw_data
+
+      def self.mask_email(author)
+        parts = author.split("@", 2)
+        parts[0] = "#{parts[0][0...-3]}..."
+        parts.join("@")
+      end
+
+      def initialize(raw_data)
+        @raw_data = raw_data
+      end
+
+      def valid?
+        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects")
+      end
+
+      def repos
+        @repos ||= raw_data["projects"].map { |raw_repo| GoogleCodeImport::Repository.new(raw_repo) }.select(&:git?)
+      end
+
+      def repo(id)
+        repos.find { |repo| repo.id == id }
+      end
+
+      def user_map
+        user_map = Hash.new { |hash, user| hash[user] = self.class.mask_email(user) }
+
+        repos.each do |repo|
+          next unless repo.valid? && repo.issues
+
+          repo.issues.each do |raw_issue|
+            # Touching is enough to add the entry and masked email.
+            user_map[raw_issue["author"]["name"]]
+
+            raw_issue["comments"]["items"].each do |raw_comment|
+              user_map[raw_comment["author"]["name"]]
+            end
+          end
+        end
+
+        Hash[user_map.sort]
+      end
+    end
+  end
+end

lib/gitlab/google_code_import/project_creator.rb

+module Gitlab
+  module GoogleCodeImport
+    class ProjectCreator
+      attr_reader :repo, :namespace, :current_user, :user_map
+
+      def initialize(repo, namespace, current_user, user_map = nil)
+        @repo = repo
+        @namespace = namespace
+        @current_user = current_user
+        @user_map = user_map
+      end
+
+      def execute
+        import_data = {
+          "repo"      => repo.raw_data,
+          "user_map"  => user_map
+        }
+
+        @project = Project.new(
+          name: repo.name,
+          path: repo.name,
+          description: repo.summary,
+          namespace: namespace,
+          creator: current_user,
+          visibility_level: Gitlab::VisibilityLevel::PUBLIC,
+          import_type: "google_code",
+          import_source: repo.name,
+          import_url: repo.import_url,
+          import_data: import_data
+        )
+
+        if @project.save!
+          @project.reload
+
+          if @project.import_failed?
+            @project.import_retry
+          else
+            @project.import_start
+          end
+        end
+
+        @project
+      end
+    end
+  end
+end

lib/gitlab/google_code_import/repository.rb

+module Gitlab
+  module GoogleCodeImport
+    class Repository
+      attr_accessor :raw_data
+
+      def initialize(raw_data)
+        @raw_data = raw_data
+      end
+
+      def valid?
+        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#project"
+      end
+
+      def id
+        raw_data["externalId"]
+      end
+
+      def name
+        raw_data["name"]
+      end
+
+      def summary
+        raw_data["summary"]
+      end
+
+      def description
+        raw_data["description"]
+      end
+
+      def git?
+        raw_data["versionControlSystem"] == "git"
+      end
+
+      def import_url
+        raw_data["repositoryUrls"].first
+      end
+
+      def issues
+        raw_data["issues"] && raw_data["issues"]["items"]
+      end
+    end
+  end
+end

spec/controllers/import/google_code_controller_spec.rb

+require 'spec_helper'
+
+describe Import::GoogleCodeController do
+  let(:user) { create(:user) }
+  let(:dump_file) { fixture_file_upload(Rails.root + 'spec/fixtures/GoogleCodeProjectHosting.json', 'application/json') }
+
+  before do
+    sign_in(user)
+  end
+
+  describe "POST callback" do
+    it "stores Google Takeout dump list in session" do
+      post :callback, dump_file: dump_file
+
+      expect(session[:google_code_dump]).to be_a(Hash)
+      expect(session[:google_code_dump]["kind"]).to eq("projecthosting#user")
+      expect(session[:google_code_dump]).to have_key("projects")
+    end
+  end
+
+  describe "GET status" do
+    before do
+      @repo = OpenStruct.new(name: 'vim')
+      controller.stub_chain(:client, :valid?).and_return(true)
+    end
+
+    it "assigns variables" do
+      @project = create(:project, import_type: 'google_code', creator_id: user.id)
+      controller.stub_chain(:client, :repos).and_return([@repo])
+
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([@repo])
+    end
+
+    it "does not show already added project" do
+      @project = create(:project, import_type: 'google_code', creator_id: user.id, import_source: 'vim')
+      controller.stub_chain(:client, :repos).and_return([@repo])
+
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([])
+    end
+  end
+end

spec/lib/gitlab/gitorious_import/project_creator.rb → spec/lib/gitlab/gitorious_import/project_creator_spec.rb

@@ -9,8 +9,7 @@ describe Gitlab::GitoriousImport::ProjectCreator do
     allow_any_instance_of(Project).to receive(:add_import_job)
 
     project_creator = Gitlab::GitoriousImport::ProjectCreator.new(repo, namespace, user)
-    project_creator.execute
-    project = Project.last
+    project = project_creator.execute
 
     expect(project.name).to eq("Bar Baz Qux")
     expect(project.path).to eq("bar-baz-qux")

spec/lib/gitlab/google_code_import/client_spec.rb

+require "spec_helper"
+
+describe Gitlab::GoogleCodeImport::Client do
+  let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) }
+  subject { described_class.new(raw_data) }
+
+  describe "#valid?" do
+    context "when the data is valid" do
+      it "returns true" do
+        expect(subject).to be_valid
+      end
+    end
+
+    context "when the data is invalid" do
+      let(:raw_data) { "No clue" }
+
+      it "returns true" do
+        expect(subject).to_not be_valid
+      end
+    end
+  end
+
+  describe "#repos" do
+    it "returns only Git repositories" do
+      expect(subject.repos.length).to eq(1)
+    end
+  end
+
+  describe "#repo" do
+    it "returns the referenced repository" do
+      expect(subject.repo("tint2").name).to eq("tint2")
+    end
+  end
+end

spec/lib/gitlab/google_code_import/importer_spec.rb

+require "spec_helper"
+
+describe Gitlab::GoogleCodeImport::Importer do
+  let(:mapped_user) { create(:user, username: "thilo123") }
+  let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) }
+  let(:client) { Gitlab::GoogleCodeImport::Client.new(raw_data) }
+  let(:import_data) { 
+    {
+      "repo"      => client.repo("tint2").raw_data,
+      "user_map"  => {
+        "thilo..." => "@#{mapped_user.username}"
+      }
+    } 
+  }
+  let(:project) { create(:project, import_data: import_data) }
+  subject       { described_class.new(project) }
+
+  describe "#execute" do
+
+    it "imports status labels" do
+      subject.execute
+
+      %w(New NeedInfo Accepted Wishlist Started Fixed Invalid Duplicate WontFix Incomplete).each do |status|
+        expect(project.labels.find_by(name: "Status: #{status}")).to_not be_nil
+      end
+    end
+
+    it "imports labels" do
+      subject.execute
+
+      %w(
+        Type-Defect Type-Enhancement Type-Task Type-Review Type-Other Milestone-0.12 Priority-Critical 
+        Priority-High Priority-Medium Priority-Low OpSys-All OpSys-Windows OpSys-Linux OpSys-OSX Security 
+        Performance Usability Maintainability Component-Panel Component-Taskbar Component-Battery 
+        Component-Systray Component-Clock Component-Launcher Component-Tint2conf Component-Docs Component-New
+      ).each do |label|
+        label.sub!("-", ": ")
+        expect(project.labels.find_by(name: label)).to_not be_nil
+      end
+    end
+
+    it "imports issues" do
+      subject.execute
+
+      issue = project.issues.first
+      expect(issue).to_not be_nil
+      expect(issue.iid).to eq(169)
+      expect(issue.author).to eq(project.creator)
+      expect(issue.assignee).to eq(mapped_user)
+      expect(issue.state).to eq("closed")
+      expect(issue.label_names).to include("Priority: Medium")
+      expect(issue.label_names).to include("Status: Fixed")
+      expect(issue.label_names).to include("Type: Enhancement")
+      expect(issue.title).to eq("Scrolling through tasks")
+      expect(issue.state).to eq("closed")
+      expect(issue.description).to include("schattenpr...")
+      expect(issue.description).to include("November 18, 2009 00:20")
+      expect(issue.description).to include('I like to scroll through the tasks with my scrollwheel \(like in fluxbox\).')
+      expect(issue.description).to include('Patch is attached that adds two new mouse\-actions \(next\_taskprev\_task\)')
+      expect(issue.description).to include('that can be used for exactly that purpose.')
+      expect(issue.description).to include('all the best!')
+      expect(issue.description).to include('[tint2_task_scrolling.diff](https://storage.googleapis.com/google-code-attachments/tint2/issue-169/comment-0/tint2_task_scrolling.diff)')
+      expect(issue.description).to include('![screenshot.png](https://storage.googleapis.com/google-code-attachments/tint2/issue-169/comment-0/screenshot.png)')
+    end
+
+    it "imports issue comments" do
+      subject.execute
+
+      note = project.issues.first.notes.first
+      expect(note).to_not be_nil
+      expect(note.note).to include("Comment 1")
+      expect(note.note).to include("@#{mapped_user.username}")
+      expect(note.note).to include("November 18, 2009 05:14")
+      expect(note.note).to include("applied, thanks.")
+      expect(note.note).to include("Status: Fixed")
+      expect(note.note).to include("~~Type: Defect~~")
+      expect(note.note).to include("Type: Enhancement")
+    end
+  end
+end

spec/lib/gitlab/google_code_import/project_creator_spec.rb

+require 'spec_helper'
+
+describe Gitlab::GoogleCodeImport::ProjectCreator do
+  let(:user) { create(:user) }
+  let(:repo) { 
+    Gitlab::GoogleCodeImport::Repository.new(
+      "name"            => 'vim',
+      "summary"         => 'VI Improved',
+      "repositoryUrls"  => [ "https://vim.googlecode.com/git/" ]
+    )
+  }
+  let(:namespace) { create(:namespace) }
+
+  it 'creates project' do
+    allow_any_instance_of(Project).to receive(:add_import_job)
+
+    project_creator = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, user)
+    project = project_creator.execute
+    
+    expect(project.import_url).to eq("https://vim.googlecode.com/git/")
+    expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PUBLIC)
+  end
+end