Merge branch 'web_hooks' of https://github.com/ariejan/gitlabhq into ariejan-web_hooks

Gemfile

@@ -23,6 +23,8 @@ gem "rdiscount"
 gem "acts-as-taggable-on", "~> 2.1.0"
 gem "drapper"
 gem "rchardet19", "~> 1.3.5"
+gem "resque"
+gem "httparty"
 
 group :assets do
   gem "sass-rails",   "~> 3.1.0"
@@ -47,6 +49,7 @@ group :development, :test do
   gem "awesome_print"
   gem "database_cleaner"
   gem "launchy"
+  gem "webmock"
 end
 
 group :test do

Gemfile.lock

@@ -87,6 +87,7 @@ GEM
       execjs
     coffee-script-source (1.1.3)
     columnize (0.3.4)
+    crack (0.3.1)
     daemons (1.1.4)
     database_cleaner (0.7.0)
     devise (1.5.0)
@@ -111,6 +112,9 @@ GEM
       railties (~> 3.0)
     hashery (1.4.0)
     hike (1.2.1)
+    httparty (0.8.1)
+      multi_json
+      multi_xml
     i18n (0.6.0)
     jquery-rails (1.0.17)
       railties (~> 3.0)
@@ -132,6 +136,7 @@ GEM
       treetop (~> 1.4.8)
     mime-types (1.17.2)
     multi_json (1.0.3)
+    multi_xml (0.4.1)
     nokogiri (1.5.0)
     orm_adapter (0.0.5)
     polyglot (0.3.3)
@@ -143,6 +148,8 @@ GEM
       rack (>= 0.4)
     rack-mount (0.8.3)
       rack (>= 1.0.0)
+    rack-protection (1.1.4)
+      rack
     rack-ssl (1.3.2)
       rack
     rack-test (0.6.1)
@@ -169,6 +176,14 @@ GEM
     rdiscount (1.6.8)
     rdoc (3.11)
       json (~> 1.4)
+    redis (2.2.2)
+    redis-namespace (1.0.3)
+      redis (< 3.0.0)
+    resque (1.19.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.0.2)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
     rspec (2.7.0)
       rspec-core (~> 2.7.0)
       rspec-expectations (~> 2.7.0)
@@ -220,6 +235,10 @@ GEM
       multi_json (~> 1.0.3)
       simplecov-html (~> 0.5.3)
     simplecov-html (0.5.3)
+    sinatra (1.3.1)
+      rack (~> 1.3, >= 1.3.4)
+      rack-protection (~> 1.1, >= 1.1.2)
+      tilt (~> 1.3, >= 1.3.3)
     six (0.2.0)
     sprockets (2.0.3)
       hike (~> 1.2)
@@ -244,8 +263,13 @@ GEM
     uglifier (1.1.0)
       execjs (>= 0.3.0)
       multi_json (>= 1.0.2)
+    vegas (0.1.8)
+      rack (>= 1.0.0)
     warden (1.1.0)
       rack (>= 1.0)
+    webmock (1.7.8)
+      addressable (~> 2.2, > 2.2.5)
+      crack (>= 0.1.7)
     xpath (0.1.4)
       nokogiri (~> 1.3)
 
@@ -270,6 +294,7 @@ DEPENDENCIES
   gitolite!
   grit!
   haml-rails
+  httparty
   jquery-rails
   kaminari
   launchy
@@ -279,6 +304,7 @@ DEPENDENCIES
   rails-footnotes (~> 3.7.5)
   rchardet19 (~> 1.3.5)
   rdiscount
+  resque
   rspec-rails
   ruby-debug19
   sass-rails (~> 3.1.0)
@@ -292,3 +318,4 @@ DEPENDENCIES
   thin
   turn
   uglifier
+  webmock

app/models/project.rb

@@ -14,6 +14,7 @@ class Project < ActiveRecord::Base
   has_many :users, :through => :users_projects
   has_many :notes, :dependent => :destroy
   has_many :snippets, :dependent => :destroy
+  has_many :web_hooks, :dependent => :destroy
 
   acts_as_taggable
 
@@ -82,12 +83,58 @@ class Project < ActiveRecord::Base
     :heads,
     :commits_since,
     :fresh_commits,
+    :commits_between,
     :to => :repository, :prefix => nil
 
   def to_param
     code
   end
 
+  def web_url
+    [GIT_HOST['host'], code].join("/")
+  end
+
+  def execute_web_hooks(oldrev, newrev, ref)
+    ref_parts = ref.split('/')
+
+    # Return if this is not a push to a branch (e.g. new commits)
+    return if ref_parts[1] !~ /heads/ || oldrev == "00000000000000000000000000000000"
+
+    data = web_hook_data(oldrev, newrev, ref)
+    web_hooks.each { |web_hook| web_hook.execute(data) }
+  end
+
+  def web_hook_data(oldrev, newrev, ref)
+    data = {
+      before: oldrev,
+      after: newrev,
+      ref: ref,
+      repository: {
+        name: name,
+        url: web_url,
+        description: description,
+        homepage: web_url,
+        private: private?
+      },
+      commits: []
+    }
+
+    commits_between(oldrev, newrev).each do |commit|
+      data[:commits] << {
+        id: commit.id,
+        message: commit.safe_message,
+        timestamp: commit.date.xmlschema,
+        url: "http://#{GIT_HOST['host']}/#{code}/commits/#{commit.id}",
+        author: {
+          name: commit.author_name,
+          email: commit.author_email
+        }
+      }
+    end
+
+    data
+  end
+
   def team_member_by_name_or_email(email = nil, name = nil)
     user = users.where("email like ? or name like ?", email, name).first
     users_projects.find_by_user_id(user.id) if user

app/models/repository.rb

@@ -31,6 +31,22 @@ class Repository
     project.id
   end
 
+  def write_hooks
+    %w(post-receive).each do |hook|
+      write_hook(hook, File.read(File.join(Rails.root, 'lib', "#{hook}-hook")))
+    end
+  end
+
+  def write_hook(name, content)
+    hook_file = File.join(project.path_to_repo, 'hooks', name)
+
+    File.open(hook_file, 'w') do |f|
+      f.write(content)
+    end
+
+    File.chmod(0775, hook_file)
+  end
+
   def repo
     @repo ||= Grit::Repo.new(project.path_to_repo)
   end
@@ -47,6 +63,8 @@ class Repository
     Gitlabhq::GitHost.system.new.configure do |c|
       c.update_project(path, project)
     end
+
+    write_hooks
   end
 
   def destroy_repository
@@ -115,4 +133,8 @@ class Repository
       repo.commits(ref)
     end.map{ |c| Commit.new(c) } 
   end
+
+  def commits_between(from, to)
+    repo.commits_between(from, to).map { |c| Commit.new(c) }
+  end
 end

app/models/web_hook.rb

+class WebHook < ActiveRecord::Base
+  include HTTParty
+
+  # HTTParty timeout
+  default_timeout 10
+
+  belongs_to :project
+
+  validates :url,
+            presence: true,
+            format: {
+              with: URI::regexp(%w(http https)),
+              message: "should be a valid url" }
+
+  def execute(data)
+    WebHook.post(url, body: data.to_json)
+  rescue
+    # There was a problem calling this web hook, let's forget about it.
+  end
+end

app/workers/post_receive.rb

+class PostReceive
+  def self.perform(reponame, oldrev, newrev, ref)
+    project = Project.find_by_path(reponame)
+    return false if project.nil?
+
+    project.execute_web_hooks(oldrev, newrev, ref)
+  end
+end

config/routes.rb

 Gitlab::Application.routes.draw do
 
+  # Optionally, enable Resque here
+  # require 'resque/server'
+  # mount Resque::Server.new, at: '/info/resque'
 
   get 'tags'=> 'tags#index'
   get 'tags/:tag' => 'projects#index'

db/migrate/20111214091851_create_web_hooks.rb

+class CreateWebHooks < ActiveRecord::Migration
+  def change
+    create_table :web_hooks do |t|
+      t.string :url
+      t.integer :project_id
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -145,4 +145,18 @@ ActiveRecord::Schema.define(:version => 20111220190817) do
     t.integer  "project_access", :default => 0, :null => false
   end
 
+  create_table "web_hook_urls", :force => true do |t|
+    t.string   "url"
+    t.integer  "project_id"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "web_hooks", :force => true do |t|
+    t.string   "url"
+    t.integer  "project_id"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
 end

lib/gitlabhq/gitolite.rb

@@ -43,14 +43,14 @@ module Gitlabhq
 
     def destroy_project(project)
       FileUtils.rm_rf(project.path_to_repo)
-      
+
       ga_repo = ::Gitolite::GitoliteAdmin.new(File.join(@local_dir,'gitolite'))
       conf = ga_repo.config
       conf.rm_repo(project.path)
       ga_repo.save
     end
 
-     #update or create
+    #update or create
     def update_keys(user, key)
       File.open(File.join(@local_dir, 'gitolite/keydir',"#{user}.pub"), 'w') {|f| f.write(key.gsub(/\n/,'')) }
     end

lib/post-receive-hook

+#!/bin/bash
+
+# This file was placed here by Gitlab. It makes sure that your pushed commits
+# will be processed properly.
+
+while read oldrev newrev ref
+do
+  # For every branch or tag that was pushed, create a Resque job in redis.
+  pwd=`pwd`
+  reponame=`basename "$pwd" | cut -d. -f1`
+  env -i redis-cli rpush "resque:queue:post-receive" "{\"class\":\"PostReceive\",\"args\":[\"$reponame\",\"$oldrev\",\"$newrev\",\"$ref\"]}" > /dev/null 2>&1
+done

lib/tasks/resque.rake

+require 'resque/tasks'

spec/factories.rb

@@ -55,3 +55,7 @@ Factory.add(:key, Key) do |obj|
   obj.title = "Example key"
   obj.key = File.read(File.join(Rails.root, "db", "pkey.example"))
 end
+
+Factory.add(:web_hook, WebHook) do |obj|
+  obj.url = Faker::Internet.url
+end

spec/models/project_spec.rb

@@ -7,6 +7,7 @@ describe Project do
     it { should have_many(:issues) }
     it { should have_many(:notes) }
     it { should have_many(:snippets) }
+    it { should have_many(:web_hooks).dependent(:destroy) }
   end
 
   describe "Validation" do
@@ -33,6 +34,7 @@ describe Project do
     it { should respond_to(:repo) }
     it { should respond_to(:tags) }
     it { should respond_to(:commit) }
+    it { should respond_to(:commits_between) }
   end
 
   it "should not allow 'gitolite-admin' as repo name" do
@@ -50,6 +52,11 @@ describe Project do
     project.path_to_repo.should == File.join(Rails.root, "tmp", "tests", "somewhere")
   end
 
+  it "returns the full web URL for this repo" do
+    project = Project.new(:code => "somewhere")
+    project.web_url.should == "#{GIT_HOST['host']}/somewhere"
+  end
+
   describe :valid_repo? do
     it "should be valid repo" do
       project = Factory :project
@@ -62,6 +69,106 @@ describe Project do
     end
   end
 
+  describe "web hooks" do
+    let(:project) { Factory :project }
+
+    context "with no web hooks" do
+      it "raises no errors" do
+        lambda {
+          project.execute_web_hooks('oldrev', 'newrev', 'ref')
+        }.should_not raise_error
+      end
+    end
+
+    context "with web hooks" do
+      before do
+        @webhook = Factory(:web_hook)
+        @webhook_2 = Factory(:web_hook)
+        project.web_hooks << [@webhook, @webhook_2]
+      end
+
+      it "executes multiple web hook" do
+        @webhook.should_receive(:execute).once
+        @webhook_2.should_receive(:execute).once
+
+        project.execute_web_hooks('oldrev', 'newrev', 'refs/heads/master')
+      end
+    end
+
+    context "does not execute web hooks" do
+      before do
+        @webhook = Factory(:web_hook)
+        project.web_hooks << [@webhook]
+      end
+
+      it "when pushing a branch for the first time" do
+        @webhook.should_not_receive(:execute)
+        project.execute_web_hooks('00000000000000000000000000000000', 'newrev', 'refs/heads/master')
+      end
+
+      it "when pushing tags" do
+        @webhook.should_not_receive(:execute)
+        project.execute_web_hooks('oldrev', 'newrev', 'refs/tags/v1.0.0')
+      end
+    end
+
+    context "when pushing new branches" do
+
+    end
+
+    context "when gathering commit data" do
+      before do
+        @oldrev, @newrev, @ref = project.fresh_commits(2).last.sha, project.fresh_commits(2).first.sha, 'refs/heads/master'
+        @commit = project.fresh_commits(2).first
+
+        # Fill nil/empty attributes
+        project.description = "This is a description"
+
+        @data = project.web_hook_data(@oldrev, @newrev, @ref)
+      end
+
+      subject { @data }
+
+      it { should include(before: @oldrev) }
+      it { should include(after: @newrev) }
+      it { should include(ref: @ref) }
+
+      context "with repository data" do
+        subject { @data[:repository] }
+
+        it { should include(name: project.name) }
+        it { should include(url: project.web_url) }
+        it { should include(description: project.description) }
+        it { should include(homepage: project.web_url) }
+        it { should include(private: project.private?) }
+      end
+
+      context "with commits" do
+        subject { @data[:commits] }
+
+        it { should be_an(Array) }
+        it { should have(1).element }
+
+        context "the commit" do
+          subject { @data[:commits].first }
+
+          it { should include(id: @commit.id) }
+          it { should include(message: @commit.safe_message) }
+          it { should include(timestamp: @commit.date.xmlschema) }
+          it { should include(url: "http://localhost/#{project.code}/commits/#{@commit.id}") }
+
+          context "with a author" do
+            subject { @data[:commits].first[:author] }
+
+            it { should include(name: @commit.author_name) }
+            it { should include(email: @commit.author_email) }
+          end
+        end
+      end
+
+    end
+  end
+
   describe "updates" do
     let(:project) { Factory :project }
 
@@ -107,6 +214,21 @@ describe Project do
     it { project.fresh_commits.last.id.should == "0dac878dbfe0b9c6104a87d65fe999149a8d862c" }
   end
 
+  describe "commits_between" do
+    let(:project) { Factory :project }
+
+    subject do
+      commits = project.commits_between("a6d1d4aca0c85816ddfd27d93773f43a31395033",
+                                        "2fb376f61875b58bceee0492e270e9c805294b1a")
+      commits.map { |c| c.id }
+    end
+
+    it { should have(2).elements }
+    it { should include("2fb376f61875b58bceee0492e270e9c805294b1a") }
+    it { should include("4571e226fbcd7be1af16e9fa1e13b7ac003bebdf") }
+    it { should_not include("a6d1d4aca0c85816ddfd27d93773f43a31395033") }
+  end
+
   describe "Git methods" do
     let(:project) { Factory :project }
 

spec/models/web_hook_spec.rb

+require 'spec_helper'
+
+describe WebHook do
+  describe "Associations" do
+    it { should belong_to :project }
+  end
+
+  describe "Validations" do
+    it { should validate_presence_of(:url) }
+
+    context "url format" do
+      it { should allow_value("http://example.com").for(:url) }
+      it { should allow_value("https://excample.com").for(:url) }
+      it { should allow_value("http://test.com/api").for(:url) }
+      it { should allow_value("http://test.com/api?key=abc").for(:url) }
+      it { should allow_value("http://test.com/api?key=abc&type=def").for(:url) }
+
+      it { should_not allow_value("example.com").for(:url) }
+      it { should_not allow_value("ftp://example.com").for(:url) }
+      it { should_not allow_value("herp-and-derp").for(:url) }
+    end
+  end
+
+  describe "execute" do
+    before(:each) do
+      @webhook = Factory :web_hook
+      @project = Factory :project
+      @project.web_hooks << [@webhook]
+      @data = { before: 'oldrev', after: 'newrev', ref: 'ref'}
+
+      WebMock.stub_request(:post, @webhook.url)
+    end
+
+    it "POSTs to the web hook URL" do
+      @webhook.execute(@data)
+      WebMock.should have_requested(:post, @webhook.url).once
+    end
+
+    it "POSTs the data as JSON" do
+      json = @data.to_json
+
+      @webhook.execute(@data)
+      WebMock.should have_requested(:post, @webhook.url).with(body: json).once
+    end
+
+    it "catches exceptions" do
+      WebHook.should_receive(:post).and_raise("Some HTTP Post error")
+
+      lambda {
+        @webhook.execute(@data)
+      }.should_not raise_error
+    end
+  end
+end

spec/requests/projects_tree_perfomance_spec.rb

@@ -9,7 +9,6 @@ describe "Projects" do
       before do
         @project = Factory :project
         @project.add_access(@user, :read)
-
       end
 
       it "should be fast" do

spec/spec_helper.rb

@@ -8,6 +8,7 @@ require 'rspec/rails'
 require 'capybara/rails'
 require 'capybara/rspec'
 require 'capybara/dsl'
+require 'webmock/rspec'
 require 'factories'
 require 'monkeypatch'
 
@@ -48,6 +49,8 @@ RSpec.configure do |config|
     end
 
     DatabaseCleaner.start
+
+    WebMock.disable_net_connect!(allow_localhost: true)
   end
 
   config.after do

spec/workers/post_receive_spec.rb

+require 'spec_helper'
+
+describe PostReceive do
+
+  context "as a resque worker" do
+    it "reponds to #perform" do
+      PostReceive.should respond_to(:perform)
+    end
+  end
+
+  context "web hooks" do
+    let(:project) { Factory :project }
+
+    it "it retrieves the correct project" do
+      Project.should_receive(:find_by_path).with(project.path)
+      PostReceive.perform(project.path, 'sha-old', 'sha-new', 'refs/heads/master')
+    end
+
+    it "asks the project to execute web hooks" do
+      Project.stub(find_by_path: project)
+      project.should_receive(:execute_web_hooks).with('sha-old', 'sha-new', 'refs/heads/master')
+
+      PostReceive.perform(project.path, 'sha-old', 'sha-new', 'refs/heads/master')
+    end
+  end
+end