BigW Consortium Gitlab

Skip to content

G
gitlab-ce
Commit 263abda3 by Kirilll Zaitsev
Options

Drone CI service

parent 308c6428
Showing with 544 additions and 128 deletions
CHANGELOG
......@@ -31,6 +31,9 @@ v 8.0.0 (unreleased)
- Don't notify users without access to the project when they are (accidentally) mentioned in a note.
- Retrieving oauth token with LDAP credentials
- Load Application settings from running database unless env var USE_DB=false
- Added Drone CI integration (Kirill Zaitsev)
- Refactored service API and added automatically service docs generator (Kirill Zaitsev)
- Added web_url key project hook_attrs (Kirill Zaitsev)
v 7.14.1
- Improve abuse reports management from admin area
......
app/controllers/projects/services_controller.rb
......@@ -2,7 +2,7 @@ class Projects::ServicesController < Projects::ApplicationController
ALLOWED_PARAMS = [:title, :token, :type, :active, :api_key, :api_version, :subdomain,
:room, :recipients, :project_url, :webhook,
:user_key, :device, :priority, :sound, :bamboo_url, :username, :password,
:build_key, :server, :teamcity_url, :build_type,
:build_key, :server, :teamcity_url, :drone_url, :build_type,
:description, :issues_url, :new_issue_url, :restrict_to_branch, :channel,
:colorize_messages, :channels,
:push_events, :issues_events, :merge_requests_events, :tag_push_events,
......
app/models/project.rb
......@@ -73,6 +73,7 @@ class Project < ActiveRecord::Base
has_many :services
has_one :gitlab_ci_service, dependent: :destroy
has_one :campfire_service, dependent: :destroy
has_one :drone_ci_service, dependent: :destroy
has_one :emails_on_push_service, dependent: :destroy
has_one :irker_service, dependent: :destroy
has_one :pivotaltracker_service, dependent: :destroy
......@@ -613,6 +614,7 @@ class Project < ActiveRecord::Base
name: name,
ssh_url: ssh_url_to_repo,
http_url: http_url_to_repo,
web_url: web_url,
namespace: namespace.name,
visibility_level: visibility_level
}
......
app/models/project_services/ci_service.rb
......@@ -25,12 +25,24 @@ class CiService < Service
def category
:ci
end
def valid_token?(token)
self.respond_to?(:token) && self.token.present? && self.token == token
end
def supported_events
%w(push)
end
# Return complete url to build page
def merge_request_page(iid, sha, ref)
commit_page(sha, ref)
end
def commit_page(sha, ref)
build_page(sha, ref)
end
# Return complete url to merge_request page
#
# Ex.
# http://jenkins.example.com:8888/job/test1/scm/bySHA1/12d65c
......@@ -45,10 +57,27 @@ class CiService < Service
#
#
# Ex.
# @service.commit_status('13be4ac')
# @service.merge_request_status(9, '13be4ac', 'dev')
# # => 'success'
#
# @service.merge_request_status(10, '2abe4ac', 'dev)
# # => 'running'
#
#
def merge_request_status(iid, sha, ref)
commit_status(sha, ref)
end
# Return string with build status or :error symbol
#
# Allowed states: 'success', 'failed', 'running', 'pending', 'skipped'
#
#
# Ex.
# @service.commit_status('13be4ac', 'master')
# # => 'success'
#
# @service.commit_status('2abe4ac')
# @service.commit_status('2abe4ac', 'dev')
# # => 'running'
#
#
......
app/models/project_services/drone_ci_service.rb 0 → 100644
class DroneCiService < CiService
prop_accessor :drone_url, :token, :enable_ssl_verification
validates :drone_url,
presence: true,
format: { with: /\A#{URI.regexp(%w(http https))}\z/, message: "should be a valid url" }, if: :activated?
validates :token,
presence: true,
format: { with: /\A([A-Za-z0-9]+)\z/ }, if: :activated?
after_save :compose_service_hook, if: :activated?
def compose_service_hook
hook = service_hook || build_service_hook
hook.url = [drone_url, "/api/hook", "?owner=#{project.namespace.path}", "&name=#{project.path}", "&access_token=#{token}"].join
hook.enable_ssl_verification = enable_ssl_verification
hook.save
end
def execute(data)
case data[:object_kind]
when 'push'
service_hook.execute(data) if push_valid?(data)
when 'merge_request'
service_hook.execute(data) if merge_request_valid?(data)
when 'tag_push'
service_hook.execute(data) if tag_push_valid?(data)
end
end
def allow_target_ci?
true
end
def supported_events
%w(push merge_request tag_push)
end
def merge_request_status_path(iid, sha = nil, ref = nil)
url = [drone_url,
"gitlab/#{project.namespace.path}/#{project.path}/pulls/#{iid}",
"?access_token=#{token}"]
URI.join(*url).to_s
end
def commit_status_path(sha, ref)
url = [drone_url,
"gitlab/#{project.namespace.path}/#{project.path}/commits/#{sha}",
"?branch=#{URI::encode(ref.to_s)}&access_token=#{token}"]
URI.join(*url).to_s
end
def merge_request_status(iid, sha, ref)
response = HTTParty.get(merge_request_status_path(iid), verify: enable_ssl_verification)
if response.code == 200 and response['status']
case response['status']
when 'killed'
:canceled
when 'failure', 'error'
# Because drone return error if some test env failed
:failed
else
response["status"]
end
else
:error
end
rescue Errno::ECONNREFUSED
:error
end
def commit_status(sha, ref)
response = HTTParty.get(commit_status_path(sha, ref), verify: enable_ssl_verification)
if response.code == 200 and response['status']
case response['status']
when 'killed'
:canceled
when 'failure', 'error'
# Because drone return error if some test env failed
:failed
else
response["status"]
end
else
:error
end
rescue Errno::ECONNREFUSED
:error
end
def merge_request_page(iid, sha, ref)
url = [drone_url,
"gitlab/#{project.namespace.path}/#{project.path}/redirect/pulls/#{iid}"]
URI.join(*url).to_s
end
def commit_page(sha, ref)
url = [drone_url,
"gitlab/#{project.namespace.path}/#{project.path}/redirect/commits/#{sha}",
"?branch=#{URI::encode(ref.to_s)}"]
URI.join(*url).to_s
end
def commit_coverage(sha, ref)
nil
end
def build_page(sha, ref)
commit_page(sha, ref)
end
def builds_path
url = [drone_url, "#{project.namespace.path}/#{project.path}"]
URI.join(*url).to_s
end
def status_img_path
url = [drone_url,
"api/badges/#{project.namespace.path}/#{project.path}/status.svg",
"?branch=#{URI::encode(project.default_branch)}"]
URI.join(*url).to_s
end
def title
'Drone CI'
end
def description
'Drone is a Continuous Integration platform built on Docker, written in Go'
end
def to_param
'drone_ci'
end
def fields
[
{ type: 'text', name: 'token', placeholder: 'Drone CI project specific token' },
{ type: 'text', name: 'drone_url', placeholder: 'http://drone.example.com' },
{ type: 'checkbox', name: 'enable_ssl_verification', title: "Enable SSL verification" }
]
end
private
def tag_push_valid?(data)
data[:total_commits_count] > 0 && !Gitlab::Git.blank_ref?(data[:after])
end
def push_valid?(data)
opened_merge_requests = project.merge_requests.opened.where(source_project_id: project.id,
source_branch: Gitlab::Git.ref_name(data[:ref]))
opened_merge_requests.empty? && data[:total_commits_count] > 0 &&
!Gitlab::Git.blank_ref?(data[:after])
end
def merge_request_valid?(data)
['opened', 'reopened'].include?(data[:object_attributes][:state]) &&
data[:object_attributes][:merge_status] == 'unchecked'
end
end
app/models/service.rb
......@@ -135,6 +135,7 @@ class Service < ActiveRecord::Base
buildkite
campfire
custom_issue_tracker
drone_ci
emails_on_push
external_wiki
flowdock
......
doc/web_hooks/web_hooks.md
......@@ -279,6 +279,7 @@ X-Gitlab-Event: Note Hook
"name": "Gitlab Test",
"ssh_url": "git@example.com:gitlab-org/gitlab-test.git",
"http_url": "http://example.com/gitlab-org/gitlab-test.git",
"web_url": "http://example.com/gitlab-org/gitlab-test",
"namespace": "Gitlab Org",
"visibility_level": 10
},
......@@ -286,6 +287,7 @@ X-Gitlab-Event: Note Hook
"name": "Gitlab Test",
"ssh_url": "git@example.com:gitlab-org/gitlab-test.git",
"http_url": "http://example.com/gitlab-org/gitlab-test.git",
"web_url": "http://example.com/gitlab-org/gitlab-test",
"namespace": "Gitlab Org",
"visibility_level": 10
},
......@@ -462,6 +464,7 @@ X-Gitlab-Event: Merge Request Hook
"name": "awesome_project",
"ssh_url": "ssh://git@example.com/awesome_space/awesome_project.git",
"http_url": "http://example.com/awesome_space/awesome_project.git",
"web_url": "http://example.com/awesome_space/awesome_project",
"visibility_level": 20,
"namespace": "awesome_space"
},
......@@ -469,6 +472,7 @@ X-Gitlab-Event: Merge Request Hook
"name": "awesome_project",
"ssh_url": "ssh://git@example.com/awesome_space/awesome_project.git",
"http_url": "http://example.com/awesome_space/awesome_project.git",
"web_url": "http://example.com/awesome_space/awesome_project",
"visibility_level": 20,
"namespace": "awesome_space"
},
......
lib/api/helpers.rb
......@@ -55,6 +55,32 @@ module API
end
end
def project_service
@project_service ||= begin
underscored_service = params[:service_slug].underscore
if Service.available_services_names.include?(underscored_service)
user_project.build_missing_services
service_method = "#{underscored_service}_service"
send_service(service_method)
end
end
@project_service || not_found!("Service")
end
def send_service(service_method)
user_project.send(service_method)
end
def service_attributes
@service_attributes ||= project_service.fields.inject([]) do |arr, hash|
arr << hash[:name].to_sym
end
end
def find_group(id)
begin
group = Group.find(id)
......
lib/api/services.rb
......@@ -4,73 +4,49 @@ module API
before { authenticate! }
before { authorize_admin_project }
resource :projects do
# Set GitLab CI service for project
#
# Parameters:
# token (required) - CI project token
# project_url (required) - CI project url
# Set <service_slug> service for project
#
# Example Request:
#
# PUT /projects/:id/services/gitlab-ci
put ":id/services/gitlab-ci" do
required_attributes! [:token, :project_url]
attrs = attributes_for_keys [:token, :project_url]
user_project.build_missing_services
if user_project.gitlab_ci_service.update_attributes(attrs.merge(active: true))
true
else
not_found!
end
end
# Delete GitLab CI service settings
#
# Example Request:
# DELETE /projects/:id/services/gitlab-ci
delete ":id/services/gitlab-ci" do
if user_project.gitlab_ci_service
user_project.gitlab_ci_service.update_attributes(
active: false,
token: nil,
project_url: nil
)
end
end
put ':id/services/:service_slug' do
if project_service
validators = project_service.class.validators.select do |s|
s.class == ActiveRecord::Validations::PresenceValidator &&
s.attributes != [:project_id]
end
# Set Hipchat service for project
#
# Parameters:
# token (required) - Hipchat token
# room (required) - Hipchat room name
#
# Example Request:
# PUT /projects/:id/services/hipchat
put ':id/services/hipchat' do
required_attributes! [:token, :room]
attrs = attributes_for_keys [:token, :room]
user_project.build_missing_services
required_attributes! validators.map(&:attributes).flatten.uniq
attrs = attributes_for_keys service_attributes
if user_project.hipchat_service.update_attributes(
attrs.merge(active: true))
true
else
not_found!
if project_service.update_attributes(attrs.merge(active: true))
true
else
not_found!
end
end
end
# Delete Hipchat service settings
# Delete <service_slug> service for project
#
# Example Request:
# DELETE /projects/:id/services/hipchat
delete ':id/services/hipchat' do
if user_project.hipchat_service
user_project.hipchat_service.update_attributes(
active: false,
token: nil,
room: nil
)
#
# DELETE /project/:id/services/gitlab-ci
#
delete ':id/services/:service_slug' do
if project_service
attrs = service_attributes.inject({}) do |hash, key|
hash.merge!(key => nil)
end
if project_service.update_attributes(attrs.merge(active: false))
true
else
not_found!
end
end
end
end
......
lib/gitlab/backend/grack_auth.rb
......@@ -10,7 +10,7 @@ module Grack
@request = Rack::Request.new(env)
@auth = Request.new(env)
@gitlab_ci = false
@ci = false
# Need this patch due to the rails mount
# Need this if under RELATIVE_URL_ROOT
......@@ -28,7 +28,7 @@ module Grack
if project && authorized_request?
# Tell gitlab-git-http-server the request is OK, and what the GL_ID is
render_grack_auth_ok
elsif @user.nil? && !@gitlab_ci
elsif @user.nil? && !@ci
unauthorized
else
render_not_found
......@@ -47,8 +47,8 @@ module Grack
# Allow authentication for GitLab CI service
# if valid token passed
if gitlab_ci_request?(login, password)
@gitlab_ci = true
if ci_request?(login, password)
@ci = true
return
end
......@@ -60,12 +60,17 @@ module Grack
end
end
def gitlab_ci_request?(login, password)
if login == "gitlab-ci-token" && project && project.gitlab_ci?
token = project.gitlab_ci_service.token
def ci_request?(login, password)
matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login)
if token.present? && token == password && git_cmd == 'git-upload-pack'
return true
if project && matched_login.present? && git_cmd == 'git-upload-pack'
underscored_service = matched_login['s'].underscore
if Service.available_services_names.include?(underscored_service)
service_method = "#{underscored_service}_service"
service = project.send(service_method)
return service && service.activated? && service.valid_token?(password)
end
end
......@@ -124,7 +129,7 @@ module Grack
end
def authorized_request?
return true if @gitlab_ci
return true if @ci
case git_cmd
when *Gitlab::GitAccess::DOWNLOAD_COMMANDS
......
lib/tasks/services.rake 0 → 100644
services_template = <<-ERB
# Services
<% services.each do |service| %>
## <%= service[:title] %>
<% unless service[:description].blank? %>
<%= service[:description] %>
<% end %>
### Create/Edit <%= service[:title] %> service
Set <%= service[:title] %> service for a project.
<% unless service[:help].blank? %>
> <%= service[:help].gsub("\n", ' ') %>
<% end %>
```
PUT /projects/:id/services/<%= service[:dashed_name] %>
```
Parameters:
<% service[:params].each do |param| %>
- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)" %><%= [" -", param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %>
<% end %>
### Delete <%= service[:title] %> service
Delete <%= service[:title] %> service for a project.
```
DELETE /projects/:id/services/<%= service[:dashed_name] %>
```
<% end %>
ERB
namespace :services do
task :doc do
services = Service.available_services_names.map do |s|
service_start = Time.now
klass = "#{s}_service".classify.constantize
service = klass.new
service_hash = {}
service_hash[:title] = service.title
service_hash[:dashed_name] = s.dasherize
service_hash[:description] = service.description
service_hash[:help] = service.help
service_hash[:params] = service.fields.map do |p|
param_hash = {}
param_hash[:name] = p[:name]
param_hash[:description] = p[:placeholder] || p[:title]
param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v|
v.class == ActiveRecord::Validations::PresenceValidator
end
param_hash
end.sort_by { |p| p[:required] ? 0 : 1 }
puts "Collected data for: #{service.title}, #{Time.now-service_start}"
service_hash
end
doc_start = Time.now
doc_path = File.join(Rails.root, 'doc', 'api', 'services.md')
result = ERB.new(services_template, 0 , '>')
.result(OpenStruct.new(services: services).instance_eval { binding })
File.open(doc_path, 'w') do |f|
f.write result
end
puts "write a new service.md to: #{doc_path.to_s}, #{Time.now-doc_start}"
end
end
spec/models/project_services/drone_ci_service_spec.rb 0 → 100644
# == Schema Information
#
# Table name: services
#
# id :integer not null, primary key
# type :string(255)
# title :string(255)
# project_id :integer
# created_at :datetime
# updated_at :datetime
# active :boolean default(FALSE), not null
# properties :text
# template :boolean default(FALSE)
# push_events :boolean default(TRUE)
# issues_events :boolean default(TRUE)
# merge_requests_events :boolean default(TRUE)
# tag_push_events :boolean default(TRUE)
# note_events :boolean default(TRUE), not null
#
require 'spec_helper'
describe DroneCiService do
describe 'associations' do
it { is_expected.to belong_to(:project) }
it { is_expected.to have_one(:service_hook) }
end
describe 'validations' do
context 'active' do
before { allow(subject).to receive(:activated?).and_return(true) }
it { is_expected.to validate_presence_of(:token) }
it { is_expected.to validate_presence_of(:drone_url) }
it { is_expected.to allow_value('ewf9843kdnfdfs89234n').for(:token) }
it { is_expected.to allow_value('http://ci.example.com').for(:drone_url) }
it { is_expected.not_to allow_value('token with spaces').for(:token) }
it { is_expected.not_to allow_value('token/with%spaces').for(:token) }
it { is_expected.not_to allow_value('this is not url').for(:drone_url) }
it { is_expected.not_to allow_value('http//noturl').for(:drone_url) }
it { is_expected.not_to allow_value('ftp://ci.example.com').for(:drone_url) }
end
context 'inactive' do
before { allow(subject).to receive(:activated?).and_return(false) }
it { is_expected.not_to validate_presence_of(:token) }
it { is_expected.not_to validate_presence_of(:drone_url) }
it { is_expected.to allow_value('ewf9843kdnfdfs89234n').for(:token) }
it { is_expected.to allow_value('http://drone.example.com').for(:drone_url) }
it { is_expected.to allow_value('token with spaces').for(:token) }
it { is_expected.to allow_value('ftp://drone.example.com').for(:drone_url) }
end
end
shared_context :drone_ci_service do
let(:drone) { DroneCiService.new }
let(:project) { create(:project, name: 'project') }
let(:path) { "#{project.namespace.path}/#{project.path}" }
let(:drone_url) { 'http://drone.example.com' }
let(:sha) { '2ab7834c' }
let(:branch) { 'dev' }
let(:token) { 'secret' }
let(:iid) { rand(1..9999) }
before(:each) do
allow(drone).to receive_messages(
project_id: project.id,
project: project,
active: true,
drone_url: drone_url,
token: token
)
end
end
describe "service page/path methods" do
include_context :drone_ci_service
# URL's
let(:commit_page) { "#{drone_url}/gitlab/#{path}/redirect/commits/#{sha}?branch=#{branch}" }
let(:merge_request_page) { "#{drone_url}/gitlab/#{path}/redirect/pulls/#{iid}" }
let(:commit_status_path) { "#{drone_url}/gitlab/#{path}/commits/#{sha}?branch=#{branch}&access_token=#{token}" }
let(:merge_request_status_path) { "#{drone_url}/gitlab/#{path}/pulls/#{iid}?access_token=#{token}" }
it { expect(drone.build_page(sha, branch)).to eq(commit_page) }
it { expect(drone.commit_page(sha, branch)).to eq(commit_page) }
it { expect(drone.merge_request_page(iid, sha, branch)).to eq(merge_request_page) }
it { expect(drone.commit_status_path(sha, branch)).to eq(commit_status_path) }
it { expect(drone.merge_request_status_path(iid, sha, branch)).to eq(merge_request_status_path) }
end
describe "execute" do
include_context :drone_ci_service
let(:user) { create(:user, username: 'username') }
let(:push_sample_data) { Gitlab::PushDataBuilder.build_sample(project, user) }
it do
service_hook = double
expect(service_hook).to receive(:execute)
expect(drone).to receive(:service_hook).and_return(service_hook)
drone.execute(push_sample_data)
end
end
end
spec/requests/api/services_spec.rb
......@@ -5,64 +5,47 @@ describe API::API, api: true do
let(:user) { create(:user) }
let(:project) {create(:project, creator_id: user.id, namespace: user.namespace) }
describe "POST /projects/:id/services/gitlab-ci" do
it "should update gitlab-ci settings" do
put api("/projects/#{project.id}/services/gitlab-ci", user), token: 'secrettoken', project_url: "http://ci.example.com/projects/1"
expect(response.status).to eq(200)
end
it "should return if required fields missing" do
put api("/projects/#{project.id}/services/gitlab-ci", user), project_url: "http://ci.example.com/projects/1", active: true
expect(response.status).to eq(400)
end
it "should return if the format of token is invalid" do
put api("/projects/#{project.id}/services/gitlab-ci", user), token: 'token-with dashes and spaces%', project_url: "http://ci.example.com/projects/1", active: true
expect(response.status).to eq(404)
end
it "should return if the format of token is invalid" do
put api("/projects/#{project.id}/services/gitlab-ci", user), token: 'token-with dashes and spaces%', project_url: "ftp://ci.example/projects/1", active: true
expect(response.status).to eq(404)
end
end
describe "DELETE /projects/:id/services/gitlab-ci" do
it "should update gitlab-ci settings" do
delete api("/projects/#{project.id}/services/gitlab-ci", user)
expect(response.status).to eq(200)
expect(project.gitlab_ci_service).to be_nil
end
end
describe 'PUT /projects/:id/services/hipchat' do
it 'should update hipchat settings' do
put api("/projects/#{project.id}/services/hipchat", user),
token: 'secret-token', room: 'test'
expect(response.status).to eq(200)
expect(project.hipchat_service).not_to be_nil
end
it 'should return if required fields missing' do
put api("/projects/#{project.id}/services/gitlab-ci", user),
token: 'secret-token', active: true
expect(response.status).to eq(400)
end
end
describe 'DELETE /projects/:id/services/hipchat' do
it 'should delete hipchat settings' do
delete api("/projects/#{project.id}/services/hipchat", user)
expect(response.status).to eq(200)
expect(project.hipchat_service).to be_nil
Service.available_services_names.each do |service|
describe "PUT /projects/:id/services/#{service.dasherize}" do
include_context service
it "should update #{service} settings" do
put api("/projects/#{project.id}/services/#{dashed_service}", user), service_attrs
expect(response.status).to eq(200)
end
it "should return if required fields missing" do
attrs = service_attrs
required_attributes = service_attrs_list.select do |attr|
service_klass.validators_on(attr).any? do |v|
v.class == ActiveRecord::Validations::PresenceValidator
end
end
if required_attributes.empty?
expected_code = 200
else
attrs.delete(required_attributes.shuffle.first)
expected_code = 400
end
put api("/projects/#{project.id}/services/#{dashed_service}", user), attrs
expect(response.status).to eq(expected_code)
end
end
describe "DELETE /projects/:id/services/#{service.dasherize}" do
include_context service
it "should delete #{service}" do
delete api("/projects/#{project.id}/services/#{dashed_service}", user)
expect(response.status).to eq(200)
expect(project.send(service_method).activated?).to be_falsey
end
end
end
end
spec/support/services_shared_context.rb 0 → 100644
Service.available_services_names.each do |service|
shared_context service do
let(:dashed_service) { service.dasherize }
let(:service_method) { "#{service}_service".to_sym }
let(:service_klass) { "#{service}_service".classify.constantize }
let(:service_attrs_list) { service_klass.new.fields.inject([]) {|arr, hash| arr << hash[:name].to_sym } }
let(:service_attrs) do
service_attrs_list.inject({}) do |hash, k|
if k =~ /^(token*|.*_token|.*_key)/
hash.merge!(k => 'secrettoken')
elsif k =~ /^(.*_url|url|webhook)/
hash.merge!(k => "http://example.com")
elsif service == 'irker' && k == :recipients
hash.merge!(k => 'irc://irc.network.net:666/#channel')
else
hash.merge!(k => "someword")
end
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment