BigW Consortium Gitlab

Skip to content

G
gitlab-ce
Commit 34da5a5d by Sean McGivern
Options

Merge branch 'gitlab-git-repository-log' into 'master'

Gitlab::Git shuffling and Gitaly annotations See merge request !12849
parents 7324d86f 5b18f733
Showing with 113 additions and 87 deletions
app/models/repository.rb
......@@ -123,6 +123,7 @@ class Repository
commits
end
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/384
def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0)
unless exists? && has_visible_content? && query.present?
return []
......@@ -610,6 +611,7 @@ class Repository
commit(sha)
end
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/383
def last_commit_id_for_path(sha, path)
key = path.blank? ? "last_commit_id_for_path:#{sha}" : "last_commit_id_for_path:#{sha}:#{Digest::SHA1.hexdigest(path)}"
......
app/services/git_operation_service.rb
......@@ -129,6 +129,7 @@ class GitOperationService
end
end
# Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites.
def update_ref(ref, newrev, oldrev)
# We use 'git update-ref' because libgit2/rugged currently does not
# offer 'compare and swap' ref updates. Without compare-and-swap we can
......
lib/gitlab/git/commit.rb
......@@ -38,7 +38,7 @@ module Gitlab
repo = options.delete(:repo)
raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
repo.log(options).map { |c| decorate(c) }
repo.log(options)
end
# Get single commit
......
lib/gitlab/git/repository.rb
......@@ -331,85 +331,10 @@ module Gitlab
# )
#
def log(options)
default_options = {
limit: 10,
offset: 0,
path: nil,
follow: false,
skip_merges: false,
disable_walk: false,
after: nil,
before: nil
}
options = default_options.merge(options)
options[:limit] ||= 0
options[:offset] ||= 0
actual_ref = options[:ref] || root_ref
begin
sha = sha_from_ref(actual_ref)
rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
# Return an empty array if the ref wasn't found
return []
end
if log_using_shell?(options)
log_by_shell(sha, options)
else
log_by_walk(sha, options)
end
end
def log_using_shell?(options)
options[:path].present? ||
options[:disable_walk] ||
options[:skip_merges] ||
options[:after] ||
options[:before]
end
def log_by_walk(sha, options)
walk_options = {
show: sha,
sort: Rugged::SORT_NONE,
limit: options[:limit],
offset: options[:offset]
}
Rugged::Walker.walk(rugged, walk_options).to_a
end
def log_by_shell(sha, options)
limit = options[:limit].to_i
offset = options[:offset].to_i
use_follow_flag = options[:follow] && options[:path].present?
# We will perform the offset in Ruby because --follow doesn't play well with --skip.
# See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
offset_in_ruby = use_follow_flag && options[:offset].present?
limit += offset if offset_in_ruby
cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
cmd << "--max-count=#{limit}"
cmd << '--format=%H'
cmd << "--skip=#{offset}" unless offset_in_ruby
cmd << '--follow' if use_follow_flag
cmd << '--no-merges' if options[:skip_merges]
cmd << "--after=#{options[:after].iso8601}" if options[:after]
cmd << "--before=#{options[:before].iso8601}" if options[:before]
cmd << sha
# :path can be a string or an array of strings
if options[:path].present?
cmd << '--'
cmd += Array(options[:path])
end
raw_output = IO.popen(cmd) { |io| io.read }
lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
raw_log(options).map { |c| Commit.decorate(c) }
end
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382
def count_commits(options)
cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
cmd << "--after=#{options[:after].iso8601}" if options[:after]
......@@ -454,7 +379,7 @@ module Gitlab
# Counts the amount of commits between `from` and `to`.
def count_commits_between(from, to)
commits_between(from, to).size
Commit.between(self, from, to).size
end
# Returns the SHA of the most recent common ancestor of +from+ and +to+
......@@ -912,6 +837,89 @@ module Gitlab
private
def raw_log(options)
default_options = {
limit: 10,
offset: 0,
path: nil,
follow: false,
skip_merges: false,
disable_walk: false,
after: nil,
before: nil
}
options = default_options.merge(options)
options[:limit] ||= 0
options[:offset] ||= 0
actual_ref = options[:ref] || root_ref
begin
sha = sha_from_ref(actual_ref)
rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
# Return an empty array if the ref wasn't found
return []
end
if log_using_shell?(options)
log_by_shell(sha, options)
else
log_by_walk(sha, options)
end
end
def log_using_shell?(options)
options[:path].present? ||
options[:disable_walk] ||
options[:skip_merges] ||
options[:after] ||
options[:before]
end
def log_by_walk(sha, options)
walk_options = {
show: sha,
sort: Rugged::SORT_NONE,
limit: options[:limit],
offset: options[:offset]
}
Rugged::Walker.walk(rugged, walk_options).to_a
end
# Gitaly note: JV: although #log_by_shell shells out to Git I think the
# complexity is such that we should migrate it as Ruby before trying to
# do it in Go.
def log_by_shell(sha, options)
limit = options[:limit].to_i
offset = options[:offset].to_i
use_follow_flag = options[:follow] && options[:path].present?
# We will perform the offset in Ruby because --follow doesn't play well with --skip.
# See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
offset_in_ruby = use_follow_flag && options[:offset].present?
limit += offset if offset_in_ruby
cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
cmd << "--max-count=#{limit}"
cmd << '--format=%H'
cmd << "--skip=#{offset}" unless offset_in_ruby
cmd << '--follow' if use_follow_flag
cmd << '--no-merges' if options[:skip_merges]
cmd << "--after=#{options[:after].iso8601}" if options[:after]
cmd << "--before=#{options[:before].iso8601}" if options[:before]
cmd << sha
# :path can be a string or an array of strings
if options[:path].present?
cmd << '--'
cmd += Array(options[:path])
end
raw_output = IO.popen(cmd) { |io| io.read }
lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
end
# We are trying to deprecate this method because it does a lot of work
# but it seems to be used only to look up submodule URL's.
# https://gitlab.com/gitlab-org/gitaly/issues/329
......
lib/gitlab/git_ref_validator.rb
# Gitaly note: JV: does not need to be migrated, works without a repo.
module Gitlab
module GitRefValidator
extend self
......
lib/gitlab/shell.rb
# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
# SSH key operations are not part of Gitaly so will never be migrated.
require 'securerandom'
module Gitlab
......@@ -68,6 +71,7 @@ module Gitlab
# Ex.
# add_repository("/path/to/storage", "gitlab/gitlab-ci")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def add_repository(storage, name)
gitlab_shell_fast_execute([gitlab_shell_projects_path,
'add-project', storage, "#{name}.git"])
......@@ -81,6 +85,7 @@ module Gitlab
# Ex.
# import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def import_repository(storage, name, url)
# Timeout should be less than 900 ideally, to prevent the memory killer
# to silently kill the process without knowing we are timing out here.
......@@ -99,6 +104,7 @@ module Gitlab
# Ex.
# fetch_remote("gitlab/gitlab-ci", "upstream")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def fetch_remote(storage, name, remote, forced: false, no_tags: false)
args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
args << '--force' if forced
......@@ -115,6 +121,7 @@ module Gitlab
# Ex.
# mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def mv_repository(storage, path, new_path)
gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
storage, "#{path}.git", "#{new_path}.git"])
......@@ -129,6 +136,7 @@ module Gitlab
# Ex.
# fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
#
# Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
forked_from_storage, "#{path}.git", forked_to_storage,
......@@ -143,6 +151,7 @@ module Gitlab
# Ex.
# remove_repository("/path/to/storage", "gitlab/gitlab-ci")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def remove_repository(storage, name)
gitlab_shell_fast_execute([gitlab_shell_projects_path,
'rm-project', storage, "#{name}.git"])
......@@ -194,6 +203,7 @@ module Gitlab
# Ex.
# add_namespace("/path/to/storage", "gitlab")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def add_namespace(storage, name)
path = full_path(storage, name)
FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
......@@ -207,6 +217,7 @@ module Gitlab
# Ex.
# rm_namespace("/path/to/storage", "gitlab")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def rm_namespace(storage, name)
FileUtils.rm_r(full_path(storage, name), force: true)
end
......@@ -216,6 +227,7 @@ module Gitlab
# Ex.
# mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def mv_namespace(storage, old_name, new_name)
return false if exists?(storage, new_name) || !exists?(storage, old_name)
......@@ -241,6 +253,7 @@ module Gitlab
# exists?(storage, 'gitlab')
# exists?(storage, 'gitlab/cookies.git')
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def exists?(storage, dir_name)
File.exist?(full_path(storage, dir_name))
end
......
spec/lib/gitlab/git/repository_spec.rb
......@@ -705,9 +705,9 @@ describe Gitlab::Git::Repository, seed_helper: true do
# Add new commits so that there's a renamed file in the commit history
repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
commit_with_old_name = new_commit_edit_old_file(repo)
rename_commit = new_commit_move_file(repo)
commit_with_new_name = new_commit_edit_new_file(repo)
commit_with_old_name = Gitlab::Git::Commit.decorate(new_commit_edit_old_file(repo))
rename_commit = Gitlab::Git::Commit.decorate(new_commit_move_file(repo))
commit_with_new_name = Gitlab::Git::Commit.decorate(new_commit_edit_new_file(repo))
end
after(:context) do
......@@ -880,8 +880,8 @@ describe Gitlab::Git::Repository, seed_helper: true do
context "compare results between log_by_walk and log_by_shell" do
let(:options) { { ref: "master" } }
let(:commits_by_walk) { repository.log(options).map(&:oid) }
let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:oid) }
let(:commits_by_walk) { repository.log(options).map(&:id) }
let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:id) }
it { expect(commits_by_walk).to eq(commits_by_shell) }
......@@ -924,7 +924,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
expect(commits.size).to be > 0
expect(commits).to satisfy do |commits|
commits.all? { |commit| commit.time >= options[:after] }
commits.all? { |commit| commit.committed_date >= options[:after] }
end
end
end
......@@ -937,7 +937,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
expect(commits.size).to be > 0
expect(commits).to satisfy do |commits|
commits.all? { |commit| commit.time <= options[:before] }
commits.all? { |commit| commit.committed_date <= options[:before] }
end
end
end
......@@ -946,7 +946,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
let(:options) { { ref: 'master', path: ['PROCESS.md', 'README.md'] } }
def commit_files(commit)
commit.diff(commit.parent_ids.first).deltas.flat_map do |delta|
commit.diff_from_parent.deltas.flat_map do |delta|
[delta.old_file[:path], delta.new_file[:path]].uniq.compact
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment