BigW Consortium Gitlab

Commit dcea4501 by Yorick Peterse Committed by Dmitriy Zaporozhets

Fix parallel renaming of reserved projects

This ensures threads don't re-use the same connection object, and use fewer queries to perform the work.
parent d72b4042
require 'thread'
class RenameReservedProjectNames < ActiveRecord::Migration class RenameReservedProjectNames < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers include Gitlab::Database::MigrationHelpers
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
DOWNTIME = false DOWNTIME = false
class Project < ActiveRecord::Base; end THREAD_COUNT = 8
KNOWN_PATHS = %w(.well-known
all
assets
blame
blob
commits
create
create_dir
edit
files
files
find_file
groups
hooks
issues
logs_tree
merge_requests
new
new
preview
profile
projects
public
raw
repository
robots.txt
s
snippets
teams
tree
u
unsubscribes
update
users
wikis)
def up def up
threads = reserved_projects.each_slice(100).map do |slice| queues = Array.new(THREAD_COUNT) { Queue.new }
start = false
threads = Array.new(THREAD_COUNT) do |index|
Thread.new do Thread.new do
rename_projects(slice) queue = queues[index]
# Wait until we have input to process.
until start; end
rename_projects(queue.pop) until queue.empty?
end
end
enum = queues.each
reserved_projects.each_slice(100) do |slice|
begin
queue = enum.next
rescue StopIteration
enum.rewind
retry
end end
queue << slice
end end
start = true
threads.each(&:join) threads.each(&:join)
end end
...@@ -23,18 +84,17 @@ class RenameReservedProjectNames < ActiveRecord::Migration ...@@ -23,18 +84,17 @@ class RenameReservedProjectNames < ActiveRecord::Migration
private private
def reserved_projects def reserved_projects
select_all("SELECT p.id, p.path, p.repository_storage, n.path AS namespace_path, n.id AS namespace_id FROM projects p Project.unscoped.
INNER JOIN namespaces n ON n.id = p.namespace_id includes(:namespace).
WHERE p.path IN ( where('EXISTS (SELECT 1 FROM namespaces WHERE projects.namespace_id = namespaces.id)').
'.well-known', 'all', 'assets', 'files', 'groups', 'hooks', 'issues', where('projects.path' => KNOWN_PATHS)
'merge_requests', 'new', 'profile', 'projects', 'public', 'repository',
'robots.txt', 's', 'snippets', 'teams', 'u', 'unsubscribes', 'users',
'tree', 'commits', 'wikis', 'new', 'edit', 'create', 'update', 'logs_tree',
'preview', 'blob', 'blame', 'raw', 'files', 'create_dir', 'find_file')")
end end
def route_exists?(full_path) def route_exists?(full_path)
select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(full_path)}'").present? quoted_path = ActiveRecord::Base.connection.quote_string(full_path)
ActiveRecord::Base.connection.
select_all("SELECT id, path FROM routes WHERE path = '#{quoted_path}'").present?
end end
# Adds number to the end of the path that is not taken by other route # Adds number to the end of the path that is not taken by other route
...@@ -51,26 +111,27 @@ class RenameReservedProjectNames < ActiveRecord::Migration ...@@ -51,26 +111,27 @@ class RenameReservedProjectNames < ActiveRecord::Migration
end end
def rename_projects(projects) def rename_projects(projects)
projects.each do |row| projects.each do |project|
id = row['id'] next unless project.namespace
path_was = row['path']
namespace_path = row['namespace_path'] id = project.id
path_was = project.path
namespace_path = project.namespace.path
path = rename_path(namespace_path, path_was) path = rename_path(namespace_path, path_was)
project = Project.find_by(id: id)
begin begin
# Because project path update is quite complex operation we can't safely # Because project path update is quite complex operation we can't safely
# copy-paste all code from GitLab. As exception we use Rails code here # copy-paste all code from GitLab. As exception we use Rails code here
if project && project.rename_repo if rename_project_row(project, path)
project.respond_to?(:update_attributes) && rescue Exception => e # rubocop: disable Lint/RescueException
project.update_attributes(path: path) && Rails.logger.error "Exception when renaming project #{id}: #{e.message}"
project.respond_to?(:rename_repo)
project.rename_repo
end
rescue => e
Rails.logger.error "Exception when rename project #{id}: #{e.message}"
end end
end end
end end
def rename_project_row(project, path)
project.respond_to?(:update_attributes) &&
project.update_attributes(path: path) &&
project.respond_to?(:rename_repo)
end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment