BigW Consortium Gitlab

Commit 5cf56e56 by Jacob Vosmaer

Rename almost all the things

parent e3558ed6
class Admin::ProjectsController < Admin::ApplicationController class Admin::ProjectsController < Admin::ApplicationController
before_action :project, only: [:show, :transfer, :repo_check] before_action :project, only: [:show, :transfer, :repository_check]
before_action :group, only: [:show, :transfer] before_action :group, only: [:show, :transfer]
def index def index
...@@ -8,7 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController ...@@ -8,7 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController
@projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present? @projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present?
@projects = @projects.with_push if params[:with_push].present? @projects = @projects.with_push if params[:with_push].present?
@projects = @projects.abandoned if params[:abandoned].present? @projects = @projects.abandoned if params[:abandoned].present?
@projects = @projects.where(last_repo_check_failed: true) if params[:last_repo_check_failed].present? @projects = @projects.where(last_repository_check_failed: true) if params[:last_repository_check_failed].present?
@projects = @projects.non_archived unless params[:with_archived].present? @projects = @projects.non_archived unless params[:with_archived].present?
@projects = @projects.search(params[:name]) if params[:name].present? @projects = @projects.search(params[:name]) if params[:name].present?
@projects = @projects.sort(@sort = params[:sort]) @projects = @projects.sort(@sort = params[:sort])
...@@ -31,24 +31,24 @@ class Admin::ProjectsController < Admin::ApplicationController ...@@ -31,24 +31,24 @@ class Admin::ProjectsController < Admin::ApplicationController
redirect_to admin_namespace_project_path(@project.namespace, @project) redirect_to admin_namespace_project_path(@project.namespace, @project)
end end
def repo_check def repository_check
SingleRepoCheckWorker.perform_async(@project.id) SingleRepositoryCheckWorker.perform_async(@project.id)
redirect_to( redirect_to(
admin_namespace_project_path(@project.namespace, @project), admin_namespace_project_path(@project.namespace, @project),
notice: 'Repo check was triggered' notice: 'Repository check was triggered'
) )
end end
def clear_repo_check_states def clear_repository_check_states
Project.update_all( Project.update_all(
last_repo_check_failed: false, last_repository_check_failed: false,
last_repo_check_at: nil last_repository_check_at: nil
) )
redirect_to( redirect_to(
admin_namespaces_projects_path, admin_namespaces_projects_path,
notice: 'All project repo check states were cleared' notice: 'All project states were cleared'
) )
end end
......
class RepoCheckMailer < BaseMailer class RepositoryCheckMailer < BaseMailer
include ActionView::Helpers::TextHelper include ActionView::Helpers::TextHelper
def notify(failed_count) def notify(failed_count)
......
- page_title "Logs" - page_title "Logs"
- loggers = [Gitlab::GitLogger, Gitlab::AppLogger, - loggers = [Gitlab::GitLogger, Gitlab::AppLogger,
Gitlab::ProductionLogger, Gitlab::SidekiqLogger, Gitlab::ProductionLogger, Gitlab::SidekiqLogger,
Gitlab::RepoCheckLogger] Gitlab::RepositoryCheckLogger]
%ul.nav-links.log-tabs %ul.nav-links.log-tabs
- loggers.each do |klass| - loggers.each do |klass|
%li{ class: (klass == Gitlab::GitLogger ? 'active' : '') } %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') }
......
...@@ -41,19 +41,19 @@ ...@@ -41,19 +41,19 @@
%fieldset %fieldset
%strong Problems %strong Problems
.checkbox .checkbox
= label_tag :last_repo_check_failed do = label_tag :last_repository_check_failed do
= check_box_tag :last_repo_check_failed, 1, params[:last_repo_check_failed] = check_box_tag :last_repository_check_failed, 1, params[:last_repository_check_failed]
%span Last repo check failed %span Last repository check failed
= hidden_field_tag :sort, params[:sort] = hidden_field_tag :sort, params[:sort]
= button_tag "Search", class: "btn submit btn-primary" = button_tag "Search", class: "btn submit btn-primary"
= link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel" = link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel"
.panel.panel-default.repo-check-states .panel.panel-default.repository-check-states
.panel-heading .panel-heading
Repo check states Repository check states
.panel-body .panel-body
= link_to 'Clear all', clear_repo_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repo check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove" = link_to 'Clear all', clear_repository_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repository check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
%section.col-md-9 %section.col-md-9
.panel.panel-default .panel.panel-default
.panel-heading .panel-heading
......
...@@ -5,12 +5,12 @@ ...@@ -5,12 +5,12 @@
%i.fa.fa-pencil-square-o %i.fa.fa-pencil-square-o
Edit Edit
%hr %hr
- if @project.last_repo_check_failed? - if @project.last_repository_check_failed?
.row .row
.col-md-12 .col-md-12
.panel .panel
.panel-heading.alert.alert-danger .panel-heading.alert.alert-danger
Last repo check failed. See Last repository check failed. See
= link_to 'repocheck.log', admin_logs_path = link_to 'repocheck.log', admin_logs_path
for error messages. for error messages.
.row .row
...@@ -103,19 +103,19 @@ ...@@ -103,19 +103,19 @@
.col-sm-offset-2.col-sm-10 .col-sm-offset-2.col-sm-10
= f.submit 'Transfer', class: 'btn btn-primary' = f.submit 'Transfer', class: 'btn btn-primary'
.panel.panel-default.repo-check .panel.panel-default.repository-check
.panel-heading .panel-heading
Repo check Repository check
.panel-body .panel-body
= form_for @project, url: repo_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f| = form_for @project, url: repository_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f|
.form-group .form-group
- if @project.last_repo_check_at.nil? - if @project.last_repository_check_at.nil?
This repository has never been checked. This repository has never been checked.
- else - else
This repository was last checked This repository was last checked
= @project.last_repo_check_at.to_s(:medium) + '.' = @project.last_repository_check_at.to_s(:medium) + '.'
The check The check
- if @project.last_repo_check_failed? - if @project.last_repository_check_failed?
= succeed '.' do = succeed '.' do
%strong.cred failed %strong.cred failed
See See
...@@ -125,7 +125,7 @@ ...@@ -125,7 +125,7 @@
passed. passed.
.form-group .form-group
= f.submit 'Trigger repo check', class: 'btn btn-primary' = f.submit 'Trigger repository check', class: 'btn btn-primary'
.col-md-6 .col-md-6
- if @group - if @group
......
...@@ -2,4 +2,4 @@ ...@@ -2,4 +2,4 @@
#{@message}. #{@message}.
%p %p
= link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repo_check_failed: 1) = link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repository_check_failed: 1)
#{@message}. #{@message}.
\ \
View details: #{admin_namespaces_projects_url(last_repo_check_failed: 1)} View details: #{admin_namespaces_projects_url(last_repository_check_failed: 1)}
...@@ -4,9 +4,9 @@ class AdminEmailWorker ...@@ -4,9 +4,9 @@ class AdminEmailWorker
sidekiq_options retry: false # this job auto-repeats via sidekiq-cron sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
def perform def perform
repo_check_failed_count = Project.where(last_repo_check_failed: true).count repository_check_failed_count = Project.where(last_repository_check_failed: true).count
return if repo_check_failed_count.zero? return if repository_check_failed_count.zero?
RepoCheckMailer.notify(repo_check_failed_count).deliver_now RepositoryCheckMailer.notify(repository_check_failed_count).deliver_now
end end
end end
class RepoCheckWorker class RepositoryCheckWorker
include Sidekiq::Worker include Sidekiq::Worker
RUN_TIME = 3600 RUN_TIME = 3600
...@@ -11,14 +11,14 @@ class RepoCheckWorker ...@@ -11,14 +11,14 @@ class RepoCheckWorker
# This loop will break after a little more than one hour ('a little # This loop will break after a little more than one hour ('a little
# more' because `git fsck` may take a few minutes), or if it runs out of # more' because `git fsck` may take a few minutes), or if it runs out of
# projects to check. By default sidekiq-cron will start a new # projects to check. By default sidekiq-cron will start a new
# RepoCheckWorker each hour so that as long as there are repositories to # RepositoryCheckWorker each hour so that as long as there are repositories to
# check, only one (or two) will be checked at a time. # check, only one (or two) will be checked at a time.
project_ids.each do |project_id| project_ids.each do |project_id|
break if Time.now - start >= RUN_TIME break if Time.now - start >= RUN_TIME
next if !try_obtain_lease(project_id) next if !try_obtain_lease(project_id)
SingleRepoCheckWorker.new.perform(project_id) SingleRepositoryCheckWorker.new.perform(project_id)
end end
end end
...@@ -29,16 +29,16 @@ class RepoCheckWorker ...@@ -29,16 +29,16 @@ class RepoCheckWorker
# an array of IDs instead. # an array of IDs instead.
def project_ids def project_ids
limit = 10_000 limit = 10_000
never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit). never_checked_projects = Project.where('last_repository_check_at IS NULL').limit(limit).
pluck(:id) pluck(:id)
old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago). old_check_projects = Project.where('last_repository_check_at < ?', 1.week.ago).
reorder('last_repo_check_at ASC').limit(limit).pluck(:id) reorder('last_repository_check_at ASC').limit(limit).pluck(:id)
never_checked_projects + old_check_projects never_checked_projects + old_check_projects
end end
def try_obtain_lease(id) def try_obtain_lease(id)
lease = Gitlab::ExclusiveLease.new( lease = Gitlab::ExclusiveLease.new(
"project_repo_check:#{id}", "project_repository_check:#{id}",
timeout: RUN_TIME timeout: RUN_TIME
) )
lease.try_obtain lease.try_obtain
......
class SingleRepoCheckWorker class SingleRepositoryCheckWorker
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: false sidekiq_options retry: false
...@@ -21,14 +21,14 @@ class SingleRepoCheckWorker ...@@ -21,14 +21,14 @@ class SingleRepoCheckWorker
output, status = Gitlab::Popen.popen(cmd) output, status = Gitlab::Popen.popen(cmd)
return true if status.zero? return true if status.zero?
Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}") Gitlab::RepositoryCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
false false
end end
def update(project, success:) def update(project, success:)
project.update_columns( project.update_columns(
last_repo_check_failed: !success, last_repository_check_failed: !success,
last_repo_check_at: Time.now, last_repository_check_at: Time.now,
) )
end end
end end
...@@ -157,7 +157,7 @@ production: &base ...@@ -157,7 +157,7 @@ production: &base
cron: "0 0 * * *" cron: "0 0 * * *"
# Periodically run 'git fsck' on all repositories. If started more than # Periodically run 'git fsck' on all repositories. If started more than
# once per hour you will have concurrent 'git fsck' jobs. # once per hour you will have concurrent 'git fsck' jobs.
repo_check_worker: repository_check_worker:
cron: "20 * * * *" cron: "20 * * * *"
# Send admin emails once a day # Send admin emails once a day
admin_email_worker: admin_email_worker:
......
...@@ -239,9 +239,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) ...@@ -239,9 +239,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker' Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
Settings.cron_jobs['repo_check_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repo_check_worker']['cron'] ||= '20 * * * *' Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
Settings.cron_jobs['repo_check_worker']['job_class'] = 'RepoCheckWorker' Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheckWorker'
Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
......
...@@ -264,11 +264,11 @@ Rails.application.routes.draw do ...@@ -264,11 +264,11 @@ Rails.application.routes.draw do
member do member do
put :transfer put :transfer
post :repo_check post :repository_check
end end
collection do collection do
put :clear_repo_check_states put :clear_repository_check_states
end end
resources :runner_projects resources :runner_projects
......
class ProjectAddRepoCheck < ActiveRecord::Migration class ProjectAddRepositoryCheck < ActiveRecord::Migration
def change def change
add_column :projects, :last_repo_check_failed, :boolean, default: false add_column :projects, :last_repository_check_failed, :boolean, default: false
add_column :projects, :last_repo_check_at, :datetime add_column :projects, :last_repository_check_at, :datetime
end end
end end
...@@ -732,8 +732,8 @@ ActiveRecord::Schema.define(version: 20160331133914) do ...@@ -732,8 +732,8 @@ ActiveRecord::Schema.define(version: 20160331133914) do
t.boolean "public_builds", default: true, null: false t.boolean "public_builds", default: true, null: false
t.string "main_language" t.string "main_language"
t.integer "pushes_since_gc", default: 0 t.integer "pushes_since_gc", default: 0
t.boolean "last_repo_check_failed", default: false t.boolean "last_repository_check_failed", default: false
t.datetime "last_repo_check_at" t.datetime "last_repository_check_at"
end end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
# Repo checks # Repository checks
_**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_ _**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_
...@@ -13,10 +13,10 @@ output on the admin log page under 'repocheck.log'. ...@@ -13,10 +13,10 @@ output on the admin log page under 'repocheck.log'.
## Periodical checks ## Periodical checks
GitLab periodically runs a repo check on all project repositories and GitLab periodically runs a repository check on all project repositories and
wiki repositories in order to detect data corruption problems. A wiki repositories in order to detect data corruption problems. A
project will be checked no more than once per week. If any projects project will be checked no more than once per week. If any projects
fail their repo checks all GitLab administrators will receive an email fail their repository checks all GitLab administrators will receive an email
notification of the situation. This notification is sent out no more notification of the situation. This notification is sent out no more
than once a day. than once a day.
...@@ -27,27 +27,27 @@ schedule in gitlab.yml. ...@@ -27,27 +27,27 @@ schedule in gitlab.yml.
``` ```
# For omnibus installations, in /etc/gitlab/gitlab.rb: # For omnibus installations, in /etc/gitlab/gitlab.rb:
gitlab_rails['cron_jobs_repo_check_worker_cron'] = '' gitlab_rails['cron_jobs_repository_check_worker_cron'] = ''
``` ```
``` ```
# For installations from source, in config/gitlab.yml: # For installations from source, in config/gitlab.yml:
cron_jobs: cron_jobs:
repo_check_worker: repository_check_worker:
cron: "" cron: ""
``` ```
## What to do if a check failed ## What to do if a check failed
If the repo check fails for some repository you should look up the error If the repository check fails for some repository you should look up the error
in repocheck.log (in the admin panel or on disk; see in repocheck.log (in the admin panel or on disk; see
`/var/log/gitlab/gitlab-rails` for Omnibus installations or `/var/log/gitlab/gitlab-rails` for Omnibus installations or
`/home/git/gitlab/log` for installations from source). Once you have `/home/git/gitlab/log` for installations from source). Once you have
resolved the issue use the admin panel to trigger a new repo check on resolved the issue use the admin panel to trigger a new repository check on
the project. This will clear the 'check failed' state. the project. This will clear the 'check failed' state.
If for some reason the periodical repo check caused a lot of false If for some reason the periodical repository check caused a lot of false
alarms you can choose to clear ALL repo check states from the admin alarms you can choose to clear ALL repository check states from the admin
project index page. project index page.
--- ---
......
module Gitlab module Gitlab
class RepoCheckLogger < Gitlab::Logger class RepositoryCheckLogger < Gitlab::Logger
def self.file_name_noext def self.file_name_noext
'repocheck' 'repocheck'
end end
......
...@@ -33,33 +33,33 @@ describe "Admin Projects", feature: true do ...@@ -33,33 +33,33 @@ describe "Admin Projects", feature: true do
end end
end end
feature 'repo checks' do feature 'repository checks' do
scenario 'trigger repo check' do scenario 'trigger repository check' do
visit_admin_project_page visit_admin_project_page
page.within('.repo-check') do page.within('.repository-check') do
click_button 'Trigger repo check' click_button 'Trigger repository check'
end end
expect(page).to have_content('Repo check was triggered') expect(page).to have_content('Repository check was triggered')
end end
scenario 'see failed repo check' do scenario 'see failed repository check' do
@project.update_column(:last_repo_check_failed, true) @project.update_column(:last_repository_check_failed, true)
visit_admin_project_page visit_admin_project_page
expect(page).to have_content('Last repo check failed') expect(page).to have_content('Last repository check failed')
end end
scenario 'clear repo checks', js: true do scenario 'clear repository checks', js: true do
@project.update_column(:last_repo_check_failed, true) @project.update_column(:last_repository_check_failed, true)
visit admin_namespaces_projects_path visit admin_namespaces_projects_path
page.within('.repo-check-states') do page.within('.repository-check-states') do
click_link 'Clear all' # pop-up should be auto confirmed click_link 'Clear all' # pop-up should be auto confirmed
end end
expect(@project.reload.last_repo_check_failed).to eq(false) expect(@project.reload.last_repository_check_failed).to eq(false)
end end
end end
......
require 'rails_helper' require 'rails_helper'
describe RepoCheckMailer do describe RepositoryCheckMailer do
include EmailSpec::Matchers include EmailSpec::Matchers
describe '.notify' do describe '.notify' do
......
require 'spec_helper' require 'spec_helper'
describe RepoCheckWorker do describe RepositoryCheckWorker do
subject { RepoCheckWorker.new } subject { RepositoryCheckWorker.new }
it 'prefers projects that have never been checked' do it 'prefers projects that have never been checked' do
projects = 3.times.map { create(:project) } projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 1.month.ago) projects[0].update_column(:last_repository_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.weeks.ago) projects[2].update_column(:last_repository_check_at, 3.weeks.ago)
expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id)) expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id))
end end
it 'sorts projects by last_repo_check_at' do it 'sorts projects by last_repository_check_at' do
projects = 3.times.map { create(:project) } projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 2.weeks.ago) projects[0].update_column(:last_repository_check_at, 2.weeks.ago)
projects[1].update_column(:last_repo_check_at, 1.month.ago) projects[1].update_column(:last_repository_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.weeks.ago) projects[2].update_column(:last_repository_check_at, 3.weeks.ago)
expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id)) expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id))
end end
it 'excludes projects that were checked recently' do it 'excludes projects that were checked recently' do
projects = 3.times.map { create(:project) } projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 2.days.ago) projects[0].update_column(:last_repository_check_at, 2.days.ago)
projects[1].update_column(:last_repo_check_at, 1.month.ago) projects[1].update_column(:last_repository_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.days.ago) projects[2].update_column(:last_repository_check_at, 3.days.ago)
expect(subject.perform).to eq([projects[1].id]) expect(subject.perform).to eq([projects[1].id])
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment