BigW Consortium Gitlab

Commit 5cf56e56 by Jacob Vosmaer

Rename almost all the things

parent e3558ed6
class Admin::ProjectsController < Admin::ApplicationController
before_action :project, only: [:show, :transfer, :repo_check]
before_action :project, only: [:show, :transfer, :repository_check]
before_action :group, only: [:show, :transfer]
def index
......@@ -8,7 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController
@projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present?
@projects = @projects.with_push if params[:with_push].present?
@projects = @projects.abandoned if params[:abandoned].present?
@projects = @projects.where(last_repo_check_failed: true) if params[:last_repo_check_failed].present?
@projects = @projects.where(last_repository_check_failed: true) if params[:last_repository_check_failed].present?
@projects = @projects.non_archived unless params[:with_archived].present?
@projects = @projects.search(params[:name]) if params[:name].present?
@projects = @projects.sort(@sort = params[:sort])
......@@ -31,24 +31,24 @@ class Admin::ProjectsController < Admin::ApplicationController
redirect_to admin_namespace_project_path(@project.namespace, @project)
end
def repo_check
SingleRepoCheckWorker.perform_async(@project.id)
def repository_check
SingleRepositoryCheckWorker.perform_async(@project.id)
redirect_to(
admin_namespace_project_path(@project.namespace, @project),
notice: 'Repo check was triggered'
notice: 'Repository check was triggered'
)
end
def clear_repo_check_states
def clear_repository_check_states
Project.update_all(
last_repo_check_failed: false,
last_repo_check_at: nil
last_repository_check_failed: false,
last_repository_check_at: nil
)
redirect_to(
admin_namespaces_projects_path,
notice: 'All project repo check states were cleared'
notice: 'All project states were cleared'
)
end
......
class RepoCheckMailer < BaseMailer
class RepositoryCheckMailer < BaseMailer
include ActionView::Helpers::TextHelper
def notify(failed_count)
......
- page_title "Logs"
- loggers = [Gitlab::GitLogger, Gitlab::AppLogger,
Gitlab::ProductionLogger, Gitlab::SidekiqLogger,
Gitlab::RepoCheckLogger]
Gitlab::RepositoryCheckLogger]
%ul.nav-links.log-tabs
- loggers.each do |klass|
%li{ class: (klass == Gitlab::GitLogger ? 'active' : '') }
......
......@@ -41,19 +41,19 @@
%fieldset
%strong Problems
.checkbox
= label_tag :last_repo_check_failed do
= check_box_tag :last_repo_check_failed, 1, params[:last_repo_check_failed]
%span Last repo check failed
= label_tag :last_repository_check_failed do
= check_box_tag :last_repository_check_failed, 1, params[:last_repository_check_failed]
%span Last repository check failed
= hidden_field_tag :sort, params[:sort]
= button_tag "Search", class: "btn submit btn-primary"
= link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel"
.panel.panel-default.repo-check-states
.panel.panel-default.repository-check-states
.panel-heading
Repo check states
Repository check states
.panel-body
= link_to 'Clear all', clear_repo_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repo check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
= link_to 'Clear all', clear_repository_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repository check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
%section.col-md-9
.panel.panel-default
.panel-heading
......
......@@ -5,12 +5,12 @@
%i.fa.fa-pencil-square-o
Edit
%hr
- if @project.last_repo_check_failed?
- if @project.last_repository_check_failed?
.row
.col-md-12
.panel
.panel-heading.alert.alert-danger
Last repo check failed. See
Last repository check failed. See
= link_to 'repocheck.log', admin_logs_path
for error messages.
.row
......@@ -103,19 +103,19 @@
.col-sm-offset-2.col-sm-10
= f.submit 'Transfer', class: 'btn btn-primary'
.panel.panel-default.repo-check
.panel.panel-default.repository-check
.panel-heading
Repo check
Repository check
.panel-body
= form_for @project, url: repo_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f|
= form_for @project, url: repository_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f|
.form-group
- if @project.last_repo_check_at.nil?
- if @project.last_repository_check_at.nil?
This repository has never been checked.
- else
This repository was last checked
= @project.last_repo_check_at.to_s(:medium) + '.'
= @project.last_repository_check_at.to_s(:medium) + '.'
The check
- if @project.last_repo_check_failed?
- if @project.last_repository_check_failed?
= succeed '.' do
%strong.cred failed
See
......@@ -125,7 +125,7 @@
passed.
.form-group
= f.submit 'Trigger repo check', class: 'btn btn-primary'
= f.submit 'Trigger repository check', class: 'btn btn-primary'
.col-md-6
- if @group
......
......@@ -2,4 +2,4 @@
#{@message}.
%p
= link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repo_check_failed: 1)
= link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repository_check_failed: 1)
#{@message}.
\
View details: #{admin_namespaces_projects_url(last_repo_check_failed: 1)}
View details: #{admin_namespaces_projects_url(last_repository_check_failed: 1)}
......@@ -4,9 +4,9 @@ class AdminEmailWorker
sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
def perform
repo_check_failed_count = Project.where(last_repo_check_failed: true).count
return if repo_check_failed_count.zero?
repository_check_failed_count = Project.where(last_repository_check_failed: true).count
return if repository_check_failed_count.zero?
RepoCheckMailer.notify(repo_check_failed_count).deliver_now
RepositoryCheckMailer.notify(repository_check_failed_count).deliver_now
end
end
class RepoCheckWorker
class RepositoryCheckWorker
include Sidekiq::Worker
RUN_TIME = 3600
......@@ -11,14 +11,14 @@ class RepoCheckWorker
# This loop will break after a little more than one hour ('a little
# more' because `git fsck` may take a few minutes), or if it runs out of
# projects to check. By default sidekiq-cron will start a new
# RepoCheckWorker each hour so that as long as there are repositories to
# RepositoryCheckWorker each hour so that as long as there are repositories to
# check, only one (or two) will be checked at a time.
project_ids.each do |project_id|
break if Time.now - start >= RUN_TIME
next if !try_obtain_lease(project_id)
SingleRepoCheckWorker.new.perform(project_id)
SingleRepositoryCheckWorker.new.perform(project_id)
end
end
......@@ -29,16 +29,16 @@ class RepoCheckWorker
# an array of IDs instead.
def project_ids
limit = 10_000
never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit).
never_checked_projects = Project.where('last_repository_check_at IS NULL').limit(limit).
pluck(:id)
old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago).
reorder('last_repo_check_at ASC').limit(limit).pluck(:id)
old_check_projects = Project.where('last_repository_check_at < ?', 1.week.ago).
reorder('last_repository_check_at ASC').limit(limit).pluck(:id)
never_checked_projects + old_check_projects
end
def try_obtain_lease(id)
lease = Gitlab::ExclusiveLease.new(
"project_repo_check:#{id}",
"project_repository_check:#{id}",
timeout: RUN_TIME
)
lease.try_obtain
......
class SingleRepoCheckWorker
class SingleRepositoryCheckWorker
include Sidekiq::Worker
sidekiq_options retry: false
......@@ -21,14 +21,14 @@ class SingleRepoCheckWorker
output, status = Gitlab::Popen.popen(cmd)
return true if status.zero?
Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
Gitlab::RepositoryCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
false
end
def update(project, success:)
project.update_columns(
last_repo_check_failed: !success,
last_repo_check_at: Time.now,
last_repository_check_failed: !success,
last_repository_check_at: Time.now,
)
end
end
......@@ -157,7 +157,7 @@ production: &base
cron: "0 0 * * *"
# Periodically run 'git fsck' on all repositories. If started more than
# once per hour you will have concurrent 'git fsck' jobs.
repo_check_worker:
repository_check_worker:
cron: "20 * * * *"
# Send admin emails once a day
admin_email_worker:
......
......@@ -239,9 +239,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
Settings.cron_jobs['repo_check_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repo_check_worker']['cron'] ||= '20 * * * *'
Settings.cron_jobs['repo_check_worker']['job_class'] = 'RepoCheckWorker'
Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheckWorker'
Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
......
......@@ -264,11 +264,11 @@ Rails.application.routes.draw do
member do
put :transfer
post :repo_check
post :repository_check
end
collection do
put :clear_repo_check_states
put :clear_repository_check_states
end
resources :runner_projects
......
class ProjectAddRepoCheck < ActiveRecord::Migration
class ProjectAddRepositoryCheck < ActiveRecord::Migration
def change
add_column :projects, :last_repo_check_failed, :boolean, default: false
add_column :projects, :last_repo_check_at, :datetime
add_column :projects, :last_repository_check_failed, :boolean, default: false
add_column :projects, :last_repository_check_at, :datetime
end
end
......@@ -732,8 +732,8 @@ ActiveRecord::Schema.define(version: 20160331133914) do
t.boolean "public_builds", default: true, null: false
t.string "main_language"
t.integer "pushes_since_gc", default: 0
t.boolean "last_repo_check_failed", default: false
t.datetime "last_repo_check_at"
t.boolean "last_repository_check_failed", default: false
t.datetime "last_repository_check_at"
end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
# Repo checks
# Repository checks
_**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_
......@@ -13,10 +13,10 @@ output on the admin log page under 'repocheck.log'.
## Periodical checks
GitLab periodically runs a repo check on all project repositories and
GitLab periodically runs a repository check on all project repositories and
wiki repositories in order to detect data corruption problems. A
project will be checked no more than once per week. If any projects
fail their repo checks all GitLab administrators will receive an email
fail their repository checks all GitLab administrators will receive an email
notification of the situation. This notification is sent out no more
than once a day.
......@@ -27,27 +27,27 @@ schedule in gitlab.yml.
```
# For omnibus installations, in /etc/gitlab/gitlab.rb:
gitlab_rails['cron_jobs_repo_check_worker_cron'] = ''
gitlab_rails['cron_jobs_repository_check_worker_cron'] = ''
```
```
# For installations from source, in config/gitlab.yml:
cron_jobs:
repo_check_worker:
repository_check_worker:
cron: ""
```
## What to do if a check failed
If the repo check fails for some repository you should look up the error
If the repository check fails for some repository you should look up the error
in repocheck.log (in the admin panel or on disk; see
`/var/log/gitlab/gitlab-rails` for Omnibus installations or
`/home/git/gitlab/log` for installations from source). Once you have
resolved the issue use the admin panel to trigger a new repo check on
resolved the issue use the admin panel to trigger a new repository check on
the project. This will clear the 'check failed' state.
If for some reason the periodical repo check caused a lot of false
alarms you can choose to clear ALL repo check states from the admin
If for some reason the periodical repository check caused a lot of false
alarms you can choose to clear ALL repository check states from the admin
project index page.
---
......
module Gitlab
class RepoCheckLogger < Gitlab::Logger
class RepositoryCheckLogger < Gitlab::Logger
def self.file_name_noext
'repocheck'
end
......
......@@ -33,33 +33,33 @@ describe "Admin Projects", feature: true do
end
end
feature 'repo checks' do
scenario 'trigger repo check' do
feature 'repository checks' do
scenario 'trigger repository check' do
visit_admin_project_page
page.within('.repo-check') do
click_button 'Trigger repo check'
page.within('.repository-check') do
click_button 'Trigger repository check'
end
expect(page).to have_content('Repo check was triggered')
expect(page).to have_content('Repository check was triggered')
end
scenario 'see failed repo check' do
@project.update_column(:last_repo_check_failed, true)
scenario 'see failed repository check' do
@project.update_column(:last_repository_check_failed, true)
visit_admin_project_page
expect(page).to have_content('Last repo check failed')
expect(page).to have_content('Last repository check failed')
end
scenario 'clear repo checks', js: true do
@project.update_column(:last_repo_check_failed, true)
scenario 'clear repository checks', js: true do
@project.update_column(:last_repository_check_failed, true)
visit admin_namespaces_projects_path
page.within('.repo-check-states') do
page.within('.repository-check-states') do
click_link 'Clear all' # pop-up should be auto confirmed
end
expect(@project.reload.last_repo_check_failed).to eq(false)
expect(@project.reload.last_repository_check_failed).to eq(false)
end
end
......
require 'rails_helper'
describe RepoCheckMailer do
describe RepositoryCheckMailer do
include EmailSpec::Matchers
describe '.notify' do
......
require 'spec_helper'
describe RepoCheckWorker do
subject { RepoCheckWorker.new }
describe RepositoryCheckWorker do
subject { RepositoryCheckWorker.new }
it 'prefers projects that have never been checked' do
projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.weeks.ago)
projects[0].update_column(:last_repository_check_at, 1.month.ago)
projects[2].update_column(:last_repository_check_at, 3.weeks.ago)
expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id))
end
it 'sorts projects by last_repo_check_at' do
it 'sorts projects by last_repository_check_at' do
projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 2.weeks.ago)
projects[1].update_column(:last_repo_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.weeks.ago)
projects[0].update_column(:last_repository_check_at, 2.weeks.ago)
projects[1].update_column(:last_repository_check_at, 1.month.ago)
projects[2].update_column(:last_repository_check_at, 3.weeks.ago)
expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id))
end
it 'excludes projects that were checked recently' do
projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 2.days.ago)
projects[1].update_column(:last_repo_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.days.ago)
projects[0].update_column(:last_repository_check_at, 2.days.ago)
projects[1].update_column(:last_repository_check_at, 1.month.ago)
projects[2].update_column(:last_repository_check_at, 3.days.ago)
expect(subject.perform).to eq([projects[1].id])
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment