Merge branch '42099-port-push-mirroring-to-ce-ce-port-v-2' into 'master'

app/controllers/projects/mirrors_controller.rb

+class Projects::MirrorsController < Projects::ApplicationController
+  include RepositorySettingsRedirect
+
+  # Authorize
+  before_action :remote_mirror, only: [:update]
+  before_action :check_mirror_available!
+  before_action :authorize_admin_project!
+
+  layout "project_settings"
+
+  def show
+    redirect_to_repository_settings(project)
+  end
+
+  def update
+    if project.update_attributes(mirror_params)
+      flash[:notice] = 'Mirroring settings were successfully updated.'
+    else
+      flash[:alert] = project.errors.full_messages.join(', ').html_safe
+    end
+
+    respond_to do |format|
+      format.html { redirect_to_repository_settings(project) }
+      format.json do
+        if project.errors.present?
+          render json: project.errors, status: :unprocessable_entity
+        else
+          render json: ProjectMirrorSerializer.new.represent(project)
+        end
+      end
+    end
+  end
+
+  def update_now
+    if params[:sync_remote]
+      project.update_remote_mirrors
+      flash[:notice] = "The remote repository is being updated..."
+    end
+
+    redirect_to_repository_settings(project)
+  end
+
+  private
+
+  def remote_mirror
+    @remote_mirror = project.remote_mirrors.first_or_initialize
+  end
+
+  def check_mirror_available!
+    Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
+  end
+
+  def mirror_params_attributes
+    [
+      remote_mirrors_attributes: %i[
+        url
+        id
+        enabled
+        only_protected_branches
+      ]
+    ]
+  end
+
+  def mirror_params
+    params.require(:project).permit(mirror_params_attributes)
+  end
+end

app/controllers/projects/settings/repository_controller.rb

@@ -2,6 +2,7 @@ module Projects
   module Settings
     class RepositoryController < Projects::ApplicationController
       before_action :authorize_admin_project!
+      before_action :remote_mirror, only: [:show]
 
       def show
         render_show
@@ -25,6 +26,7 @@ module Projects
 
         define_deploy_token
         define_protected_refs
+        remote_mirror
 
         render 'show'
       end
@@ -41,6 +43,10 @@ module Projects
         load_gon_index
       end
 
+      def remote_mirror
+        @remote_mirror = project.remote_mirrors.first_or_initialize
+      end
+
       def access_levels_options
         {
           create_access_levels: levels_for_dropdown,

app/helpers/application_settings_helper.rb

@@ -250,7 +250,8 @@ module ApplicationSettingsHelper
       :version_check_enabled,
       :allow_local_requests_from_hooks_and_services,
       :enforce_terms,
-      :terms
+      :terms,
+      :mirror_available
     ]
   end
 end

app/models/application_setting.rb

@@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base
       gitaly_timeout_fast: 10,
       gitaly_timeout_medium: 30,
       gitaly_timeout_default: 55,
-      allow_local_requests_from_hooks_and_services: false
+      allow_local_requests_from_hooks_and_services: false,
+      mirror_available: true
     }
   end
 

app/models/project.rb

@@ -65,6 +65,9 @@ class Project < ActiveRecord::Base
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
 
   add_authentication_token_field :runners_token
+
+  before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) }
+
   before_save :ensure_runners_token
 
   after_save :update_project_statistics, if: :namespace_id_changed?
@@ -246,11 +249,17 @@ class Project < ActiveRecord::Base
   has_many :project_badges, class_name: 'ProjectBadge'
   has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :remote_mirrors, inverse_of: :project
+
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :import_data
   accepts_nested_attributes_for :auto_devops, update_only: true
 
+  accepts_nested_attributes_for :remote_mirrors,
+                                allow_destroy: true,
+                                reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
+
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
@@ -340,6 +349,7 @@ class Project < ActiveRecord::Base
   scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
   scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
   scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
+  scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
 
   scope :with_group_runners_enabled, -> do
     joins(:ci_cd_settings)
@@ -759,6 +769,37 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
+  def has_remote_mirror?
+    remote_mirror_available? && remote_mirrors.enabled.exists?
+  end
+
+  def updating_remote_mirror?
+    remote_mirrors.enabled.started.exists?
+  end
+
+  def update_remote_mirrors
+    return unless remote_mirror_available?
+
+    remote_mirrors.enabled.each(&:sync)
+  end
+
+  def mark_stuck_remote_mirrors_as_failed!
+    remote_mirrors.stuck.update_all(
+      update_status: :failed,
+      last_error: 'The remote mirror took to long to complete.',
+      last_update_at: Time.now
+    )
+  end
+
+  def mark_remote_mirrors_for_removal
+    remote_mirrors.each(&:mark_for_delete_if_blank_url)
+  end
+
+  def remote_mirror_available?
+    remote_mirror_available_overridden ||
+      ::Gitlab::CurrentSettings.mirror_available
+  end
+
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit

app/models/remote_mirror.rb

+class RemoteMirror < ActiveRecord::Base
+  include AfterCommitQueue
+
+  PROTECTED_BACKOFF_DELAY   = 1.minute
+  UNPROTECTED_BACKOFF_DELAY = 5.minutes
+
+  attr_encrypted :credentials,
+                 key: Gitlab::Application.secrets.db_key_base,
+                 marshal: true,
+                 encode: true,
+                 mode: :per_attribute_iv_and_salt,
+                 insecure_mode: true,
+                 algorithm: 'aes-256-cbc'
+
+  default_value_for :only_protected_branches, true
+
+  belongs_to :project, inverse_of: :remote_mirrors
+
+  validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
+  validates :url, addressable_url: true, if: :url_changed?
+
+  before_save :set_new_remote_name, if: :mirror_url_changed?
+
+  after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
+  after_save :refresh_remote, if: :mirror_url_changed?
+  after_update :reset_fields, if: :mirror_url_changed?
+
+  after_commit :remove_remote, on: :destroy
+
+  scope :enabled, -> { where(enabled: true) }
+  scope :started, -> { with_update_status(:started) }
+  scope :stuck,   -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
+
+  state_machine :update_status, initial: :none do
+    event :update_start do
+      transition [:none, :finished, :failed] => :started
+    end
+
+    event :update_finish do
+      transition started: :finished
+    end
+
+    event :update_fail do
+      transition started: :failed
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started do |remote_mirror, _|
+      Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path)
+
+      remote_mirror.update(last_update_started_at: Time.now)
+    end
+
+    after_transition started: :finished do |remote_mirror, _|
+      Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path)
+
+      timestamp = Time.now
+      remote_mirror.update_attributes!(
+        last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
+      )
+    end
+
+    after_transition started: :failed do |remote_mirror, _|
+      Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path)
+
+      remote_mirror.update(last_update_at: Time.now)
+    end
+  end
+
+  def remote_name
+    super || fallback_remote_name
+  end
+
+  def update_failed?
+    update_status == 'failed'
+  end
+
+  def update_in_progress?
+    update_status == 'started'
+  end
+
+  def update_repository(options)
+    raw.update(options)
+  end
+
+  def sync?
+    enabled?
+  end
+
+  def sync
+    return unless sync?
+
+    if recently_scheduled?
+      RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
+    else
+      RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
+    end
+  end
+
+  def enabled
+    return false unless project && super
+    return false unless project.remote_mirror_available?
+    return false unless project.repository_exists?
+    return false if project.pending_delete?
+
+    true
+  end
+  alias_method :enabled?, :enabled
+
+  def updated_since?(timestamp)
+    last_update_started_at && last_update_started_at > timestamp && !update_failed?
+  end
+
+  def mark_for_delete_if_blank_url
+    mark_for_destruction if url.blank?
+  end
+
+  def mark_as_failed(error_message)
+    update_fail
+    update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
+  end
+
+  def url=(value)
+    super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
+
+    mirror_url = Gitlab::UrlSanitizer.new(value)
+    self.credentials = mirror_url.credentials
+
+    super(mirror_url.sanitized_url)
+  end
+
+  def url
+    if super
+      Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
+    end
+  rescue
+    super
+  end
+
+  def safe_url
+    return if url.nil?
+
+    result = URI.parse(url)
+    result.password = '*****' if result.password
+    result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
+    result.to_s
+  end
+
+  private
+
+  def raw
+    @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
+  end
+
+  def fallback_remote_name
+    return unless id
+
+    "remote_mirror_#{id}"
+  end
+
+  def recently_scheduled?
+    return false unless self.last_update_started_at
+
+    self.last_update_started_at >= Time.now - backoff_delay
+  end
+
+  def backoff_delay
+    if self.only_protected_branches
+      PROTECTED_BACKOFF_DELAY
+    else
+      UNPROTECTED_BACKOFF_DELAY
+    end
+  end
+
+  def reset_fields
+    update_columns(
+      last_error: nil,
+      last_update_at: nil,
+      last_successful_update_at: nil,
+      update_status: 'finished'
+    )
+  end
+
+  def set_override_remote_mirror_available
+    enabled = read_attribute(:enabled)
+
+    project.update(remote_mirror_available_overridden: enabled)
+  end
+
+  def set_new_remote_name
+    self.remote_name = "remote_mirror_#{SecureRandom.hex}"
+  end
+
+  def refresh_remote
+    return unless project
+
+    # Before adding a new remote we have to delete the data from
+    # the previous remote name
+    prev_remote_name = remote_name_was || fallback_remote_name
+    run_after_commit do
+      project.repository.async_remove_remote(prev_remote_name)
+    end
+
+    project.repository.add_remote(remote_name, url)
+  end
+
+  def remove_remote
+    return unless project # could be pending to delete so don't need to touch the git repository
+
+    project.repository.async_remove_remote(remote_name)
+  end
+
+  def mirror_url_changed?
+    url_changed? || encrypted_credentials_changed?
+  end
+end

app/models/repository.rb

@@ -854,13 +854,27 @@ class Repository
     add_remote(remote_name, url, mirror_refmap: refmap)
     fetch_remote(remote_name, forced: forced, prune: prune)
   ensure
-    remove_remote(remote_name) if tmp_remote_name
+    async_remove_remote(remote_name) if tmp_remote_name
   end
 
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true)
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
   end
 
+  def async_remove_remote(remote_name)
+    return unless remote_name
+
+    job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name)
+
+    if job_id
+      Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.")
+    else
+      Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.")
+    end
+
+    job_id
+  end
+
   def fetch_source_branch!(source_repository, source_branch, local_ref)
     raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
   end

app/policies/project_policy.rb

@@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy
     project.merge_requests_allowing_push_to_user(user).any?
   end
 
+  with_scope :global
+  condition(:mirror_available, score: 0) do
+    ::Gitlab::CurrentSettings.current_application_settings.mirror_available
+  end
+
   # We aren't checking `:read_issue` or `:read_merge_request` in this case
   # because it could be possible for a user to see an issuable-iid
   # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be
@@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy
     enable :create_cluster
   end
 
+  rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
+
   rule { archived }.policy do
     prevent :push_code
     prevent :push_to_delete_protected_branch

app/serializers/project_mirror_entity.rb

+class ProjectMirrorEntity < Grape::Entity
+  expose :id
+
+  expose :remote_mirrors_attributes do |project|
+    next [] unless project.remote_mirrors.present?
+
+    project.remote_mirrors.map do |remote|
+      remote.as_json(only: %i[id url enabled])
+    end
+  end
+end

app/services/concerns/exclusive_lease_guard.rb

+#
+# Concern that helps with getting an exclusive lease for running a block
+# of code.
+#
+# `#try_obtain_lease` takes a block which will be run if it was able to
+# obtain  the lease. Implement `#lease_timeout` to configure the timeout
+# for the exclusive lease. Optionally override `#lease_key` to set the
+# lease key, it defaults to the class name with underscores.
+#
+module ExclusiveLeaseGuard
+  extend ActiveSupport::Concern
+
+  def try_obtain_lease
+    lease = exclusive_lease.try_obtain
+
+    unless lease
+      log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+      return
+    end
+
+    begin
+      yield lease
+    ensure
+      release_lease(lease)
+    end
+  end
+
+  def exclusive_lease
+    @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
+  end
+
+  def lease_key
+    @lease_key ||= self.class.name.underscore
+  end
+
+  def lease_timeout
+    raise NotImplementedError,
+          "#{self.class.name} does not implement #{__method__}"
+  end
+
+  def release_lease(uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+  end
+
+  def renew_lease!
+    exclusive_lease.renew
+  end
+
+  def log_error(message, extra_args = {})
+    logger.error(message)
+  end
+end

app/services/git_push_service.rb

@@ -55,6 +55,7 @@ class GitPushService < BaseService
     execute_related_hooks
     perform_housekeeping
 
+    update_remote_mirrors
     update_caches
 
     update_signatures
@@ -119,6 +120,13 @@ class GitPushService < BaseService
 
   protected
 
+  def update_remote_mirrors
+    return unless @project.has_remote_mirror?
+
+    @project.mark_stuck_remote_mirrors_as_failed!
+    @project.update_remote_mirrors
+  end
+
   def execute_related_hooks
     # Update merge requests that may be affected by this push. A new branch
     # could cause the last commit of a merge request to change.

app/services/projects/update_remote_mirror_service.rb

+module Projects
+  class UpdateRemoteMirrorService < BaseService
+    attr_reader :errors
+
+    def execute(remote_mirror)
+      @errors = []
+
+      return success unless remote_mirror.enabled?
+
+      begin
+        repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
+
+        opts = {}
+        if remote_mirror.only_protected_branches?
+          opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
+        end
+
+        remote_mirror.update_repository(opts)
+      rescue => e
+        errors << e.message.strip
+      end
+
+      if errors.present?
+        error(errors.join("\n\n"))
+      else
+        success
+      end
+    end
+  end
+end

app/views/admin/application_settings/_repository_mirrors_form.html.haml

+= form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
+  = form_errors(@application_setting)
+
+  %fieldset
+    .form-group
+      = f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2'
+      .col-sm-10
+        .checkbox
+          = f.label :mirror_available do
+            = f.check_box :mirror_available
+            Allow mirrors to be setup for projects
+        %span.help-block
+          If disabled, only admins will be able to setup mirrors in projects.
+          = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring')
+
+  = f.submit 'Save changes', class: "btn btn-success"

app/views/admin/application_settings/show.html.haml

@@ -313,3 +313,14 @@
       = _('Allow requests to the local network from hooks and services.')
   .settings-content
     = render 'outbound'
+
+%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      = _('Repository mirror settings')
+    %button.btn.js-settings-toggle{ type: 'button' }
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      = _('Configure push mirrors.')
+  .settings-content
+    = render partial: 'repository_mirrors_form'

app/views/projects/mirrors/_instructions.html.haml

+.account-well.prepend-top-default.append-bottom-default
+  %ul
+    %li
+      The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>.
+    %li
+      Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>.
+    %li
+      The update action will time out after 10 minutes. For big repositories, use a clone/push combination.
+    %li
+      The Git LFS objects will <strong>not</strong> be synced.

app/views/projects/mirrors/_push.html.haml

+- expanded = Rails.env.test?
+%section.settings.no-animate{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      Push to a remote repository
+    %button.btn.js-settings-toggle
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      Set up the remote repository that you want to update with the content of the current repository
+      every time someone pushes to it.
+      = link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
+  .settings-content
+    = form_for @project, url: project_mirror_path(@project) do |f|
+      %div
+        = form_errors(@project)
+      = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
+      - if @remote_mirror.last_error.present?
+        .panel.panel-danger
+          .panel-heading
+            - if @remote_mirror.last_update_at
+              The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
+            - else
+              The remote repository failed to update.
+
+            - if @remote_mirror.last_successful_update_at
+              Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+          .panel-body
+            %pre
+              :preserve
+                #{h(@remote_mirror.last_error.strip)}
+      = f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
+        .form-group
+          = rm_form.check_box :enabled, class: "pull-left"
+          .prepend-left-20
+            = rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
+            %p.light.append-bottom-0
+              Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
+        .form-group.has-feedback
+          = rm_form.label :url, "Git repository URL", class: "label-light"
+          = rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
+
+          = render "projects/mirrors/instructions"
+
+        .form-group
+          = rm_form.check_box :only_protected_branches, class: 'pull-left'
+          .prepend-left-20
+            = rm_form.label :only_protected_branches, class: 'label-light'
+            = link_to icon('question-circle'), help_page_path('user/project/protected_branches')
+
+        = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'

app/views/projects/mirrors/_show.html.haml

+- if can?(current_user, :admin_remote_mirror, @project)
+  = render 'projects/mirrors/push'
+

app/views/projects/settings/repository/show.html.haml

@@ -2,6 +2,8 @@
 - page_title "Repository"
 - @content_class = "limit-container-width" unless fluid_layout
 
+= render "projects/mirrors/show"
+
 -# Protected branches & tags use a lot of nested partials.
 -# The shared parts of the views can be found in the `shared` directory.
 -# Those are used throughout the actual views. These `shared` views are then

app/views/shared/_remote_mirror_update_button.html.haml

+- if @project.has_remote_mirror?
+  .append-bottom-default
+    - if remote_mirror.update_in_progress?
+      %span.btn.disabled
+        = icon("refresh spin")
+        Updating…
+    - else
+      = link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do
+        = icon("refresh")
+        Update Now
+    - if @remote_mirror.last_successful_update_at
+      %p.inline.prepend-left-10
+        Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.

app/workers/all_queues.yml

@@ -107,9 +107,11 @@
 - rebase
 - repository_fork
 - repository_import
+- repository_remove_remote
 - storage_migrator
 - system_hook_push
 - update_merge_requests
 - update_user_activity
 - upload_checksum
 - web_hook
+- repository_update_remote_mirror

app/workers/repository_remove_remote_worker.rb

+class RepositoryRemoveRemoteWorker
+  include ApplicationWorker
+  include ExclusiveLeaseGuard
+
+  LEASE_TIMEOUT = 1.hour
+
+  attr_reader :project, :remote_name
+
+  def perform(project_id, remote_name)
+    @remote_name = remote_name
+    @project = Project.find_by_id(project_id)
+
+    return unless @project
+
+    logger.info("Removing remote #{remote_name} from project #{project.id}")
+
+    try_obtain_lease do
+      remove_remote = @project.repository.remove_remote(remote_name)
+
+      if remove_remote
+        logger.info("Remote #{remote_name} was successfully removed from project #{project.id}")
+      else
+        logger.error("Could not remove remote #{remote_name} from project #{project.id}")
+      end
+    end
+  end
+
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+
+  def lease_key
+    "remove_remote_#{project.id}_#{remote_name}"
+  end
+end

app/workers/repository_update_remote_mirror_worker.rb

+class RepositoryUpdateRemoteMirrorWorker
+  UpdateAlreadyInProgressError = Class.new(StandardError)
+  UpdateError = Class.new(StandardError)
+
+  include ApplicationWorker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options retry: 3, dead: false
+
+  sidekiq_retry_in { |count| 30 * count }
+
+  sidekiq_retries_exhausted do |msg, _|
+    Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
+  end
+
+  def perform(remote_mirror_id, scheduled_time)
+    remote_mirror = RemoteMirror.find(remote_mirror_id)
+    return if remote_mirror.updated_since?(scheduled_time)
+
+    raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
+
+    remote_mirror.update_start
+
+    project = remote_mirror.project
+    current_user = project.creator
+    result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
+    raise UpdateError, result[:message] if result[:status] == :error
+
+    remote_mirror.update_finish
+  rescue UpdateAlreadyInProgressError
+    raise
+  rescue UpdateError => ex
+    fail_remote_mirror(remote_mirror, ex.message)
+    raise
+  rescue => ex
+    return unless remote_mirror
+
+    fail_remote_mirror(remote_mirror, ex.message)
+    raise UpdateError, "#{ex.class}: #{ex.message}"
+  end
+
+  private
+
+  def fail_remote_mirror(remote_mirror, message)
+    remote_mirror.mark_as_failed(message)
+
+    Rails.logger.error(message)
+  end
+end

changelogs/unreleased/42099-port-push-mirroring-to-ce-ce-port-v-2.yml

+---
+title: Adds push mirrors to GitLab Community Edition
+merge_request: 18715
+author:
+type: changed

config/routes/project.rb

@@ -174,6 +174,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
+      resource :mirror, only: [:show, :update] do
+        member do
+          post :update_now
+        end
+      end
+
       resources :pipelines, only: [:index, :new, :create, :show] do
         collection do
           resource :pipelines_settings, path: 'settings', only: [:show, :update]

config/sidekiq_queues.yml

@@ -73,3 +73,6 @@
   - [object_storage, 1]
   - [plugin, 1]
   - [pipeline_background, 1]
+  - [repository_update_remote_mirror, 1]
+  - [repository_remove_remote, 1]
+ 

db/migrate/20180503131624_create_remote_mirrors.rb

+class CreateRemoteMirrors < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    return if table_exists?(:remote_mirrors)
+
+    create_table :remote_mirrors do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.string :url
+      t.boolean :enabled, default: true
+      t.string :update_status
+      t.datetime :last_update_at
+      t.datetime :last_successful_update_at
+      t.datetime :last_update_started_at
+      t.string :last_error
+      t.boolean :only_protected_branches, default: false, null: false
+      t.string :remote_name
+      t.text :encrypted_credentials
+      t.string :encrypted_credentials_iv
+      t.string :encrypted_credentials_salt
+
+      t.timestamps null: false
+    end
+  end
+
+  def down
+    drop_table(:remote_mirrors) if table_exists?(:remote_mirrors)
+  end
+end

db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb

+class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden)
+  end
+
+  def down
+    remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden)
+  end
+end

db/migrate/20180503193542_add_indexes_to_remote_mirror.rb

+class AddIndexesToRemoteMirror < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at)
+  end
+
+  def down
+    remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at
+  end
+end

db/migrate/20180503193953_add_mirror_available_to_application_settings.rb

+class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available)
+  end
+
+  def down
+    remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available)
+  end
+end

db/schema.rb

@@ -165,6 +165,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
     t.boolean "pages_domain_verification_enabled", default: true, null: false
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
     t.boolean "enforce_terms", default: false
+    t.boolean "mirror_available", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -1602,6 +1603,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
     t.boolean "merge_requests_rebase_enabled", default: false, null: false
     t.integer "jobs_cache_index"
     t.boolean "pages_https_only", default: true
+    t.boolean "remote_mirror_available_overridden"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1707,6 +1709,27 @@ ActiveRecord::Schema.define(version: 20180503200320) do
   add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
   add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree
 
+  create_table "remote_mirrors", force: :cascade do |t|
+    t.integer "project_id"
+    t.string "url"
+    t.boolean "enabled", default: true
+    t.string "update_status"
+    t.datetime "last_update_at"
+    t.datetime "last_successful_update_at"
+    t.datetime "last_update_started_at"
+    t.string "last_error"
+    t.boolean "only_protected_branches", default: false, null: false
+    t.string "remote_name"
+    t.text "encrypted_credentials"
+    t.string "encrypted_credentials_iv"
+    t.string "encrypted_credentials_salt"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
+  add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
+
   create_table "routes", force: :cascade do |t|
     t.integer "source_id", null: false
     t.string "source_type", null: false
@@ -2243,6 +2266,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
   add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
   add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
   add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
+  add_foreign_key "remote_mirrors", "projects", on_delete: :cascade
   add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade

doc/workflow/repository_mirroring.md

+# Repository mirroring
+
+Repository Mirroring is a way to mirror repositories from external sources.
+It can be used to mirror all branches, tags, and commits that you have
+in your repository.
+
+Your mirror at GitLab will be updated automatically. You can
+also manually trigger an update at most once every 5 minutes.
+
+## Overview
+
+Repository mirroring is very useful when, for some reason, you must use a
+project from another source.
+
+There are two kinds of repository mirroring features supported by GitLab:
+**push** and **pull**, the latter being only available in GitLab Enterprise Edition.
+The **push** method mirrors the repository in GitLab to another location.
+
+Once the mirror repository is updated, all new branches,
+tags, and commits will be visible in the project's activity feed.
+Users with at least [developer access][perms] to the project can also force an
+immediate update with the click of a button. This button will not be available if
+the mirror is already being updated or 5 minutes still haven't passed since its last update.
+
+A few things/limitations to consider:
+
+- The repository must be accessible over `http://`, `https://`, `ssh://` or `git://`.
+- If your HTTP repository is not publicly accessible, add authentication
+  information to the URL, like: `https://username@gitlab.company.com/group/project.git`.
+  In some cases, you might need to use a personal access token instead of a
+  password, e.g., you want to mirror to GitHub and have 2FA enabled.
+- The import will time out after 15 minutes. For repositories that take longer
+  use a clone/push combination.
+- The Git LFS objects will not be synced. You'll need to push/pull them
+  manually.
+
+## Use-case
+
+- You have old projects in another source that you don't use actively anymore,
+  but don't want to remove for archiving purposes. In that case, you can create
+  a push mirror so that your active GitLab repository can push its changes to the
+  old location.
+
+## Pushing to a remote repository **[STARTER]**
+
+>[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in
+GitLab Enterprise Edition 8.7. [Moved to GitLab Community Edition][ce-18715] in 10.8.
+
+For an existing project, you can set up push mirror from your project's
+**Settings ➔ Repository** and searching for the "Push to a remote repository"
+section. Check the "Remote mirror repository" box and fill in the Git URL of
+the repository to push to. Click **Save changes** for the changes to take
+effect.
+
+![Push settings](repository_mirroring/repository_mirroring_push_settings.png)
+
+When push mirroring is enabled, you are advised not to push commits directly
+to the mirrored repository to prevent the mirror diverging.
+All changes will end up in the mirrored repository whenever commits
+are pushed to GitLab, or when a [forced update](#forcing-an-update) is
+initiated.
+
+Pushes into GitLab are automatically pushed to the remote mirror at least once
+every 5 minutes after they are received or once every minute if **push only
+protected branches** is enabled.
+
+In case of a diverged branch, you will see an error indicated at the **Mirror
+repository** settings.
+
+![Diverged branch](
+repository_mirroring/repository_mirroring_diverged_branch_push.png)
+
+### Push only protected branches
+
+>[Introduced][ee-3350] in GitLab Enterprise Edition 10.3. [Moved to GitLab Community Edition][ce-18715] in 10.8.
+
+You can choose to only push your protected branches from GitLab to your remote repository.
+
+To use this option go to your project's repository settings page under push mirror.
+
+## Setting up a push mirror from GitLab to GitHub
+
+To set up a mirror from GitLab to GitHub, you need to follow these steps:
+
+1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked:
+    
+    ![edit personal access token GitHub](repository_mirroring/repository_mirroring_github_edit_personal_access_token.png)
+
+1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`:
+
+    ![push to remote repo](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png)
+
+1. Save
+1. And either wait or trigger the "Update Now" button:
+
+    ![update now](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png)
+    
+## Forcing an update
+
+While mirrors are scheduled to update automatically, you can always force an update
+by using the **Update now** button which is exposed in various places:
+
+- in the commits page
+- in the branches page
+- in the tags page
+- in the **Mirror repository** settings page
+
+[ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350
+[ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715
+[perms]: ../user/permissions.md
+

lib/gitlab/import_export/import_export.yml

@@ -106,6 +106,7 @@ excluded_attributes:
     - :last_repository_updated_at
     - :last_repository_check_at
     - :storage_version
+    - :remote_mirror_available_overridden
     - :description_html
   snippets:
     - :expired_at

lib/gitlab/usage_data.rb

@@ -71,6 +71,7 @@ module Gitlab
             projects_imported_from_github: Project.where(import_type: 'github').count,
             protected_branches: ProtectedBranch.count,
             releases: Release.count,
+            remote_mirrors: RemoteMirror.count,
             snippets: Snippet.count,
             todos: Todo.count,
             uploads: Upload.count,

spec/controllers/projects/mirrors_controller_spec.rb

+require 'spec_helper'
+
+describe Projects::MirrorsController do
+  include ReactiveCachingHelpers
+
+  describe 'setting up a remote mirror' do
+    set(:project) { create(:project, :repository) }
+
+    context 'when the current project is not a mirror' do
+      it 'allows to create a remote mirror' do
+        sign_in(project.owner)
+
+        expect do
+          do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
+        end.to change { RemoteMirror.count }.to(1)
+      end
+    end
+  end
+
+  describe '#update' do
+    let(:project) { create(:project, :repository, :remote_mirror) }
+
+    before do
+      sign_in(project.owner)
+    end
+
+    around do |example|
+      Sidekiq::Testing.fake! { example.run }
+    end
+
+    context 'With valid URL for a push' do
+      let(:remote_mirror_attributes) do
+        { "0" => { "enabled" => "0", url: 'https://updated.example.com' } }
+      end
+
+      it 'processes a successful update' do
+        do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:notice]).to match(/successfully updated/)
+      end
+
+      it 'should create a RemoteMirror object' do
+        expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1)
+      end
+    end
+
+    context 'With invalid URL for a push' do
+      let(:remote_mirror_attributes) do
+        { "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } }
+      end
+
+      it 'processes an unsuccessful update' do
+        do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:alert]).to match(/must be a valid URL/)
+      end
+
+      it 'should not create a RemoteMirror object' do
+        expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.not_to change(RemoteMirror, :count)
+      end
+    end
+  end
+
+  def do_put(project, options, extra_attrs = {})
+    attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
+    attrs[:project] = options
+
+    put :update, attrs
+  end
+end

spec/factories/projects.rb

@@ -183,6 +183,17 @@ FactoryBot.define do
       end
     end
 
+    trait :remote_mirror do
+      transient do
+        remote_name "remote_mirror_#{SecureRandom.hex}"
+        url "http://foo.com"
+        enabled true
+      end
+      after(:create) do |project, evaluator|
+        project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled)
+      end
+    end
+
     trait :stubbed_repository do
       after(:build) do |project|
         allow(project).to receive(:empty_repo?).and_return(false)

spec/factories/remote_mirrors.rb

+FactoryBot.define do
+  factory :remote_mirror, class: 'RemoteMirror' do
+    association :project, :repository
+    url "http://foo:bar@test.com"
+  end
+end

spec/features/projects/remote_mirror_spec.rb

+require 'spec_helper'
+
+feature 'Project remote mirror', :feature do
+  let(:project) { create(:project, :repository, :remote_mirror) }
+  let(:remote_mirror) { project.remote_mirrors.first }
+  let(:user) { create(:user) }
+
+  describe 'On a project', :js do
+    before do
+      project.add_master(user)
+      sign_in user
+    end
+
+    context 'when last_error is present but last_update_at is not' do
+      it 'renders error message without timstamp' do
+        remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: nil)
+
+        visit project_mirror_path(project)
+
+        expect(page).to have_content('The remote repository failed to update.')
+      end
+    end
+
+    context 'when last_error and last_update_at are present' do
+      it 'renders error message with timestamp' do
+        remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: Time.now - 5.minutes)
+
+        visit project_mirror_path(project)
+
+        expect(page).to have_content('The remote repository failed to update 5 minutes ago.')
+      end
+    end
+  end
+end

spec/features/projects/settings/repository_settings_spec.rb

@@ -115,5 +115,20 @@ describe 'Projects > Settings > Repository settings' do
         expect(page).to have_content('Your new project deploy token has been created')
       end
     end
+
+    context 'remote mirror settings' do
+      let(:user2) { create(:user) }
+
+      before do
+        project.add_master(user2)
+
+        visit project_settings_repository_path(project)
+      end
+
+      it 'shows push mirror settings' do
+        expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
+        expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
+      end
+    end
   end
 end

spec/lib/gitlab/import_export/all_models.yml

@@ -268,6 +268,7 @@ project:
 - pages_domains
 - authorized_users
 - project_authorizations
+- remote_mirrors
 - route
 - redirect_routes
 - statistics

spec/lib/gitlab/usage_data_spec.rb

@@ -96,6 +96,7 @@ describe Gitlab::UsageData do
         pages_domains
         protected_branches
         releases
+        remote_mirrors
         snippets
         todos
         uploads

spec/models/project_spec.rb

@@ -1852,6 +1852,85 @@ describe Project do
     it { expect(project.gitea_import?).to be true }
   end
 
+  describe '#has_remote_mirror?' do
+    let(:project) { create(:project, :remote_mirror, :import_started) }
+    subject { project.has_remote_mirror? }
+
+    before do
+      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+    end
+
+    it 'returns true when a remote mirror is enabled' do
+      is_expected.to be_truthy
+    end
+
+    it 'returns false when remote mirror is disabled' do
+      project.remote_mirrors.first.update_attributes(enabled: false)
+
+      is_expected.to be_falsy
+    end
+  end
+
+  describe '#update_remote_mirrors' do
+    let(:project) { create(:project, :remote_mirror, :import_started) }
+    delegate :update_remote_mirrors, to: :project
+
+    before do
+      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+    end
+
+    it 'syncs enabled remote mirror' do
+      expect_any_instance_of(RemoteMirror).to receive(:sync)
+
+      update_remote_mirrors
+    end
+
+    # TODO: study if remote_mirror_available_overridden is still a necessary attribute considering that
+    # it is no longer under any license
+    it 'does nothing when remote mirror is disabled globally and not overridden' do
+      stub_application_setting(mirror_available: false)
+      project.remote_mirror_available_overridden = false
+
+      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+      update_remote_mirrors
+    end
+
+    it 'does not sync disabled remote mirrors' do
+      project.remote_mirrors.first.update_attributes(enabled: false)
+
+      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+      update_remote_mirrors
+    end
+  end
+
+  describe '#remote_mirror_available?' do
+    let(:project) { create(:project) }
+
+    context 'when remote mirror global setting is enabled' do
+      it 'returns true' do
+        expect(project.remote_mirror_available?).to be(true)
+      end
+    end
+
+    context 'when remote mirror global setting is disabled' do
+      before do
+        stub_application_setting(mirror_available: false)
+      end
+
+      it 'returns true when overridden' do
+        project.remote_mirror_available_overridden = true
+
+        expect(project.remote_mirror_available?).to be(true)
+      end
+
+      it 'returns false when not overridden' do
+        expect(project.remote_mirror_available?).to be(false)
+      end
+    end
+  end
+
   describe '#ancestors_upto', :nested_groups do
     let(:parent) { create(:group) }
     let(:child) { create(:group, parent: parent) }

spec/models/remote_mirror_spec.rb

+require 'rails_helper'
+
+describe RemoteMirror do
+  describe 'URL validation' do
+    context 'with a valid URL' do
+      it 'should be valid' do
+        remote_mirror = build(:remote_mirror)
+        expect(remote_mirror).to be_valid
+      end
+    end
+
+    context 'with an invalid URL' do
+      it 'should not be valid' do
+        remote_mirror = build(:remote_mirror, url: 'ftp://invalid.invalid')
+        expect(remote_mirror).not_to be_valid
+        expect(remote_mirror.errors[:url].size).to eq(2)
+      end
+    end
+  end
+
+  describe 'encrypting credentials' do
+    context 'when setting URL for a first time' do
+      it 'stores the URL without credentials' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(mirror.read_attribute(:url)).to eq('http://test.com')
+      end
+
+      it 'stores the credentials on a separate field' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
+      end
+
+      it 'handles credentials with large content' do
+        mirror = create_mirror(url: 'http://bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif:9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75@test.com')
+
+        expect(mirror.credentials).to eq({
+          user: 'bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif',
+          password: '9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75'
+        })
+      end
+    end
+
+    context 'when updating the URL' do
+      it 'allows a new URL without credentials' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        mirror.update_attribute(:url, 'http://test.com')
+
+        expect(mirror.url).to eq('http://test.com')
+        expect(mirror.credentials).to eq({ user: nil, password: nil })
+      end
+
+      it 'allows a new URL with credentials' do
+        mirror = create_mirror(url: 'http://test.com')
+
+        mirror.update_attribute(:url, 'http://foo:bar@test.com')
+
+        expect(mirror.url).to eq('http://foo:bar@test.com')
+        expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
+      end
+
+      it 'updates the remote config if credentials changed' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+        repo = mirror.project.repository
+
+        mirror.update_attribute(:url, 'http://foo:baz@test.com')
+
+        config = repo.raw_repository.rugged.config
+        expect(config["remote.#{mirror.remote_name}.url"]).to eq('http://foo:baz@test.com')
+      end
+
+      it 'removes previous remote' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
+
+        mirror.update_attributes(url: 'http://test.com')
+      end
+    end
+  end
+
+  describe '#remote_name' do
+    context 'when remote name is persisted in the database' do
+      it 'returns remote name with random value' do
+        allow(SecureRandom).to receive(:hex).and_return('secret')
+
+        remote_mirror = create(:remote_mirror)
+
+        expect(remote_mirror.remote_name).to eq("remote_mirror_secret")
+      end
+    end
+
+    context 'when remote name is not persisted in the database' do
+      it 'returns remote name with remote mirror id' do
+        remote_mirror = create(:remote_mirror)
+        remote_mirror.remote_name = nil
+
+        expect(remote_mirror.remote_name).to eq("remote_mirror_#{remote_mirror.id}")
+      end
+    end
+
+    context 'when remote is not persisted in the database' do
+      it 'returns nil' do
+        remote_mirror = build(:remote_mirror, remote_name: nil)
+
+        expect(remote_mirror.remote_name).to be_nil
+      end
+    end
+  end
+
+  describe '#safe_url' do
+    context 'when URL contains credentials' do
+      it 'masks the credentials' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(mirror.safe_url).to eq('http://*****:*****@test.com')
+      end
+    end
+
+    context 'when URL does not contain credentials' do
+      it 'shows the full URL' do
+        mirror = create_mirror(url: 'http://test.com')
+
+        expect(mirror.safe_url).to eq('http://test.com')
+      end
+    end
+  end
+
+  context 'when remote mirror gets destroyed' do
+    it 'removes remote' do
+      mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+      expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
+
+      mirror.destroy!
+    end
+  end
+
+  context 'stuck mirrors' do
+    it 'includes mirrors stuck in started with no last_update_at set' do
+      mirror = create_mirror(url: 'http://cantbeblank',
+                             update_status: 'started',
+                             last_update_at: nil,
+                             updated_at: 25.hours.ago)
+
+      expect(described_class.stuck.last).to eq(mirror)
+    end
+  end
+
+  context '#sync' do
+    let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+
+    around do |example|
+      Timecop.freeze { example.run }
+    end
+
+    context 'with remote mirroring disabled' do
+      it 'returns nil' do
+        remote_mirror.update_attributes(enabled: false)
+
+        expect(remote_mirror.sync).to be_nil
+      end
+    end
+
+    context 'with remote mirroring enabled' do
+      context 'with only protected branches enabled' do
+        context 'when it did not update in the last minute' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+
+        context 'when it did update in the last minute' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next minute' do
+            remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::PROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+      end
+
+      context 'with only protected branches disabled' do
+        before do
+          remote_mirror.only_protected_branches = false
+        end
+
+        context 'when it did not update in the last 5 minutes' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+
+        context 'when it did update within the last 5 minutes' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next 5 minutes' do
+            remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::UNPROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+      end
+    end
+  end
+
+  context '#updated_since?' do
+    let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+    let(:timestamp) { Time.now - 5.minutes }
+
+    around do |example|
+      Timecop.freeze { example.run }
+    end
+
+    before do
+      remote_mirror.update_attributes(last_update_started_at: Time.now)
+    end
+
+    context 'when remote mirror does not have status failed' do
+      it 'returns true when last update started after the timestamp' do
+        expect(remote_mirror.updated_since?(timestamp)).to be true
+      end
+
+      it 'returns false when last update started before the timestamp' do
+        expect(remote_mirror.updated_since?(Time.now + 5.minutes)).to be false
+      end
+    end
+
+    context 'when remote mirror has status failed' do
+      it 'returns false when last update started after the timestamp' do
+        remote_mirror.update_attributes(update_status: 'failed')
+
+        expect(remote_mirror.updated_since?(timestamp)).to be false
+      end
+    end
+  end
+
+  context 'no project' do
+    it 'includes mirror with a project in pending_delete' do
+      mirror = create_mirror(url: 'http://cantbeblank',
+                             update_status: 'finished',
+                             enabled: true,
+                             last_update_at: nil,
+                             updated_at: 25.hours.ago)
+      project = mirror.project
+      project.pending_delete = true
+      project.save
+      mirror.reload
+
+      expect(mirror.sync).to be_nil
+      expect(mirror.valid?).to be_truthy
+      expect(mirror.update_status).to eq('finished')
+    end
+  end
+
+  def create_mirror(params)
+    project = FactoryBot.create(:project, :repository)
+    project.remote_mirrors.create!(params)
+  end
+end

spec/models/repository_spec.rb

@@ -758,6 +758,38 @@ describe Repository do
     end
   end
 
+  describe '#async_remove_remote' do
+    before do
+      masterrev = repository.find_branch('master').dereferenced_target
+      create_remote_branch('joe', 'remote_branch', masterrev)
+    end
+
+    context 'when worker is scheduled successfully' do
+      before do
+        masterrev = repository.find_branch('master').dereferenced_target
+        create_remote_branch('remote_name', 'remote_branch', masterrev)
+
+        allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return('1234')
+      end
+
+      it 'returns job_id' do
+        expect(repository.async_remove_remote('joe')).to eq('1234')
+      end
+    end
+
+    context 'when worker does not schedule successfully' do
+      before do
+        allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return(nil)
+      end
+
+      it 'returns nil' do
+        expect(Rails.logger).to receive(:info).with("Remove remote job failed to create for #{project.id} with remote name joe.")
+
+        expect(repository.async_remove_remote('joe')).to be_nil
+      end
+    end
+  end
+
   describe '#fetch_ref' do
     let(:broken_repository) { create(:project, :broken_storage).repository }
 
@@ -2338,6 +2370,11 @@ describe Repository do
     end
   end
 
+  def create_remote_branch(remote_name, branch_name, target)
+    rugged = repository.rugged
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
+  end
+
   describe '#ancestor?' do
     let(:commit) { repository.commit }
     let(:ancestor) { commit.parents.first }

spec/services/git_push_service_spec.rb

@@ -14,6 +14,72 @@ describe GitPushService, services: true do
     project.add_master(user)
   end
 
+  describe 'with remote mirrors' do
+    let(:project)  { create(:project, :repository, :remote_mirror) }
+
+    subject do
+      described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref)
+    end
+
+    context 'when remote mirror feature is enabled' do
+      it 'fails stuck remote mirrors' do
+        allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
+        expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+        subject.execute
+      end
+
+      it 'updates remote mirrors' do
+        expect(project).to receive(:update_remote_mirrors)
+
+        subject.execute
+      end
+    end
+
+    context 'when remote mirror feature is disabled' do
+      before do
+        stub_application_setting(mirror_available: false)
+      end
+
+      context 'with remote mirrors global setting overridden' do
+        before do
+          project.remote_mirror_available_overridden = true
+        end
+
+        it 'fails stuck remote mirrors' do
+          allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
+          expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+          subject.execute
+        end
+
+        it 'updates remote mirrors' do
+          expect(project).to receive(:update_remote_mirrors)
+
+          subject.execute
+        end
+      end
+
+      context 'without remote mirrors global setting overridden' do
+        before do
+          project.remote_mirror_available_overridden = false
+        end
+
+        it 'does not fails stuck remote mirrors' do
+          expect(project).not_to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+          subject.execute
+        end
+
+        it 'does not updates remote mirrors' do
+          expect(project).not_to receive(:update_remote_mirrors)
+
+          subject.execute
+        end
+      end
+    end
+  end
+
   describe 'Push branches' do
     subject do
       execute_service(project, user, oldrev, newrev, ref)

spec/services/projects/destroy_service_spec.rb

@@ -65,6 +65,19 @@ describe Projects::DestroyService do
       Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
     end
 
+    context 'when has remote mirrors' do
+      let!(:project) do
+        create(:project, :repository, namespace: user.namespace).tap do |project|
+          project.remote_mirrors.create(url: 'http://test.com')
+        end
+      end
+      let!(:async) { true }
+
+      it 'destroys them' do
+        expect(RemoteMirror.count).to eq(0)
+      end
+    end
+
     it_behaves_like 'deleting the project'
 
     it 'invalidates personal_project_count cache' do

spec/workers/repository_remove_remote_worker_spec.rb

+require 'rails_helper'
+
+describe RepositoryRemoveRemoteWorker do
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    let(:remote_name) { 'joe'}
+    let!(:project) { create(:project, :repository) }
+
+    context 'when it cannot obtain lease' do
+      it 'logs error' do
+        allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
+
+        expect_any_instance_of(Repository).not_to receive(:remove_remote)
+        expect(worker).to receive(:log_error).with('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+
+        worker.perform(project.id, remote_name)
+      end
+    end
+
+    context 'when it gets the lease' do
+      before do
+        allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
+      end
+
+      context 'when project does not exist' do
+        it 'returns nil' do
+          expect(worker.perform(-1, 'remote_name')).to be_nil
+        end
+      end
+
+      context 'when project exists' do
+        it 'removes remote from repository' do
+          masterrev = project.repository.find_branch('master').dereferenced_target
+
+          create_remote_branch(remote_name, 'remote_branch', masterrev)
+
+          expect_any_instance_of(Repository).to receive(:remove_remote).with(remote_name).and_call_original
+
+          worker.perform(project.id, remote_name)
+        end
+      end
+    end
+  end
+
+  def create_remote_branch(remote_name, branch_name, target)
+    rugged = project.repository.rugged
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
+  end
+end

spec/workers/repository_update_remote_mirror_worker_spec.rb

+require 'rails_helper'
+
+describe RepositoryUpdateRemoteMirrorWorker do
+  subject { described_class.new }
+
+  let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+  let(:scheduled_time) { Time.now - 5.minutes }
+
+  around do |example|
+    Timecop.freeze(Time.now) { example.run }
+  end
+
+  describe '#perform' do
+    context 'with status none' do
+      before do
+        remote_mirror.update_attributes(update_status: 'none')
+      end
+
+      it 'sets status as finished when update remote mirror service executes successfully' do
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
+
+        expect { subject.perform(remote_mirror.id, Time.now) }.to change { remote_mirror.reload.update_status }.to('finished')
+      end
+
+      it 'sets status as failed when update remote mirror service executes with errors' do
+        error_message = 'fail!'
+
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :error, message: error_message)
+        expect do
+          subject.perform(remote_mirror.id, Time.now)
+        end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError, error_message)
+
+        expect(remote_mirror.reload.update_status).to eq('failed')
+      end
+
+      it 'does nothing if last_update_started_at is higher than the time the job was scheduled in' do
+        remote_mirror.update_attributes(last_update_started_at: Time.now)
+
+        expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(true)
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).not_to receive(:execute).with(remote_mirror)
+
+        expect(subject.perform(remote_mirror.id, scheduled_time)).to be_nil
+      end
+    end
+
+    context 'with unexpected error' do
+      it 'marks mirror as failed' do
+        allow_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_raise(RuntimeError)
+
+        expect do
+          subject.perform(remote_mirror.id, Time.now)
+        end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError)
+        expect(remote_mirror.reload.update_status).to eq('failed')
+      end
+    end
+
+    context 'with another worker already running' do
+      before do
+        remote_mirror.update_attributes(update_status: 'started')
+      end
+
+      it 'raises RemoteMirrorUpdateAlreadyInProgressError' do
+        expect do
+          subject.perform(remote_mirror.id, Time.now)
+        end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateAlreadyInProgressError)
+      end
+    end
+
+    context 'with status failed' do
+      before do
+        remote_mirror.update_attributes(update_status: 'failed')
+      end
+
+      it 'sets status as finished if last_update_started_at is higher than the time the job was scheduled in' do
+        remote_mirror.update_attributes(last_update_started_at: Time.now)
+
+        expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(false)
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
+
+        expect { subject.perform(remote_mirror.id, scheduled_time) }.to change { remote_mirror.reload.update_status }.to('finished')
+      end
+    end
+  end
+end