Remove codebase dependencies from a BG migration

lib/gitlab/background_migration/populate_untracked_uploads.rb

@@ -5,11 +5,15 @@ module Gitlab
     # This class processes a batch of rows in `untracked_files_for_uploads` by
     # adding each file to the `uploads` table if it does not exist.
     class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength
+      include PopulateUntrackedUploadsDependencies
+
       # This class is responsible for producing the attributes necessary to
       # track an uploaded file in the `uploads` table.
       class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength
         self.table_name = 'untracked_files_for_uploads'
 
+        include PopulateUntrackedUploadsDependencies
+
         # Ends with /:random_hex/:filename
         FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}
         FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/
@@ -147,11 +151,6 @@ module Gitlab
         end
       end
 
-      # This class is used to query the `uploads` table.
-      class Upload < ActiveRecord::Base
-        self.table_name = 'uploads'
-      end
-
       def perform(start_id, end_id)
         return unless migrate?
 
@@ -229,7 +228,7 @@ module Gitlab
         end
 
         ids.each do |model_type, model_ids|
-          model_class = Object.const_get(model_type)
+          model_class = self.class.const_get(model_type)
           found_ids = model_class.where(id: model_ids.uniq).pluck(:id)
           deleted_ids = ids[model_type] - found_ids
           ids[model_type] = deleted_ids

lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb

+# frozen_string_literal: true
+module Gitlab
+  module BackgroundMigration
+    module PopulateUntrackedUploadsDependencies
+      # Avoid using application code
+      class Upload < ActiveRecord::Base
+        self.table_name = 'uploads'
+      end
+
+      # Avoid using application code
+      class Appearance < ActiveRecord::Base
+        self.table_name = 'appearances'
+      end
+
+      # Avoid using application code
+      class Namespace < ActiveRecord::Base
+        self.table_name = 'namespaces'
+      end
+
+      # Avoid using application code
+      class Note < ActiveRecord::Base
+        self.table_name = 'notes'
+      end
+
+      # Avoid using application code
+      class User < ActiveRecord::Base
+        self.table_name = 'users'
+      end
+
+      # Since project Markdown upload paths don't contain the project ID, we have to find the
+      # project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons,
+      # the logic is somewhat complex, so I've mostly copied it in here.
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+
+        def self.find_by_full_path(path)
+          binary = Gitlab::Database.mysql? ? 'BINARY' : ''
+          order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)"
+          where_full_path_in(path).reorder(order_sql).take
+        end
+
+        def self.where_full_path_in(path)
+          cast_lower = Gitlab::Database.postgresql?
+
+          path = connection.quote(path)
+
+          where =
+            if cast_lower
+              "(LOWER(routes.path) = LOWER(#{path}))"
+            else
+              "(routes.path = #{path})"
+            end
+
+          joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where)
+        end
+      end
+    end
+  end
+end

spec/support/track_untracked_uploads_helpers.rb

 module TrackUntrackedUploadsHelpers
-  def uploaded_file
-    fixture_path = Rails.root.join('spec/fixtures/rails_sample.jpg')
-    fixture_file_upload(fixture_path)
+  PUBLIC_DIR = File.join(Rails.root, 'tmp', 'tests', 'public')
+  UPLOADS_DIR = File.join(PUBLIC_DIR, 'uploads')
+  SYSTEM_DIR = File.join(UPLOADS_DIR, '-', 'system')
+  UPLOAD_FILENAME = 'image.png'.freeze
+  FIXTURE_FILE_PATH = File.join(Rails.root, 'spec', 'fixtures', 'dk.png')
+  FIXTURE_CHECKSUM = 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'.freeze
+
+  def create_or_update_appearance(logo: false, header_logo: false)
+    appearance = appearances.first_or_create(title: 'foo', description: 'bar', logo: (UPLOAD_FILENAME if logo), header_logo: (UPLOAD_FILENAME if header_logo))
+
+    add_upload(appearance, 'Appearance', 'logo', 'AttachmentUploader') if logo
+    add_upload(appearance, 'Appearance', 'header_logo', 'AttachmentUploader') if header_logo
+
+    appearance
   end
 
-  def ensure_temporary_tracking_table_exists
-    Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
+  def create_group(avatar: false)
+    index = unique_index(:group)
+    group = namespaces.create(name: "group#{index}", path: "group#{index}", avatar: (UPLOAD_FILENAME if avatar))
+
+    add_upload(group, 'Group', 'avatar', 'AvatarUploader') if avatar
+
+    group
+  end
+
+  def create_note(attachment: false)
+    note = notes.create(attachment: (UPLOAD_FILENAME if attachment))
+
+    add_upload(note, 'Note', 'attachment', 'AttachmentUploader') if attachment
+
+    note
+  end
+
+  def create_project(avatar: false)
+    group = create_group
+    project = projects.create(namespace_id: group.id, path: "project#{unique_index(:project)}", avatar: (UPLOAD_FILENAME if avatar))
+    routes.create(path: "#{group.path}/#{project.path}", source_id: project.id, source_type: 'Project') # so Project.find_by_full_path works
+
+    add_upload(project, 'Project', 'avatar', 'AvatarUploader') if avatar
+
+    project
+  end
+
+  def create_user(avatar: false)
+    user = users.create(email: "foo#{unique_index(:user)}@bar.com", avatar: (UPLOAD_FILENAME if avatar), projects_limit: 100)
+
+    add_upload(user, 'User', 'avatar', 'AvatarUploader') if avatar
+
+    user
+  end
+
+  def unique_index(name = :unnamed)
+    @unique_index ||= {}
+    @unique_index[name] ||= 0
+    @unique_index[name] += 1
   end
 
-  def create_or_update_appearance(attrs)
-    a = Appearance.first_or_initialize(title: 'foo', description: 'bar')
-    a.update!(attrs)
-    a
+  def add_upload(model, model_type, attachment_type, uploader)
+    file_path = upload_file_path(model, model_type, attachment_type)
+    path_relative_to_public = file_path.sub("#{PUBLIC_DIR}/", '')
+    create_file(file_path)
+
+    uploads.create!(
+      size: 1062,
+      path: path_relative_to_public,
+      model_id: model.id,
+      model_type: model_type == 'Group' ? 'Namespace' : model_type,
+      uploader: uploader,
+      checksum: FIXTURE_CHECKSUM
+    )
+  end
+
+  def add_markdown_attachment(project)
+    project_dir = project_uploads_dir(project)
+    attachment_dir = File.join(project_dir, SecureRandom.hex)
+    attachment_file_path = File.join(attachment_dir, UPLOAD_FILENAME)
+    project_attachment_path_relative_to_project = attachment_file_path.sub("#{project_dir}/", '')
+    create_file(attachment_file_path)
+
+    uploads.create!(
+      size: 1062,
+      path: project_attachment_path_relative_to_project,
+      model_id: project.id,
+      model_type: 'Project',
+      uploader: 'FileUploader',
+      checksum: FIXTURE_CHECKSUM
+    )
+  end
+
+  def project_uploads_dir(project)
+    File.join(UPLOADS_DIR, project.full_path)
+  end
+
+  def upload_file_path(model, model_type, attachment_type)
+    dir = File.join(upload_dir(model_type.downcase, attachment_type.to_s), model.id.to_s)
+    File.join(dir, UPLOAD_FILENAME)
+  end
+
+  def upload_dir(model_type, attachment_type)
+    File.join(SYSTEM_DIR, model_type, attachment_type)
+  end
+
+  def create_file(path)
+    File.delete(path) if File.exist?(path)
+    FileUtils.mkdir_p(File.dirname(path))
+    FileUtils.cp(FIXTURE_FILE_PATH, path)
+  end
+
+  def get_uploads(model, model_type)
+    uploads.where(model_type: model_type, model_id: model.id)
+  end
+
+  def get_full_path(project)
+    routes.find_by(source_id: project.id, source_type: 'Project').path
+  end
+
+  def ensure_temporary_tracking_table_exists
+    Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
   end
 end