BigW Consortium Gitlab

project_tree_restorer.rb 6.99 KB
Newer Older
1
module Gitlab
2 3
  module ImportExport
    class ProjectTreeRestorer
4
      # Relations which cannot have both group_id and project_id at the same time
5
      RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze
6

7
      def initialize(user:, shared:, project:)
8
        @path = File.join(shared.export_path, 'project.json')
9
        @user = user
10
        @shared = shared
11
        @project = project
12
        @project_id = project.id
James Lopez committed
13
        @saved = true
14 15 16
      end

      def restore
17 18 19 20 21 22 23 24
        begin
          json = IO.read(@path)
          @tree_hash = ActiveSupport::JSON.decode(json)
        rescue => e
          Rails.logger.error("Import/Export error: #{e.message}")
          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
        end

25
        @project_members = @tree_hash.delete('project_members')
26

James Lopez committed
27 28
        ActiveRecord::Base.uncached do
          ActiveRecord::Base.no_touching do
29 30
            create_relations
          end
31
        end
32
      rescue => e
33
        @shared.error(e)
34
        false
James Lopez committed
35 36
      end

37
      def restored_project
38 39
        return @project unless @tree_hash

40
        @restored_project ||= restore_project
41 42
      end

James Lopez committed
43 44
      private

45 46 47
      def members_mapper
        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
                                                                    user: @user,
48
                                                                    project: restored_project)
James Lopez committed
49 50
      end

51 52 53 54 55
      # Loops through the tree of models defined in import_export.yml and
      # finds them in the imported JSON so they can be instantiated and saved
      # in the DB. The structure and relationships between models are guessed from
      # the configuration yaml file too.
      # Finally, it updates each attribute in the newly imported project.
56 57
      def create_relations
        default_relation_list.each do |relation|
58 59
          if relation.is_a?(Hash)
            create_sub_relations(relation, @tree_hash)
60
          elsif @tree_hash[relation.to_s].present?
James Lopez committed
61
            save_relation_hash(@tree_hash[relation.to_s], relation)
62 63
          end
        end
James Lopez committed
64

65 66
        @project.merge_requests.set_latest_merge_request_diff_ids!

James Lopez committed
67
        @saved
68
      end
69

70 71
      def save_relation_hash(relation_hash_batch, relation_key)
        relation_hash = create_relation(relation_key, relation_hash_batch)
James Lopez committed
72

James Lopez committed
73
        @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
James Lopez committed
74

James Lopez committed
75
        # Restore the project again, extra query that skips holding the AR objects in memory
James Lopez committed
76
        @restored_project = Project.find(@project_id)
77 78 79
      end

      def default_relation_list
James Lopez committed
80
        Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
81
          model.is_a?(Hash) && model[:project_members]
82
        end
83 84
      end

85
      def restore_project
86 87 88 89 90
        params = project_params

        if params[:description].present?
          params[:description_html] = nil
        end
91

92
        @project.update_columns(params)
93
        @project
94
      end
95

96 97 98
      def project_params
        @tree_hash.reject do |key, value|
          # return params that are not 1 to many or 1 to 1 relations
99
          value.respond_to?(:each) && !Project.column_names.include?(key)
100 101 102
        end
      end

103 104 105 106 107 108
      # Given a relation hash containing one or more models and its relationships,
      # loops through each model and each object from a model type and
      # and assigns its correspondent attributes hash from +tree_hash+
      # Example:
      # +relation_key+ issues, loops through the list of *issues* and for each individual
      # issue, finds any subrelations such as notes, creates them and assign them back to the hash
109 110
      #
      # Recursively calls this method if the sub-relation is a hash containing more sub-relations
James Lopez committed
111
      def create_sub_relations(relation, tree_hash, save: true)
112
        relation_key = relation.keys.first.to_s
113 114
        return if tree_hash[relation_key].blank?

115
        tree_array = [tree_hash[relation_key]].flatten
116

James Lopez committed
117
        # Avoid keeping a possible heavy object in memory once we are done with it
118
        while relation_item = tree_array.shift
James Lopez committed
119 120 121
          # The transaction at this level is less speedy than one single transaction
          # But we can't have it in the upper level or GC won't get rid of the AR objects
          # after we save the batch.
James Lopez committed
122
          Project.transaction do
James Lopez committed
123
            process_sub_relation(relation, relation_item)
124

James Lopez committed
125 126 127
            # For every subrelation that hangs from Project, save the associated records alltogether
            # This effectively batches all records per subrelation item, only keeping those in memory
            # We have to keep in mind that more batch granularity << Memory, but >> Slowness
128 129
            if save
              save_relation_hash([relation_item], relation_key)
James Lopez committed
130
              tree_hash[relation_key].delete(relation_item)
131
            end
James Lopez committed
132
          end
133
        end
134 135

        tree_hash.delete(relation_key) if save
136 137
      end

James Lopez committed
138 139 140 141 142
      def process_sub_relation(relation, relation_item)
        relation.values.flatten.each do |sub_relation|
          # We just use author to get the user ID, do not attempt to create an instance.
          next if sub_relation == :author

James Lopez committed
143
          create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash)
James Lopez committed
144 145 146 147 148 149

          relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
          relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
        end
      end

150 151 152 153 154 155 156
      def assign_relation_hash(relation_item, sub_relation)
        if sub_relation.is_a?(Hash)
          relation_hash = relation_item[sub_relation.keys.first.to_s]
          sub_relation = sub_relation.keys.first
        else
          relation_hash = relation_item[sub_relation.to_s]
        end
157

James Lopez committed
158
        [relation_hash, sub_relation]
159 160
      end

James Lopez committed
161
      def create_relation(relation, relation_hash_list)
162
        relation_array = [relation_hash_list].flatten.map do |relation_hash|
James Lopez committed
163 164
          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
                                                       relation_hash: parsed_relation_hash(relation_hash, relation.to_sym),
165
                                                       members_mapper: members_mapper,
166
                                                       user: @user,
James Lopez committed
167
                                                       project: @restored_project)
168
        end.compact
169 170

        relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
171
      end
172

173 174 175 176 177 178 179 180 181 182
      def parsed_relation_hash(relation_hash, relation_type)
        if RESTRICT_PROJECT_AND_GROUP.include?(relation_type)
          params = {}
          params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id']
          params['project_id'] = restored_project.id if relation_hash['project_id']
        else
          params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id }
        end

        relation_hash.merge(params)
183
      end
184 185 186
    end
  end
end