BigW Consortium Gitlab

manager.rb 7.44 KB
Newer Older
1 2
module Backup
  class Manager
3 4 5
    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
    FOLDERS_TO_BACKUP = %w[repositories db].freeze
    FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze
6

7
    def pack
8 9 10
      # Make sure there is a connection
      ActiveRecord::Base.connection.reconnect!

11 12
      Dir.chdir(backup_path) do
        File.open("#{backup_path}/backup_information.yml", "w+") do |file|
13
          file << backup_information.to_yaml.gsub(/^---\n/, '')
14 15 16 17
        end

        # create archive
        $progress.print "Creating backup archive: #{tar_file} ... "
18
        # Set file permissions on open to prevent chmod races.
19
        tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
20
        if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
21
          $progress.puts "done".color(:green)
22
        else
23
          puts "creating archive #{tar_file} failed".color(:red)
24 25 26
          abort 'Backup failed'
        end

27
        upload
28
      end
29 30
    end

31
    def upload
32
      $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
33 34 35

      connection_settings = Gitlab.config.backup.upload.connection
      if connection_settings.blank?
36
        $progress.puts "skipped".color(:yellow)
37 38 39
        return
      end

40
      directory = connect_to_remote_directory(connection_settings)
41

42
      if directory.files.create(key: remote_target, body: File.open(tar_file), public: false,
43
                                multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
44 45
                                encryption: Gitlab.config.backup.upload.encryption,
                                storage_class: Gitlab.config.backup.upload.storage_class)
46
        $progress.puts "done".color(:green)
47
      else
48
        puts "uploading backup to #{remote_directory} failed".color(:red)
49
        abort 'Backup failed'
50 51 52 53
      end
    end

    def cleanup
54
      $progress.print "Deleting tmp directories ... "
55

56
      backup_contents.each do |dir|
57
        next unless File.exist?(File.join(backup_path, dir))
58

59
        if FileUtils.rm_rf(File.join(backup_path, dir))
60
          $progress.puts "done".color(:green)
61
        else
62
          puts "deleting tmp directory '#{dir}' failed".color(:red)
63 64
          abort 'Backup failed'
        end
65 66 67 68 69
      end
    end

    def remove_old
      # delete backups
70
      $progress.print "Deleting old backups ... "
71 72 73 74
      keep_time = Gitlab.config.backup.keep_time.to_i

      if keep_time > 0
        removed = 0
75

76 77
        Dir.chdir(backup_path) do
          backup_file_list.each do |file|
78 79 80 81
            # For backward compatibility, there are 3 names the backups can have:
            # - 1495527122_gitlab_backup.tar
            # - 1495527068_2017_05_23_gitlab_backup.tar
            # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
82
            next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/
83 84 85 86 87 88

            timestamp = $1.to_i

            if Time.at(timestamp) < (Time.now - keep_time)
              begin
                FileUtils.rm(file)
89
                removed += 1
90 91
              rescue => e
                $progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
92
              end
93 94 95
            end
          end
        end
96

97
        $progress.puts "done. (#{removed} removed)".color(:green)
98
      else
99
        $progress.puts "skipping".color(:yellow)
100 101 102 103
      end
    end

    def unpack
104 105 106 107 108 109 110 111 112 113 114
      Dir.chdir(backup_path) do
        # check for existing backups in the backup dir
        if backup_file_list.empty?
          $progress.puts "No backups found in #{backup_path}"
          $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
          exit 1
        elsif backup_file_list.many? && ENV["BACKUP"].nil?
          $progress.puts 'Found more than one backup, please specify which one you want to restore:'
          $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
          exit 1
        end
115

116 117 118 119 120
        tar_file = if ENV['BACKUP'].present?
                     "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}"
                   else
                     backup_file_list.first
                   end
121

122 123 124 125
        unless File.exist?(tar_file)
          $progress.puts "The backup file #{tar_file} does not exist!"
          exit 1
        end
126

127
        $progress.print 'Unpacking backup ... '
128

129 130 131 132 133 134
        unless Kernel.system(*%W(tar -xf #{tar_file}))
          $progress.puts 'unpacking backup failed'.color(:red)
          exit 1
        else
          $progress.puts 'done'.color(:green)
        end
135

136 137 138 139
        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0

        # restoring mismatching backups can lead to unexpected problems
        if settings[:gitlab_version] != Gitlab::VERSION
140 141 142 143 144 145
          $progress.puts(<<~HEREDOC.color(:red))
            GitLab version mismatch:
              Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
              Please switch to the following version and try again:
              version: #{settings[:gitlab_version]}
          HEREDOC
146 147 148 149
          $progress.puts
          $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
          exit 1
        end
150 151
      end
    end
152 153

    def tar_version
154
      tar_version, _ = Gitlab::Popen.popen(%w(tar --version))
155
      tar_version.force_encoding('locale').split("\n").first
156
    end
157 158

    def skipped?(item)
Kamil Trzcinski committed
159
      settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item)
160 161 162 163
    end

    private

164 165 166 167 168 169 170 171
    def backup_path
      Gitlab.config.backup.path
    end

    def backup_file_list
      @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}")
    end

172
    def connect_to_remote_directory(connection_settings)
173 174
      # our settings use string keys, but Fog expects symbols
      connection = ::Fog::Storage.new(connection_settings.symbolize_keys)
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189

      # We only attempt to create the directory for local backups. For AWS
      # and other cloud providers, we cannot guarantee the user will have
      # permission to create the bucket.
      if connection.service == ::Fog::Storage::Local
        connection.directories.create(key: remote_directory)
      else
        connection.directories.get(remote_directory)
      end
    end

    def remote_directory
      Gitlab.config.backup.upload.remote_directory
    end

190 191 192 193 194 195 196 197
    def remote_target
      if ENV['DIRECTORY']
        File.join(ENV['DIRECTORY'], tar_file)
      else
        tar_file
      end
    end

198
    def backup_contents
199
      folders_to_backup + archives_to_backup + ["backup_information.yml"]
200 201
    end

202
    def archives_to_backup
203
      ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact
204
    end
205

206
    def folders_to_backup
207
      FOLDERS_TO_BACKUP.reject { |name| skipped?(name) }
208 209 210 211 212 213
    end

    def disabled_features
      features = []
      features << 'registry' unless Gitlab.config.registry.enabled
      features
214 215 216 217 218
    end

    def settings
      @settings ||= YAML.load_file("backup_information.yml")
    end
219 220 221 222 223 224 225 226 227 228 229 230 231 232

    def tar_file
      @tar_file ||= "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
    end

    def backup_information
      @backup_information ||= {
        db_version: ActiveRecord::Migrator.current_version.to_s,
        backup_created_at: Time.now,
        gitlab_version: Gitlab::VERSION,
        tar_version: tar_version,
        skipped: ENV["SKIP"]
      }
    end
233 234
  end
end