BigW Consortium Gitlab

ee_compat_check.rb 12.1 KB
Newer Older
1
# rubocop: disable Rails/Output
2 3 4
module Gitlab
  # Checks if a set of migrations requires downtime or not.
  class EeCompatCheck
5 6
    DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
    EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
7 8
    CHECK_DIR = Rails.root.join('ee_compat_check')
    IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
9 10 11 12 13 14 15 16 17 18
    PLEASE_READ_THIS_BANNER = %Q{
      ============================================================
      ===================== PLEASE READ THIS =====================
      ============================================================
    }.freeze
    THANKS_FOR_READING_BANNER = %Q{
      ============================================================
      ==================== THANKS FOR READING ====================
      ============================================================\n
    }.freeze
19

20 21
    attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found
    attr_reader :job_id, :failed_files
22

23
    def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil)
24
      @ee_repo_dir = CHECK_DIR.join('ee-repo')
25
      @patches_dir = CHECK_DIR.join('patches')
26
      @ce_branch = branch
27 28 29
      @ce_project_url = ce_project_url
      @ce_repo_url = "#{ce_project_url}.git"
      @job_id = job_id
30 31 32
    end

    def check
33
      ensure_patches_dir
34 35
      generate_patch(ce_branch, ce_patch_full_path)

36 37 38
      ensure_ee_repo
      Dir.chdir(ee_repo_dir) do
        step("In the #{ee_repo_dir} directory")
39 40 41

        status = catch(:halt_check) do
          ce_branch_compat_check!
42
          delete_ee_branches_locally!
43 44 45 46
          ee_branch_presence_check!
          ee_branch_compat_check!
        end

47
        delete_ee_branches_locally!
48 49 50 51 52 53 54 55 56 57 58 59

        if status.nil?
          true
        else
          false
        end
      end
    end

    private

    def ensure_ee_repo
60 61
      if Dir.exist?(ee_repo_dir)
        step("#{ee_repo_dir} already exists")
62
      else
63
        step(
64 65
          "Cloning #{EE_REPO_URL} into #{ee_repo_dir}",
          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}]
66
        )
67 68 69
      end
    end

70 71 72 73 74 75 76
    def ensure_patches_dir
      FileUtils.mkdir_p(patches_dir)
    end

    def generate_patch(branch, patch_path)
      FileUtils.rm(patch_path, force: true)

77
      find_merge_base_with_master(branch: branch)
78

79 80
      step(
        "Generating the patch against origin/master in #{patch_path}",
81
        %w[git diff --binary origin/master...HEAD]
82
      ) do |output, status|
83 84 85 86 87
        throw(:halt_check, :ko) unless status.zero?

        File.write(patch_path, output)

        throw(:halt_check, :ko) unless File.exist?(patch_path)
88
      end
89 90 91 92 93
    end

    def ce_branch_compat_check!
      if check_patch(ce_patch_full_path).zero?
        puts applies_cleanly_msg(ce_branch)
94 95 96 97 98
        throw(:halt_check)
      end
    end

    def ee_branch_presence_check!
99
      _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}])
100

101 102
      if status.zero?
        @ee_branch_found = ee_branch_prefix
103
        return
104 105
      end

106 107
      _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}])

108 109 110
      if status.zero?
        @ee_branch_found = ee_branch_suffix
      else
111 112 113 114 115 116 117 118
        puts
        puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg

        throw(:halt_check, :ko)
      end
    end

    def ee_branch_compat_check!
119
      step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD])
120

121
      generate_patch(ee_branch_found, ee_patch_full_path)
122

123
      unless check_patch(ee_patch_full_path).zero?
124 125 126 127 128 129 130
        puts
        puts ee_branch_doesnt_apply_cleanly_msg

        throw(:halt_check, :ko)
      end

      puts
131
      puts applies_cleanly_msg(ee_branch_found)
132 133
    end

134 135
    def check_patch(patch_path)
      step("Checking out master", %w[git checkout master])
136
      step("Resetting to latest master", %w[git reset --hard origin/master])
137
      step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo_url} #{ce_branch}])
138 139
      step(
        "Checking if #{patch_path} applies cleanly to EE/master",
140 141 142 143 144 145 146 147 148
        # Don't use --check here because it can result in a 0-exit status even
        # though the patch doesn't apply cleanly, e.g.:
        #   > git apply --check --3way foo.patch
        #   error: patch failed: lib/gitlab/ee_compat_check.rb:74
        #   Falling back to three-way merge...
        #   Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
        #   > echo $?
        #   0
        %W[git apply --3way #{patch_path}]
149
      ) do |output, status|
150
        puts output
151 152 153 154 155 156 157
        unless status.zero?
          @failed_files = output.lines.reduce([]) do |memo, line|
            if line.start_with?('error: patch failed:')
              file = line.sub(/\Aerror: patch failed: /, '')
              memo << file unless file =~ IGNORED_FILES_REGEX
            end
            memo
158
          end
159 160

          status = 0 if failed_files.empty?
161
        end
162

163
        command(%w[git reset --hard])
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
        status
      end
    end

    def delete_ee_branches_locally!
      command(%w[git checkout master])
      command(%W[git branch --delete --force #{ee_branch_prefix}])
      command(%W[git branch --delete --force #{ee_branch_suffix}])
    end

    def merge_base_found?
      step(
        "Finding merge base with master",
        %w[git merge-base origin/master HEAD]
      ) do |output, status|
        if status.zero?
          puts "Merge base was found: #{output}"
          true
182 183
        end
      end
184
    end
185

186 187 188 189 190 191 192 193 194 195
    def find_merge_base_with_master(branch:)
      # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
      # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
      depth = 20
      success =
        (3..6).any? do |factor|
          depth += Math.exp(factor).to_i
          # Repository is initially cloned with a depth of 20 so we need to fetch
          # deeper in the case the branch has more than 20 commits on top of master
          fetch(branch: branch, depth: depth)
196
          fetch(branch: 'master', depth: depth, remote: DEFAULT_CE_PROJECT_URL)
197 198 199 200 201

          merge_base_found?
        end

      raise "\n#{branch} is too far behind master, please rebase it!\n" unless success
202 203
    end

204
    def fetch(branch:, depth:, remote: 'origin')
205 206
      step(
        "Fetching deeper...",
207
        %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/origin/#{branch}]
208 209 210
      ) do |output, status|
        raise "Fetch failed: #{output}" unless status.zero?
      end
211 212 213
    end

    def ce_patch_name
214
      @ce_patch_name ||= patch_name_from_branch(ce_branch)
215 216 217
    end

    def ce_patch_full_path
218
      @ce_patch_full_path ||= patches_dir.join(ce_patch_name)
219 220
    end

221 222 223 224 225 226
    def ee_branch_suffix
      @ee_branch_suffix ||= "#{ce_branch}-ee"
    end

    def ee_branch_prefix
      @ee_branch_prefix ||= "ee-#{ce_branch}"
227 228 229
    end

    def ee_patch_name
230
      @ee_patch_name ||= patch_name_from_branch(ee_branch_found)
231 232 233
    end

    def ee_patch_full_path
234
      @ee_patch_full_path ||= patches_dir.join(ee_patch_name)
235 236
    end

237 238 239 240
    def patch_name_from_branch(branch_name)
      branch_name.parameterize << '.patch'
    end

241
    def patch_url
242
      "#{ce_project_url}/-/jobs/#{job_id}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}"
243 244
    end

245 246 247 248
    def step(desc, cmd = nil)
      puts "\n=> #{desc}\n"

      if cmd
249
        start = Time.now
250
        puts "\n$ #{cmd.join(' ')}"
251 252 253 254 255 256 257 258 259

        output, status = command(cmd)
        puts "\n==> Finished in #{Time.now - start} seconds"

        if block_given?
          yield(output, status)
        else
          [output, status]
        end
260 261 262 263
      end
    end

    def command(cmd)
264
      Gitlab::Popen.popen(cmd)
265 266
    end

267
    def applies_cleanly_msg(branch)
268 269
      %Q{
        #{PLEASE_READ_THIS_BANNER}
270 271
        🎉 Congratulations!! 🎉

272
        The `#{branch}` branch applies cleanly to EE/master!
273

274 275
        Much ❤️! For more information, see
        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
276 277
        #{THANKS_FOR_READING_BANNER}
      }
278 279 280
    end

    def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
281 282
      %Q{
        #{PLEASE_READ_THIS_BANNER}
283 284
        💥 Oh no! 💥

285
        The `#{ce_branch}` branch does not apply cleanly to the current
286 287 288
        EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
        was found in the EE repository.

289 290 291 292 293
        If you're a community contributor, don't worry, someone from
        GitLab Inc. will take care of this, and you don't have to do anything.
        If you're willing to help, and are ok to contribute to EE as well,
        you're welcome to help. You could follow the instructions below.

294
        #{conflicting_files_msg}
295

296
        We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
297
        branch that includes changes from `#{ce_branch}` but also specific changes
298 299
        than can be applied cleanly to EE/master. In some cases, the conflicts
        are trivial and you can ignore the warning from this job. As always,
300
        use your best judgement!
301 302 303

        There are different ways to create such branch:

304
        1. Create a new branch from master and cherry-pick your CE commits
305 306 307

          # In the EE repo
          $ git fetch origin
308
          $ git checkout -b #{ee_branch_prefix} origin/master
309
          $ git fetch #{ce_repo_url} #{ce_branch}
310
          $ git cherry-pick SHA # Repeat for all the commits you want to pick
311

312
          You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
313

314
        2. Apply your branch's patch to EE
315 316

          # In the EE repo
317 318
          $ git fetch origin master
          $ git checkout -b #{ee_branch_prefix} origin/master
319 320
          $ wget #{patch_url}
          $ git apply --3way #{ce_patch_name}
321

322 323 324 325 326 327
          At this point you might have conflicts such as:

            error: patch failed: lib/gitlab/ee_compat_check.rb:5
            Falling back to three-way merge...
            Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
            U lib/gitlab/ee_compat_check.rb
328

329
          Resolve them, stage the changes and commit them.
330

331
          If the patch couldn't be applied cleanly, use the following command:
332

333
          # In the EE repo
334
          $ git apply --reject #{ce_patch_name}
335

336 337 338 339 340
          This option makes git apply the parts of the patch that are applicable,
          and leave the rejected hunks in corresponding `.rej` files.
          You can then resolve the conflicts highlighted in `.rej` by
          manually applying the correct diff from the `.rej` file to the file with conflicts.
          When finished, you can delete the `.rej` files and commit your changes.
341 342

        ⚠️ Don't forget to push your branch to gitlab-ee:
343 344

          # In the EE repo
345 346
          $ git push origin #{ee_branch_prefix}

347
        ⚠️ Also, don't forget to create a new merge request on gitlab-ee and
348
        cross-link it with the CE merge request.
349

350
        Once this is done, you can retry this failed build, and it should pass.
351

352 353
        Stay 💪 ! For more information, see
        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
354 355
        #{THANKS_FOR_READING_BANNER}
      }
356 357 358
    end

    def ee_branch_doesnt_apply_cleanly_msg
359 360
      %Q{
        #{PLEASE_READ_THIS_BANNER}
361 362
        💥 Oh no! 💥

363
        The `#{ce_branch}` does not apply cleanly to the current EE/master, and
364 365 366 367 368
        even though a `#{ee_branch_found}` branch
        exists in the EE repository, it does not apply cleanly either to
        EE/master!

        #{conflicting_files_msg}
369

370
        Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
371 372
        retry this build.

373 374
        Stay 💪 ! For more information, see
        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
375 376
        #{THANKS_FOR_READING_BANNER}
      }
377
    end
378 379 380 381 382 383

    def conflicting_files_msg
      failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file|
        memo << "\n        - #{file}"
      end
    end
384 385
  end
end