BigW Consortium Gitlab

encoding_helper.rb 3.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
module Gitlab
  module EncodingHelper
    extend self

    # This threshold is carefully tweaked to prevent usage of encodings detected
    # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
    # we're better off sticking with utf8 encoding.
    # Reason: git diff can return strings with invalid utf8 byte sequences if it
    # truncates a diff in the middle of a multibyte character. In this case
    # CharlockHolmes will try to guess the encoding and will likely suggest an
    # obscure encoding with low confidence.
    # There is a lot more info with this merge request:
    # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
14
    ENCODING_CONFIDENCE_THRESHOLD = 50
15 16

    def encode!(message)
17
      message = force_encode_utf8(message)
18 19 20 21
      return message if message.valid_encoding?

      # return message if message type is binary
      detect = CharlockHolmes::EncodingDetector.detect(message)
22
      return message.force_encoding("BINARY") if detect_binary?(message, detect)
23 24

      if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
25
        # force detected encoding if we have sufficient confidence.
26 27 28 29 30
        message.force_encoding(detect[:encoding])
      end

      # encode and clean the bad chars
      message.replace clean(message)
31 32 33
    rescue ArgumentError => e
      return unless e.message.include?('unknown encoding name')

34 35 36 37
      encoding = detect ? detect[:encoding] : "unknown"
      "--broken encoding: #{encoding}"
    end

38
    def detect_binary?(data, detect = nil)
39
      detect ||= CharlockHolmes::EncodingDetector.detect(data)
40
      detect && detect[:type] == :binary && detect[:confidence] == 100
41 42
    end

43
    def detect_libgit2_binary?(data)
44 45 46 47
      # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
      # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
      # which is what we use below to keep a consistent behavior.
      detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
48
      detect && detect[:type] == :binary
49 50
    end

51
    def encode_utf8(message)
52 53
      message = force_encode_utf8(message)
      return message if message.valid_encoding?
54

55
      detect = CharlockHolmes::EncodingDetector.detect(message)
56
      if detect && detect[:encoding]
57 58 59 60 61 62 63 64 65 66
        begin
          CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
        rescue ArgumentError => e
          Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")

          ''
        end
      else
        clean(message)
      end
67 68
    rescue ArgumentError
      return nil
69
    end
70

71 72 73 74 75 76 77 78 79 80
    def encode_binary(s)
      return "" if s.nil?

      s.dup.force_encoding(Encoding::ASCII_8BIT)
    end

    def binary_stringio(s)
      StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) }
    end

81 82
    private

83 84 85 86 87 88 89 90 91
    def force_encode_utf8(message)
      raise ArgumentError unless message.respond_to?(:force_encoding)
      return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?

      message = message.dup if message.respond_to?(:frozen?) && message.frozen?

      message.force_encoding("UTF-8")
    end

92 93 94 95 96 97 98
    def clean(message)
      message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
        .encode("UTF-8")
        .gsub("\0".encode("UTF-8"), "")
    end
  end
end