Merge branch 'rs-absorb-gitlab_git' into 'master'

Gemfile

@@ -16,6 +16,8 @@ gem 'default_value_for', '~> 3.0.0'
 gem 'mysql2', '~> 0.3.16', group: :mysql
 gem 'pg', '~> 0.18.2', group: :postgres
 
+gem 'rugged', '~> 0.24.0'
+
 # Authentication libraries
 gem 'devise',                 '~> 4.2'
 gem 'doorkeeper',             '~> 4.2.0'
@@ -49,10 +51,6 @@ gem 'u2f', '~> 0.2.1'
 # Browser detection
 gem 'browser', '~> 2.2'
 
-# Extracting information from a git repository
-# Provide access to Gitlab::Git library
-gem 'gitlab_git', '~> 10.7.0'
-
 # LDAP Auth
 # GitLab fork with several improvements to original library. For full list of changes
 # see https://github.com/intridea/omniauth-ldap/compare/master...gitlabhq:master

Gemfile.lock

@@ -255,11 +255,6 @@ GEM
       mime-types (>= 1.16, < 3)
       posix-spawn (~> 0.3)
     gitlab-markup (1.5.0)
-    gitlab_git (10.7.0)
-      activesupport (~> 4.0)
-      charlock_holmes (~> 0.7.3)
-      github-linguist (~> 4.7.0)
-      rugged (~> 0.24.0)
     gitlab_omniauth-ldap (1.2.1)
       net-ldap (~> 0.9)
       omniauth (~> 1.0)
@@ -857,7 +852,6 @@ DEPENDENCIES
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-markup (~> 1.5.0)
-  gitlab_git (~> 10.7.0)
   gitlab_omniauth-ldap (~> 1.2.1)
   gollum-lib (~> 4.2)
   gollum-rugged_adapter (~> 0.4.2)
@@ -942,6 +936,7 @@ DEPENDENCIES
   rubocop-rspec (~> 1.5.0)
   ruby-fogbugz (~> 0.2.1)
   ruby-prof (~> 0.16.2)
+  rugged (~> 0.24.0)
   sanitize (~> 2.0)
   sass-rails (~> 5.0.6)
   scss_lint (~> 0.47.0)
@@ -988,4 +983,4 @@ DEPENDENCIES
   wikicloth (= 0.8.1)
 
 BUNDLED WITH
-   1.13.6
+   1.13.7

app/models/merge_request_diff.rb

 class MergeRequestDiff < ActiveRecord::Base
   include Sortable
   include Importable
-  include EncodingHelper
+  include Gitlab::Git::EncodingHelper
 
   # Prevent store of diff if commits amount more then 500
   COMMITS_SAFE_SIZE = 100

lib/gitlab/git/attributes.rb

+module Gitlab
+  module Git
+    # Class for parsing Git attribute files and extracting the attributes for
+    # file patterns.
+    #
+    # Unlike Rugged this parser only needs a single IO call (a call to `open`),
+    # vastly reducing the time spent in extracting attributes.
+    #
+    # This class _only_ supports parsing the attributes file located at
+    # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files
+    # (`.gitattributes` is copied to this particular path).
+    #
+    # Basic usage:
+    #
+    #     attributes = Gitlab::Git::Attributes.new(some_repo.path)
+    #
+    #     attributes.attributes('README.md') # => { "eol" => "lf }
+    class Attributes
+      # path - The path to the Git repository.
+      def initialize(path)
+        @path = File.expand_path(path)
+        @patterns = nil
+      end
+
+      # Returns all the Git attributes for the given path.
+      #
+      # path - A path to a file for which to get the attributes.
+      #
+      # Returns a Hash.
+      def attributes(path)
+        full_path = File.join(@path, path)
+
+        patterns.each do |pattern, attrs|
+          return attrs if File.fnmatch?(pattern, full_path)
+        end
+
+        {}
+      end
+
+      # Returns a Hash containing the file patterns and their attributes.
+      def patterns
+        @patterns ||= parse_file
+      end
+
+      # Parses an attribute string.
+      #
+      # These strings can be in the following formats:
+      #
+      #     text      # => { "text" => true }
+      #     -text     # => { "text" => false }
+      #     key=value # => { "key" => "value" }
+      #
+      # string - The string to parse.
+      #
+      # Returns a Hash containing the attributes and their values.
+      def parse_attributes(string)
+        values = {}
+        dash = '-'
+        equal = '='
+        binary = 'binary'
+
+        string.split(/\s+/).each do |chunk|
+          # Data such as "foo = bar" should be treated as "foo" and "bar" being
+          # separate boolean attributes.
+          next if chunk == equal
+
+          key = chunk
+
+          # Input: "-foo"
+          if chunk.start_with?(dash)
+            key = chunk.byteslice(1, chunk.length - 1)
+            value = false
+
+          # Input: "foo=bar"
+          elsif chunk.include?(equal)
+            key, value = chunk.split(equal, 2)
+
+          # Input: "foo"
+          else
+            value = true
+          end
+
+          values[key] = value
+
+          # When the "binary" option is set the "diff" option should be set to
+          # the inverse. If "diff" is later set it should overwrite the
+          # automatically set value.
+          values['diff'] = false if key == binary && value
+        end
+
+        values
+      end
+
+      # Iterates over every line in the attributes file.
+      def each_line
+        full_path = File.join(@path, 'info/attributes')
+
+        return unless File.exist?(full_path)
+
+        File.open(full_path, 'r') do |handle|
+          handle.each_line do |line|
+            break unless line.valid_encoding?
+
+            yield line.strip
+          end
+        end
+      end
+
+      private
+
+      # Parses the Git attributes file.
+      def parse_file
+        pairs = []
+        comment = '#'
+
+        each_line do |line|
+          next if line.start_with?(comment) || line.empty?
+
+          pattern, attrs = line.split(/\s+/, 2)
+
+          parsed = attrs ? parse_attributes(attrs) : {}
+
+          pairs << [File.join(@path, pattern), parsed]
+        end
+
+        # Newer entries take precedence over older entries.
+        pairs.reverse.to_h
+      end
+    end
+  end
+end

lib/gitlab/git/blame.rb

+require_relative 'encoding_helper'
+
+module Gitlab
+  module Git
+    class Blame
+      include Gitlab::Git::EncodingHelper
+
+      attr_reader :lines, :blames
+
+      def initialize(repository, sha, path)
+        @repo = repository
+        @sha = sha
+        @path = path
+        @lines = []
+        @blames = load_blame
+      end
+
+      def each
+        @blames.each do |blame|
+          yield(
+            Gitlab::Git::Commit.new(blame.commit),
+            blame.line
+          )
+        end
+      end
+
+      private
+
+      def load_blame
+        cmd = %W(git --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
+        # Read in binary mode to ensure ASCII-8BIT
+        raw_output = IO.popen(cmd, 'rb') {|io| io.read }
+        output = encode_utf8(raw_output)
+        process_raw_blame output
+      end
+
+      def process_raw_blame(output)
+        lines, final = [], []
+        info, commits = {}, {}
+
+        # process the output
+        output.split("\n").each do |line|
+          if line[0, 1] == "\t"
+            lines << line[1, line.size]
+          elsif m = /^(\w{40}) (\d+) (\d+)/.match(line)
+            commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i
+            commits[commit_id] = nil unless commits.key?(commit_id)
+            info[lineno] = [commit_id, old_lineno]
+          end
+        end
+
+        # load all commits in single call
+        commits.keys.each do |key|
+          commits[key] = @repo.lookup(key)
+        end
+
+        # get it together
+        info.sort.each do |lineno, (commit_id, old_lineno)|
+          commit = commits[commit_id]
+          final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1])
+        end
+
+        @lines = final
+      end
+    end
+
+    class BlameLine
+      attr_accessor :lineno, :oldlineno, :commit, :line
+      def initialize(lineno, oldlineno, commit, line)
+        @lineno = lineno
+        @oldlineno = oldlineno
+        @commit = commit
+        @line = line
+      end
+    end
+  end
+end

lib/gitlab/git/blob_snippet.rb

+module Gitlab
+  module Git
+    class BlobSnippet
+      include Linguist::BlobHelper
+
+      attr_accessor :ref
+      attr_accessor :lines
+      attr_accessor :filename
+      attr_accessor :startline
+
+      def initialize(ref, lines, startline, filename)
+        @ref, @lines, @startline, @filename = ref, lines, startline, filename
+      end
+
+      def data
+        lines.join("\n") if lines
+      end
+
+      def name
+        filename
+      end
+
+      def size
+        data.length
+      end
+
+      def mode
+        nil
+      end
+    end
+  end
+end

lib/gitlab/git/branch.rb

+module Gitlab
+  module Git
+    class Branch < Ref
+    end
+  end
+end

lib/gitlab/git/commit.rb

+# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object
+module Gitlab
+  module Git
+    class Commit
+      include Gitlab::Git::EncodingHelper
+
+      attr_accessor :raw_commit, :head, :refs
+
+      SERIALIZE_KEYS = [
+        :id, :message, :parent_ids,
+        :authored_date, :author_name, :author_email,
+        :committed_date, :committer_name, :committer_email
+      ].freeze
+
+      attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator
+
+      def ==(other)
+        return false unless other.is_a?(Gitlab::Git::Commit)
+
+        methods = [:message, :parent_ids, :authored_date, :author_name,
+                   :author_email, :committed_date, :committer_name,
+                   :committer_email]
+
+        methods.all? do |method|
+          send(method) == other.send(method)
+        end
+      end
+
+      class << self
+        # Get commits collection
+        #
+        # Ex.
+        #   Commit.where(
+        #     repo: repo,
+        #     ref: 'master',
+        #     path: 'app/models',
+        #     limit: 10,
+        #     offset: 5,
+        #   )
+        #
+        def where(options)
+          repo = options.delete(:repo)
+          raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
+
+          repo.log(options).map { |c| decorate(c) }
+        end
+
+        # Get single commit
+        #
+        # Ex.
+        #   Commit.find(repo, '29eda46b')
+        #
+        #   Commit.find(repo, 'master')
+        #
+        def find(repo, commit_id = "HEAD")
+          return decorate(commit_id) if commit_id.is_a?(Rugged::Commit)
+
+          obj = if commit_id.is_a?(String)
+                  repo.rev_parse_target(commit_id)
+                else
+                  Gitlab::Git::Ref.dereference_object(commit_id)
+                end
+
+          return nil unless obj.is_a?(Rugged::Commit)
+
+          decorate(obj)
+        rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository
+          nil
+        end
+
+        # Get last commit for HEAD
+        #
+        # Ex.
+        #   Commit.last(repo)
+        #
+        def last(repo)
+          find(repo)
+        end
+
+        # Get last commit for specified path and ref
+        #
+        # Ex.
+        #   Commit.last_for_path(repo, '29eda46b', 'app/models')
+        #
+        #   Commit.last_for_path(repo, 'master', 'Gemfile')
+        #
+        def last_for_path(repo, ref, path = nil)
+          where(
+            repo: repo,
+            ref: ref,
+            path: path,
+            limit: 1
+          ).first
+        end
+
+        # Get commits between two revspecs
+        # See also #repository.commits_between
+        #
+        # Ex.
+        #   Commit.between(repo, '29eda46b', 'master')
+        #
+        def between(repo, base, head)
+          repo.commits_between(base, head).map do |commit|
+            decorate(commit)
+          end
+        rescue Rugged::ReferenceError
+          []
+        end
+
+        # Delegate Repository#find_commits
+        def find_all(repo, options = {})
+          repo.find_commits(options)
+        end
+
+        def decorate(commit, ref = nil)
+          Gitlab::Git::Commit.new(commit, ref)
+        end
+
+        # Returns a diff object for the changes introduced by +rugged_commit+.
+        # If +rugged_commit+ doesn't have a parent, then the diff is between
+        # this commit and an empty repo.  See Repository#diff for the keys
+        # allowed in the +options+ hash.
+        def diff_from_parent(rugged_commit, options = {})
+          options ||= {}
+          break_rewrites = options[:break_rewrites]
+          actual_options = Gitlab::Git::Diff.filter_diff_options(options)
+
+          diff = if rugged_commit.parents.empty?
+                   rugged_commit.diff(actual_options.merge(reverse: true))
+                 else
+                   rugged_commit.parents[0].diff(rugged_commit, actual_options)
+                 end
+
+          diff.find_similar!(break_rewrites: break_rewrites)
+          diff
+        end
+      end
+
+      def initialize(raw_commit, head = nil)
+        raise "Nil as raw commit passed" unless raw_commit
+
+        if raw_commit.is_a?(Hash)
+          init_from_hash(raw_commit)
+        elsif raw_commit.is_a?(Rugged::Commit)
+          init_from_rugged(raw_commit)
+        else
+          raise "Invalid raw commit type: #{raw_commit.class}"
+        end
+
+        @head = head
+      end
+
+      def sha
+        id
+      end
+
+      def short_id(length = 10)
+        id.to_s[0..length]
+      end
+
+      def safe_message
+        @safe_message ||= message
+      end
+
+      def created_at
+        committed_date
+      end
+
+      # Was this commit committed by a different person than the original author?
+      def different_committer?
+        author_name != committer_name || author_email != committer_email
+      end
+
+      def parent_id
+        parent_ids.first
+      end
+
+      # Shows the diff between the commit's parent and the commit.
+      #
+      # Cuts out the header and stats from #to_patch and returns only the diff.
+      def to_diff(options = {})
+        diff_from_parent(options).patch
+      end
+
+      # Returns a diff object for the changes from this commit's first parent.
+      # If there is no parent, then the diff is between this commit and an
+      # empty repo.  See Repository#diff for keys allowed in the +options+
+      # hash.
+      def diff_from_parent(options = {})
+        Commit.diff_from_parent(raw_commit, options)
+      end
+
+      def has_zero_stats?
+        stats.total.zero?
+      rescue
+        true
+      end
+
+      def no_commit_message
+        "--no commit message"
+      end
+
+      def to_hash
+        serialize_keys.map.with_object({}) do |key, hash|
+          hash[key] = send(key)
+        end
+      end
+
+      def date
+        committed_date
+      end
+
+      def diffs(options = {})
+        Gitlab::Git::DiffCollection.new(diff_from_parent(options), options)
+      end
+
+      def parents
+        raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) }
+      end
+
+      def tree
+        raw_commit.tree
+      end
+
+      def stats
+        Gitlab::Git::CommitStats.new(self)
+      end
+
+      def to_patch(options = {})
+        begin
+          raw_commit.to_mbox(options)
+        rescue Rugged::InvalidError => ex
+          if ex.message =~ /Commit \w+ is a merge commit/
+            'Patch format is not currently supported for merge commits.'
+          end
+        end
+      end
+
+      # Get a collection of Rugged::Reference objects for this commit.
+      #
+      # Ex.
+      #   commit.ref(repo)
+      #
+      def refs(repo)
+        repo.refs_hash[id]
+      end
+
+      # Get ref names collection
+      #
+      # Ex.
+      #   commit.ref_names(repo)
+      #
+      def ref_names(repo)
+        refs(repo).map do |ref|
+          ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "")
+        end
+      end
+
+      def message
+        encode! @message
+      end
+
+      def author_name
+        encode! @author_name
+      end
+
+      def author_email
+        encode! @author_email
+      end
+
+      def committer_name
+        encode! @committer_name
+      end
+
+      def committer_email
+        encode! @committer_email
+      end
+
+      private
+
+      def init_from_hash(hash)
+        raw_commit = hash.symbolize_keys
+
+        serialize_keys.each do |key|
+          send("#{key}=", raw_commit[key])
+        end
+      end
+
+      def init_from_rugged(commit)
+        author = commit.author
+        committer = commit.committer
+
+        @raw_commit = commit
+        @id = commit.oid
+        @message = commit.message
+        @authored_date = author[:time]
+        @committed_date = committer[:time]
+        @author_name = author[:name]
+        @author_email = author[:email]
+        @committer_name = committer[:name]
+        @committer_email = committer[:email]
+        @parent_ids = commit.parents.map(&:oid)
+      end
+
+      def serialize_keys
+        SERIALIZE_KEYS
+      end
+    end
+  end
+end

lib/gitlab/git/commit_stats.rb

+# Gitlab::Git::CommitStats counts the additions, deletions, and total changes
+# in a commit.
+module Gitlab
+  module Git
+    class CommitStats
+      attr_reader :id, :additions, :deletions, :total
+
+      # Instantiate a CommitStats object
+      def initialize(commit)
+        @id = commit.id
+        @additions = 0
+        @deletions = 0
+        @total = 0
+
+        diff = commit.diff_from_parent
+
+        diff.each_patch do |p|
+          # TODO: Use the new Rugged convenience methods when they're released
+          @additions += p.stat[0]
+          @deletions += p.stat[1]
+          @total += p.changes
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/compare.rb

+module Gitlab
+  module Git
+    class Compare
+      attr_reader :head, :base, :straight
+
+      def initialize(repository, base, head, straight = false)
+        @repository = repository
+        @straight = straight
+
+        unless base && head
+          @commits = []
+          return
+        end
+
+        @base = Gitlab::Git::Commit.find(repository, base.try(:strip))
+        @head = Gitlab::Git::Commit.find(repository, head.try(:strip))
+
+        @commits = [] unless @base && @head
+        @commits = [] if same
+      end
+
+      def same
+        @base && @head && @base.id == @head.id
+      end
+
+      def commits
+        return @commits if defined?(@commits)
+
+        @commits = Gitlab::Git::Commit.between(@repository, @base.id, @head.id)
+      end
+
+      def diffs(options = {})
+        unless @head && @base
+          return Gitlab::Git::DiffCollection.new([])
+        end
+
+        paths = options.delete(:paths) || []
+        options[:straight] = @straight
+        Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths)
+      end
+    end
+  end
+end

lib/gitlab/git/diff_collection.rb

+module Gitlab
+  module Git
+    class DiffCollection
+      include Enumerable
+
+      DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze
+
+      def initialize(iterator, options = {})
+        @iterator = iterator
+        @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
+        @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
+        @max_bytes = @max_files * 5120 # Average 5 KB per file
+        @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min
+        @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min
+        @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file
+        @all_diffs = !!options.fetch(:all_diffs, false)
+        @no_collapse = !!options.fetch(:no_collapse, true)
+        @deltas_only = !!options.fetch(:deltas_only, false)
+
+        @line_count = 0
+        @byte_count = 0
+        @overflow = false
+        @array = Array.new
+      end
+
+      def each(&block)
+        if @populated
+          # @iterator.each is slower than just iterating the array in place
+          @array.each(&block)
+        elsif @deltas_only
+          each_delta(&block)
+        else
+          each_patch(&block)
+        end
+      end
+
+      def empty?
+        !@iterator.any?
+      end
+
+      def overflow?
+        populate!
+        !!@overflow
+      end
+
+      def size
+        @size ||= count # forces a loop using each method
+      end
+
+      def real_size
+        populate!
+
+        if @overflow
+          "#{size}+"
+        else
+          size.to_s
+        end
+      end
+
+      def decorate!
+        collection = each_with_index do |element, i|
+          @array[i] = yield(element)
+        end
+        @populated = true
+        collection
+      end
+
+      private
+
+      def populate!
+        return if @populated
+
+        each { nil } # force a loop through all diffs
+        @populated = true
+        nil
+      end
+
+      def over_safe_limits?(files)
+        files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes
+      end
+
+      def each_delta
+        @iterator.each_delta.with_index do |delta, i|
+          diff = Gitlab::Git::Diff.new(delta)
+
+          yield @array[i] = diff
+        end
+      end
+
+      def each_patch
+        @iterator.each_with_index do |raw, i|
+          # First yield cached Diff instances from @array
+          if @array[i]
+            yield @array[i]
+            next
+          end
+
+          # We have exhausted @array, time to create new Diff instances or stop.
+          break if @overflow
+
+          if !@all_diffs && i >= @max_files
+            @overflow = true
+            break
+          end
+
+          collapse = !@all_diffs && !@no_collapse
+
+          diff = Gitlab::Git::Diff.new(raw, collapse: collapse)
+
+          if collapse && over_safe_limits?(i)
+            diff.prune_collapsed_diff!
+          end
+
+          @line_count += diff.line_count
+          @byte_count += diff.diff.bytesize
+
+          if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes)
+            # This last Diff instance pushes us over the lines limit. We stop and
+            # discard it.
+            @overflow = true
+            break
+          end
+
+          yield @array[i] = diff
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/encoding_helper.rb

+module Gitlab
+  module Git
+    module EncodingHelper
+      extend self
+
+      # This threshold is carefully tweaked to prevent usage of encodings detected
+      # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
+      # we're better off sticking with utf8 encoding.
+      # Reason: git diff can return strings with invalid utf8 byte sequences if it
+      # truncates a diff in the middle of a multibyte character. In this case
+      # CharlockHolmes will try to guess the encoding and will likely suggest an
+      # obscure encoding with low confidence.
+      # There is a lot more info with this merge request:
+      # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
+      ENCODING_CONFIDENCE_THRESHOLD = 40
+
+      def encode!(message)
+        return nil unless message.respond_to? :force_encoding
+
+        # if message is utf-8 encoding, just return it
+        message.force_encoding("UTF-8")
+        return message if message.valid_encoding?
+
+        # return message if message type is binary
+        detect = CharlockHolmes::EncodingDetector.detect(message)
+        return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+        # force detected encoding if we have sufficient confidence.
+        if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+          message.force_encoding(detect[:encoding])
+        end
+
+        # encode and clean the bad chars
+        message.replace clean(message)
+      rescue
+        encoding = detect ? detect[:encoding] : "unknown"
+        "--broken encoding: #{encoding}"
+      end
+
+      def encode_utf8(message)
+        detect = CharlockHolmes::EncodingDetector.detect(message)
+        if detect
+          CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+        else
+          clean(message)
+        end
+      end
+
+      private
+
+      def clean(message)
+        message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+          .encode("UTF-8")
+          .gsub("\0".encode("UTF-8"), "")
+      end
+    end
+  end
+end

lib/gitlab/git/path_helper.rb

+module Gitlab
+  module Git
+    class PathHelper
+      class << self
+        def normalize_path(filename)
+          # Strip all leading slashes so that //foo -> foo
+          filename[/^\/*/] = ''
+
+          # Expand relative paths (e.g. foo/../bar)
+          filename = Pathname.new(filename)
+          filename.relative_path_from(Pathname.new(''))
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/popen.rb

+require 'open3'
+
+module Gitlab
+  module Git
+    module Popen
+      def popen(cmd, path)
+        unless cmd.is_a?(Array)
+          raise "System commands must be given as an array of strings"
+        end
+
+        vars = { "PWD" => path }
+        options = { chdir: path }
+
+        @cmd_output = ""
+        @cmd_status = 0
+        Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+          @cmd_output << stdout.read
+          @cmd_output << stderr.read
+          @cmd_status = wait_thr.value.exitstatus
+        end
+
+        [@cmd_output, @cmd_status]
+      end
+    end
+  end
+end

lib/gitlab/git/ref.rb

+module Gitlab
+  module Git
+    class Ref
+      include Gitlab::Git::EncodingHelper
+
+      # Branch or tag name
+      # without "refs/tags|heads" prefix
+      attr_reader :name
+
+      # Target sha.
+      # Usually it is commit sha but in case
+      # when tag reference on other tag it can be tag sha
+      attr_reader :target
+
+      # Dereferenced target
+      # Commit object to which the Ref points to
+      attr_reader :dereferenced_target
+
+      # Extract branch name from full ref path
+      #
+      # Ex.
+      #   Ref.extract_branch_name('refs/heads/master') #=> 'master'
+      def self.extract_branch_name(str)
+        str.gsub(/\Arefs\/heads\//, '')
+      end
+
+      def self.dereference_object(object)
+        object = object.target while object.is_a?(Rugged::Tag::Annotation)
+
+        object
+      end
+
+      def initialize(repository, name, target)
+        encode! name
+        @name = name.gsub(/\Arefs\/(tags|heads)\//, '')
+        @dereferenced_target = Gitlab::Git::Commit.find(repository, target)
+        @target = if target.respond_to?(:oid)
+                    target.oid
+                  elsif target.respond_to?(:name)
+                    target.name
+                  elsif target.is_a? String
+                    target
+                  else
+                    nil
+                  end
+      end
+    end
+  end
+end

lib/gitlab/git/tag.rb

+module Gitlab
+  module Git
+    class Tag < Ref
+      attr_reader :object_sha
+
+      def initialize(repository, name, target, message = nil)
+        super(repository, name, target)
+
+        @message = message
+      end
+
+      def message
+        encode! @message
+      end
+    end
+  end
+end

lib/gitlab/git/tree.rb

+module Gitlab
+  module Git
+    class Tree
+      include Gitlab::Git::EncodingHelper
+
+      attr_accessor :id, :root_id, :name, :path, :type,
+        :mode, :commit_id, :submodule_url
+
+      class << self
+        # Get list of tree objects
+        # for repository based on commit sha and path
+        # Uses rugged for raw objects
+        def where(repository, sha, path = nil)
+          path = nil if path == '' || path == '/'
+
+          commit = repository.lookup(sha)
+          root_tree = commit.tree
+
+          tree = if path
+                   id = find_id_by_path(repository, root_tree.oid, path)
+                   if id
+                     repository.lookup(id)
+                   else
+                     []
+                   end
+                 else
+                   root_tree
+                 end
+
+          tree.map do |entry|
+            new(
+              id: entry[:oid],
+              root_id: root_tree.oid,
+              name: entry[:name],
+              type: entry[:type],
+              mode: entry[:filemode],
+              path: path ? File.join(path, entry[:name]) : entry[:name],
+              commit_id: sha,
+            )
+          end
+        end
+
+        # Recursive search of tree id for path
+        #
+        # Ex.
+        #   blog/            # oid: 1a
+        #     app/           # oid: 2a
+        #       models/      # oid: 3a
+        #       views/       # oid: 4a
+        #
+        #
+        # Tree.find_id_by_path(repo, '1a', 'app/models') # => '3a'
+        #
+        def find_id_by_path(repository, root_id, path)
+          root_tree = repository.lookup(root_id)
+          path_arr = path.split('/')
+
+          entry = root_tree.find do |entry|
+            entry[:name] == path_arr[0] && entry[:type] == :tree
+          end
+
+          return nil unless entry
+
+          if path_arr.size > 1
+            path_arr.shift
+            find_id_by_path(repository, entry[:oid], path_arr.join('/'))
+          else
+            entry[:oid]
+          end
+        end
+      end
+
+      def initialize(options)
+        %w(id root_id name path type mode commit_id).each do |key|
+          self.send("#{key}=", options[key.to_sym])
+        end
+      end
+
+      def name
+        encode! @name
+      end
+
+      def dir?
+        type == :tree
+      end
+
+      def file?
+        type == :blob
+      end
+
+      def submodule?
+        type == :commit
+      end
+
+      def readme?
+        name =~ /^readme/i
+      end
+
+      def contributing?
+        name =~ /^contributing/i
+      end
+    end
+  end
+end

lib/gitlab/git/util.rb

+module Gitlab
+  module Git
+    module Util
+      LINE_SEP = "\n".freeze
+
+      def self.count_lines(string)
+        case string[-1]
+        when nil
+          0
+        when LINE_SEP
+          string.count(LINE_SEP)
+        else
+          string.count(LINE_SEP) + 1
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/git/attributes_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Git::Attributes, seed_helper: true do
+  let(:path) do
+    File.join(SEED_REPOSITORY_PATH, 'with-git-attributes.git')
+  end
+
+  subject { described_class.new(path) }
+
+  describe '#attributes' do
+    context 'using a path with attributes' do
+      it 'returns the attributes as a Hash' do
+        expect(subject.attributes('test.txt')).to eq({ 'text' => true })
+      end
+
+      it 'returns a Hash containing multiple attributes' do
+        expect(subject.attributes('test.sh')).
+          to eq({ 'eol' => 'lf', 'gitlab-language' => 'shell' })
+      end
+
+      it 'returns a Hash containing attributes for a file with multiple extensions' do
+        expect(subject.attributes('test.haml.html')).
+          to eq({ 'gitlab-language' => 'haml' })
+      end
+
+      it 'returns a Hash containing attributes for a file in a directory' do
+        expect(subject.attributes('foo/bar.txt')).to eq({ 'foo' => true })
+      end
+
+      it 'returns a Hash containing attributes with query string parameters' do
+        expect(subject.attributes('foo.cgi')).
+          to eq({ 'key' => 'value?p1=v1&p2=v2' })
+      end
+
+      it 'returns a Hash containing the attributes for an absolute path' do
+        expect(subject.attributes('/test.txt')).to eq({ 'text' => true })
+      end
+
+      it 'returns a Hash containing the attributes when a pattern is defined using an absolute path' do
+        # When a path is given without a leading slash it should still match
+        # patterns defined with a leading slash.
+        expect(subject.attributes('foo.png')).
+          to eq({ 'gitlab-language' => 'png' })
+
+        expect(subject.attributes('/foo.png')).
+          to eq({ 'gitlab-language' => 'png' })
+      end
+
+      it 'returns an empty Hash for a defined path without attributes' do
+        expect(subject.attributes('bla/bla.txt')).to eq({})
+      end
+
+      context 'when the "binary" option is set for a path' do
+        it 'returns true for the "binary" option' do
+          expect(subject.attributes('test.binary')['binary']).to eq(true)
+        end
+
+        it 'returns false for the "diff" option' do
+          expect(subject.attributes('test.binary')['diff']).to eq(false)
+        end
+      end
+    end
+
+    context 'using a path without any attributes' do
+      it 'returns an empty Hash' do
+        expect(subject.attributes('test.foo')).to eq({})
+      end
+    end
+  end
+
+  describe '#patterns' do
+    it 'parses a file with entries' do
+      expect(subject.patterns).to be_an_instance_of(Hash)
+    end
+
+    it 'parses an entry that uses a tab to separate the pattern and attributes' do
+      expect(subject.patterns[File.join(path, '*.md')]).
+        to eq({ 'gitlab-language' => 'markdown' })
+    end
+
+    it 'stores patterns in reverse order' do
+      first = subject.patterns.to_a[0]
+
+      expect(first[0]).to eq(File.join(path, 'bla/bla.txt'))
+    end
+
+    # It's a bit hard to test for something _not_ being processed. As such we'll
+    # just test the number of entries.
+    it 'ignores any comments and empty lines' do
+      expect(subject.patterns.length).to eq(10)
+    end
+
+    it 'does not parse anything when the attributes file does not exist' do
+      expect(File).to receive(:exist?).
+        with(File.join(path, 'info/attributes')).
+        and_return(false)
+
+      expect(subject.patterns).to eq({})
+    end
+  end
+
+  describe '#parse_attributes' do
+    it 'parses a boolean attribute' do
+      expect(subject.parse_attributes('text')).to eq({ 'text' => true })
+    end
+
+    it 'parses a negated boolean attribute' do
+      expect(subject.parse_attributes('-text')).to eq({ 'text' => false })
+    end
+
+    it 'parses a key-value pair' do
+      expect(subject.parse_attributes('foo=bar')).to eq({ 'foo' => 'bar' })
+    end
+
+    it 'parses multiple attributes' do
+      input = 'boolean key=value -negated'
+
+      expect(subject.parse_attributes(input)).
+        to eq({ 'boolean' => true, 'key' => 'value', 'negated' => false })
+    end
+
+    it 'parses attributes with query string parameters' do
+      expect(subject.parse_attributes('foo=bar?baz=1')).
+        to eq({ 'foo' => 'bar?baz=1' })
+    end
+  end
+
+  describe '#each_line' do
+    it 'iterates over every line in the attributes file' do
+      args = [String] * 14 # the number of lines in the file
+
+      expect { |b| subject.each_line(&b) }.to yield_successive_args(*args)
+    end
+
+    it 'does not yield when the attributes file does not exist' do
+      expect(File).to receive(:exist?).
+        with(File.join(path, 'info/attributes')).
+        and_return(false)
+
+      expect { |b| subject.each_line(&b) }.not_to yield_control
+    end
+
+    it 'does not yield when the attributes file has an unsupported encoding' do
+      path = File.join(SEED_REPOSITORY_PATH, 'with-invalid-git-attributes.git')
+      attrs = described_class.new(path)
+
+      expect { |b| attrs.each_line(&b) }.not_to yield_control
+    end
+  end
+end

spec/lib/gitlab/git/blame_spec.rb

+# coding: utf-8
+require "spec_helper"
+
+describe Gitlab::Git::Blame, seed_helper: true do
+  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:blame) do
+    Gitlab::Git::Blame.new(repository, SeedRepo::Commit::ID, "CONTRIBUTING.md")
+  end
+
+  context "each count" do
+    it do
+      data = []
+      blame.each do |commit, line|
+        data << {
+          commit: commit,
+          line: line
+        }
+      end
+
+      expect(data.size).to eq(95)
+      expect(data.first[:commit]).to be_kind_of(Gitlab::Git::Commit)
+      expect(data.first[:line]).to eq("# Contribute to GitLab")
+    end
+  end
+
+  context "ISO-8859 encoding" do
+    let(:blame) do
+      Gitlab::Git::Blame.new(repository, SeedRepo::EncodingCommit::ID, "encoding/iso8859.txt")
+    end
+
+    it 'converts to UTF-8' do
+      data = []
+      blame.each do |commit, line|
+        data << {
+          commit: commit,
+          line: line
+        }
+      end
+
+      expect(data.size).to eq(1)
+      expect(data.first[:commit]).to be_kind_of(Gitlab::Git::Commit)
+      expect(data.first[:line]).to eq("Ä ü")
+    end
+  end
+
+  context "unknown encoding" do
+    let(:blame) do
+      Gitlab::Git::Blame.new(repository, SeedRepo::EncodingCommit::ID, "encoding/iso8859.txt")
+    end
+
+    it 'converts to UTF-8' do
+      expect(CharlockHolmes::EncodingDetector).to receive(:detect).and_return(nil)
+      data = []
+      blame.each do |commit, line|
+        data << {
+            commit: commit,
+            line: line
+        }
+      end
+
+      expect(data.size).to eq(1)
+      expect(data.first[:commit]).to be_kind_of(Gitlab::Git::Commit)
+      expect(data.first[:line]).to eq(" ")
+    end
+  end
+end

spec/lib/gitlab/git/blob_snippet_spec.rb

+# encoding: UTF-8
+
+require "spec_helper"
+
+describe Gitlab::Git::BlobSnippet, seed_helper: true do
+  describe :data do
+    context 'empty lines' do
+      let(:snippet) { Gitlab::Git::BlobSnippet.new('master', nil, nil, nil) }
+
+      it { expect(snippet.data).to be_nil }
+    end
+
+    context 'present lines' do
+      let(:snippet) { Gitlab::Git::BlobSnippet.new('master', ['wow', 'much'], 1, 'wow.rb') }
+
+      it { expect(snippet.data).to eq("wow\nmuch") }
+    end
+  end
+end

spec/lib/gitlab/git/branch_spec.rb

+require "spec_helper"
+
+describe Gitlab::Git::Branch, seed_helper: true do
+  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+
+  subject { repository.branches }
+
+  it { is_expected.to be_kind_of Array }
+
+  describe '#size' do
+    subject { super().size }
+    it { is_expected.to eq(SeedRepo::Repo::BRANCHES.size) }
+  end
+
+  describe 'first branch' do
+    let(:branch) { repository.branches.first }
+
+    it { expect(branch.name).to eq(SeedRepo::Repo::BRANCHES.first) }
+    it { expect(branch.dereferenced_target.sha).to eq("0b4bc9a49b562e85de7cc9e834518ea6828729b9") }
+  end
+
+  describe 'master branch' do
+    let(:branch) do
+      repository.branches.find { |branch| branch.name == 'master' }
+    end
+
+    it { expect(branch.dereferenced_target.sha).to eq(SeedRepo::LastCommit::ID) }
+  end
+
+  it { expect(repository.branches.size).to eq(SeedRepo::Repo::BRANCHES.size) }
+end

spec/lib/gitlab/git/compare_spec.rb

+require "spec_helper"
+
+describe Gitlab::Git::Compare, seed_helper: true do
+  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::Commit::ID, false) }
+  let(:compare_straight) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::Commit::ID, true) }
+
+  describe :commits do
+    subject do
+      compare.commits.map(&:id)
+    end
+
+    it 'has 8 elements' do
+      expect(subject.size).to eq(8)
+    end
+
+    it { is_expected.to include(SeedRepo::Commit::PARENT_ID) }
+    it { is_expected.not_to include(SeedRepo::BigCommit::PARENT_ID) }
+
+    context 'non-existing base ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, 'no-such-branch', SeedRepo::Commit::ID) }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'non-existing head ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, '1234567890') }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'base ref is equal to head ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::BigCommit::ID) }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'providing nil as base ref or head ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, nil, nil) }
+
+      it { is_expected.to be_empty }
+    end
+  end
+
+  describe :diffs do
+    subject do
+      compare.diffs.map(&:new_path)
+    end
+
+    it 'has 10 elements' do
+      expect(subject.size).to eq(10)
+    end
+
+    it { is_expected.to include('files/ruby/popen.rb') }
+    it { is_expected.not_to include('LICENSE') }
+
+    context 'non-existing base ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, 'no-such-branch', SeedRepo::Commit::ID) }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'non-existing head ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, '1234567890') }
+
+      it { is_expected.to be_empty }
+    end
+  end
+
+  describe :same  do
+    subject do
+      compare.same
+    end
+
+    it { is_expected.to eq(false) }
+
+    context 'base ref is equal to head ref' do
+      let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::BigCommit::ID) }
+
+      it { is_expected.to eq(true) }
+    end
+  end
+
+  describe :commits_straight do
+    subject do
+      compare_straight.commits.map(&:id)
+    end
+
+    it 'has 8 elements' do
+      expect(subject.size).to eq(8)
+    end
+
+    it { is_expected.to include(SeedRepo::Commit::PARENT_ID) }
+    it { is_expected.not_to include(SeedRepo::BigCommit::PARENT_ID) }
+  end
+
+  describe :diffs_straight do
+    subject do
+      compare_straight.diffs.map(&:new_path)
+    end
+
+    it 'has 10 elements' do
+      expect(subject.size).to eq(10)
+    end
+
+    it { is_expected.to include('files/ruby/popen.rb') }
+    it { is_expected.not_to include('LICENSE') }
+  end
+end

spec/lib/gitlab/git/diff_spec.rb

+require "spec_helper"
+
+describe Gitlab::Git::Diff, seed_helper: true do
+  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+
+  before do
+    @raw_diff_hash = {
+      diff: <<EOT.gsub(/^ {8}/, "").sub(/\n$/, ""),
+        --- a/.gitmodules
+        +++ b/.gitmodules
+        @@ -4,3 +4,6 @@
+         [submodule "gitlab-shell"]
+         \tpath = gitlab-shell
+         \turl = https://github.com/gitlabhq/gitlab-shell.git
+        +[submodule "gitlab-grack"]
+        +	path = gitlab-grack
+        +	url = https://gitlab.com/gitlab-org/gitlab-grack.git
+
+EOT
+      new_path: ".gitmodules",
+      old_path: ".gitmodules",
+      a_mode: '100644',
+      b_mode: '100644',
+      new_file: false,
+      renamed_file: false,
+      deleted_file: false,
+      too_large: false
+    }
+
+    @rugged_diff = repository.rugged.diff("5937ac0a7beb003549fc5fd26fc247adbce4a52e^", "5937ac0a7beb003549fc5fd26fc247adbce4a52e", paths:
+                                          [".gitmodules"]).patches.first
+  end
+
+  describe '.new' do
+    context 'using a Hash' do
+      context 'with a small diff' do
+        let(:diff) { described_class.new(@raw_diff_hash) }
+
+        it 'initializes the diff' do
+          expect(diff.to_hash).to eq(@raw_diff_hash)
+        end
+
+        it 'does not prune the diff' do
+          expect(diff).not_to be_too_large
+        end
+      end
+
+      context 'using a diff that is too large' do
+        it 'prunes the diff' do
+          diff = described_class.new(diff: 'a' * 204800)
+
+          expect(diff.diff).to be_empty
+          expect(diff).to be_too_large
+        end
+      end
+    end
+
+    context 'using a Rugged::Patch' do
+      context 'with a small diff' do
+        let(:diff) { described_class.new(@rugged_diff) }
+
+        it 'initializes the diff' do
+          expect(diff.to_hash).to eq(@raw_diff_hash.merge(too_large: nil))
+        end
+
+        it 'does not prune the diff' do
+          expect(diff).not_to be_too_large
+        end
+      end
+
+      context 'using a diff that is too large' do
+        it 'prunes the diff' do
+          expect_any_instance_of(String).to receive(:bytesize).
+            and_return(1024 * 1024 * 1024)
+
+          diff = described_class.new(@rugged_diff)
+
+          expect(diff.diff).to be_empty
+          expect(diff).to be_too_large
+        end
+      end
+
+      context 'using a collapsable diff that is too large' do
+        before do
+          # The patch total size is 200, with lines between 21 and 54.
+          # This is a quick-and-dirty way to test this. Ideally, a new patch is
+          # added to the test repo with a size that falls between the real limits.
+          stub_const("#{described_class}::DIFF_SIZE_LIMIT", 150)
+          stub_const("#{described_class}::DIFF_COLLAPSE_LIMIT", 100)
+        end
+
+        it 'prunes the diff as a large diff instead of as a collapsed diff' do
+          diff = described_class.new(@rugged_diff, collapse: true)
+
+          expect(diff.diff).to be_empty
+          expect(diff).to be_too_large
+          expect(diff).not_to be_collapsed
+        end
+      end
+
+      context 'using a large binary diff' do
+        it 'does not prune the diff' do
+          expect_any_instance_of(Rugged::Diff::Delta).to receive(:binary?).
+            and_return(true)
+
+          diff = described_class.new(@rugged_diff)
+
+          expect(diff.diff).not_to be_empty
+        end
+      end
+    end
+  end
+
+  describe 'straight diffs' do
+    let(:options) { { straight: true } }
+    let(:diffs) { described_class.between(repository, 'feature', 'master', options) }
+
+    it 'has the correct size' do
+      expect(diffs.size).to eq(24)
+    end
+
+    context 'diff' do
+      it 'is an instance of Diff' do
+        expect(diffs.first).to be_kind_of(described_class)
+      end
+
+      it 'has the correct new_path' do
+        expect(diffs.first.new_path).to eq('.DS_Store')
+      end
+
+      it 'has the correct diff' do
+        expect(diffs.first.diff).to include('Binary files /dev/null and b/.DS_Store differ')
+      end
+    end
+  end
+
+  describe '.between' do
+    let(:diffs) { described_class.between(repository, 'feature', 'master') }
+    subject { diffs }
+
+    it { is_expected.to be_kind_of Gitlab::Git::DiffCollection }
+
+    describe '#size' do
+      subject { super().size }
+
+      it { is_expected.to eq(1) }
+    end
+
+    context 'diff' do
+      subject { diffs.first }
+
+      it { is_expected.to be_kind_of described_class }
+
+      describe '#new_path' do
+        subject { super().new_path }
+
+        it { is_expected.to eq('files/ruby/feature.rb') }
+      end
+
+      describe '#diff' do
+        subject { super().diff }
+
+        it { is_expected.to include '+class Feature' }
+      end
+    end
+  end
+
+  describe '.filter_diff_options' do
+    let(:options) { { max_size: 100, invalid_opt: true } }
+
+    context "without default options" do
+      let(:filtered_options) { described_class.filter_diff_options(options) }
+
+      it "should filter invalid options" do
+        expect(filtered_options).not_to have_key(:invalid_opt)
+      end
+    end
+
+    context "with default options" do
+      let(:filtered_options) do
+        default_options = { max_size: 5, bad_opt: 1, ignore_whitespace: true }
+        described_class.filter_diff_options(options, default_options)
+      end
+
+      it "should filter invalid options" do
+        expect(filtered_options).not_to have_key(:invalid_opt)
+        expect(filtered_options).not_to have_key(:bad_opt)
+      end
+
+      it "should merge with default options" do
+        expect(filtered_options).to have_key(:ignore_whitespace)
+      end
+
+      it "should override default options" do
+        expect(filtered_options).to have_key(:max_size)
+        expect(filtered_options[:max_size]).to eq(100)
+      end
+    end
+  end
+
+  describe '#submodule?' do
+    before do
+      commit = repository.lookup('5937ac0a7beb003549fc5fd26fc247adbce4a52e')
+      @diffs = commit.parents[0].diff(commit).patches
+    end
+
+    it { expect(described_class.new(@diffs[0]).submodule?).to eq(false) }
+    it { expect(described_class.new(@diffs[1]).submodule?).to eq(true) }
+  end
+
+  describe '#line_count' do
+    it 'returns the correct number of lines' do
+      diff = described_class.new(@rugged_diff)
+
+      expect(diff.line_count).to eq(9)
+    end
+  end
+
+  describe '#too_large?' do
+    it 'returns true for a diff that is too large' do
+      diff = described_class.new(diff: 'a' * 204800)
+
+      expect(diff.too_large?).to eq(true)
+    end
+
+    it 'returns false for a diff that is small enough' do
+      diff = described_class.new(diff: 'a')
+
+      expect(diff.too_large?).to eq(false)
+    end
+
+    it 'returns true for a diff that was explicitly marked as being too large' do
+      diff = described_class.new(diff: 'a')
+
+      diff.prune_large_diff!
+
+      expect(diff.too_large?).to eq(true)
+    end
+  end
+
+  describe '#collapsed?' do
+    it 'returns false by default even on quite big diff' do
+      diff = described_class.new(diff: 'a' * 20480)
+
+      expect(diff).not_to be_collapsed
+    end
+
+    it 'returns false by default for a diff that is small enough' do
+      diff = described_class.new(diff: 'a')
+
+      expect(diff).not_to be_collapsed
+    end
+
+    it 'returns true for a diff that was explicitly marked as being collapsed' do
+      diff = described_class.new(diff: 'a')
+
+      diff.prune_collapsed_diff!
+
+      expect(diff).to be_collapsed
+    end
+  end
+
+  describe '#collapsible?' do
+    it 'returns true for a diff that is quite large' do
+      diff = described_class.new(diff: 'a' * 20480)
+
+      expect(diff).to be_collapsible
+    end
+
+    it 'returns false for a diff that is small enough' do
+      diff = described_class.new(diff: 'a')
+
+      expect(diff).not_to be_collapsible
+    end
+  end
+
+  describe '#prune_collapsed_diff!' do
+    it 'prunes the diff' do
+      diff = described_class.new(diff: "foo\nbar")
+
+      diff.prune_collapsed_diff!
+
+      expect(diff.diff).to eq('')
+      expect(diff.line_count).to eq(0)
+    end
+  end
+end

spec/lib/gitlab/git/encoding_helper_spec.rb

+require "spec_helper"
+
+describe Gitlab::Git::EncodingHelper do
+  let(:ext_class) { Class.new { extend Gitlab::Git::EncodingHelper } }
+  let(:binary_string) { File.join(SEED_REPOSITORY_PATH, 'gitlab_logo.png') }
+
+  describe '#encode!' do
+    [
+      [
+        'leaves ascii only string as is',
+        'ascii only string',
+        'ascii only string'
+      ],
+      [
+        'leaves valid utf8 string as is',
+        'multibyte string №∑∉',
+        'multibyte string №∑∉'
+      ],
+      [
+        'removes invalid bytes from ASCII-8bit encoded multibyte string. This can occur when a git diff match line truncates in the middle of a multibyte character. This occurs after the second word in this example. The test string is as short as we can get while still triggering the error condition when not looking at `detect[:confidence]`.',
+        "mu ns\xC3\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi ".force_encoding('ASCII-8BIT'),
+        "mu ns\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi ",
+      ],
+    ].each do |description, test_string, xpect|
+      it description do
+        expect(ext_class.encode!(test_string)).to eq(xpect)
+      end
+    end
+
+    it 'leaves binary string as is' do
+      expect(ext_class.encode!(binary_string)).to eq(binary_string)
+    end
+  end
+
+  describe '#encode_utf8' do
+    [
+      [
+        "encodes valid utf8 encoded string to utf8",
+        "λ, λ, λ".encode("UTF-8"),
+        "λ, λ, λ".encode("UTF-8"),
+      ],
+      [
+        "encodes valid ASCII-8BIT encoded string to utf8",
+        "ascii only".encode("ASCII-8BIT"),
+        "ascii only".encode("UTF-8"),
+      ],
+      [
+        "encodes valid ISO-8859-1 encoded string to utf8",
+        "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("ISO-8859-1", "UTF-8"),
+        "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("UTF-8"),
+      ],
+    ].each do |description, test_string, xpect|
+      it description do
+        r = ext_class.encode_utf8(test_string.force_encoding('UTF-8'))
+        expect(r).to eq(xpect)
+        expect(r.encoding.name).to eq('UTF-8')
+      end
+    end
+  end
+
+  describe '#clean' do
+    [
+      [
+        'leaves ascii only string as is',
+        'ascii only string',
+        'ascii only string'
+      ],
+      [
+        'leaves valid utf8 string as is',
+        'multibyte string №∑∉',
+        'multibyte string №∑∉'
+      ],
+      [
+        'removes invalid bytes from ASCII-8bit encoded multibyte string.',
+        "Lorem ipsum\xC3\n dolor sit amet, xy\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg".force_encoding('ASCII-8BIT'),
+        "Lorem ipsum\n dolor sit amet, xyàyùabcdùefg",
+      ],
+    ].each do |description, test_string, xpect|
+      it description do
+        expect(ext_class.encode!(test_string)).to eq(xpect)
+      end
+    end
+  end
+end

spec/lib/gitlab/git/tag_spec.rb

+require "spec_helper"
+
+describe Gitlab::Git::Tag, seed_helper: true do
+  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+
+  describe 'first tag' do
+    let(:tag) { repository.tags.first }
+
+    it { expect(tag.name).to eq("v1.0.0") }
+    it { expect(tag.target).to eq("f4e6814c3e4e7a0de82a9e7cd20c626cc963a2f8") }
+    it { expect(tag.dereferenced_target.sha).to eq("6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9") }
+    it { expect(tag.message).to eq("Release") }
+  end
+
+  describe 'last tag' do
+    let(:tag) { repository.tags.last }
+
+    it { expect(tag.name).to eq("v1.2.1") }
+    it { expect(tag.target).to eq("2ac1f24e253e08135507d0830508febaaccf02ee") }
+    it { expect(tag.dereferenced_target.sha).to eq("fa1b1e6c004a68b7d8763b86455da9e6b23e36d6") }
+    it { expect(tag.message).to eq("Version 1.2.1") }
+  end
+
+  it { expect(repository.tags.size).to eq(SeedRepo::Repo::TAGS.size) }
+end

spec/lib/gitlab/git/tree_spec.rb

+require "spec_helper"
+
+describe Gitlab::Git::Tree, seed_helper: true do
+  context :repo do
+    let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+    let(:tree) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID) }
+
+    it { expect(tree).to be_kind_of Array }
+    it { expect(tree.empty?).to be_falsey }
+    it { expect(tree.select(&:dir?).size).to eq(2) }
+    it { expect(tree.select(&:file?).size).to eq(10) }
+    it { expect(tree.select(&:submodule?).size).to eq(2) }
+
+    describe :dir do
+      let(:dir) { tree.select(&:dir?).first }
+
+      it { expect(dir).to be_kind_of Gitlab::Git::Tree }
+      it { expect(dir.id).to eq('3c122d2b7830eca25235131070602575cf8b41a1') }
+      it { expect(dir.commit_id).to eq(SeedRepo::Commit::ID) }
+      it { expect(dir.name).to eq('encoding') }
+      it { expect(dir.path).to eq('encoding') }
+
+      context :subdir do
+        let(:subdir) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID, 'files').first }
+
+        it { expect(subdir).to be_kind_of Gitlab::Git::Tree }
+        it { expect(subdir.id).to eq('a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba') }
+        it { expect(subdir.commit_id).to eq(SeedRepo::Commit::ID) }
+        it { expect(subdir.name).to eq('html') }
+        it { expect(subdir.path).to eq('files/html') }
+      end
+
+      context :subdir_file do
+        let(:subdir_file) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID, 'files/ruby').first }
+
+        it { expect(subdir_file).to be_kind_of Gitlab::Git::Tree }
+        it { expect(subdir_file.id).to eq('7e3e39ebb9b2bf433b4ad17313770fbe4051649c') }
+        it { expect(subdir_file.commit_id).to eq(SeedRepo::Commit::ID) }
+        it { expect(subdir_file.name).to eq('popen.rb') }
+        it { expect(subdir_file.path).to eq('files/ruby/popen.rb') }
+      end
+    end
+
+    describe :file do
+      let(:file) { tree.select(&:file?).first }
+
+      it { expect(file).to be_kind_of Gitlab::Git::Tree }
+      it { expect(file.id).to eq('dfaa3f97ca337e20154a98ac9d0be76ddd1fcc82') }
+      it { expect(file.commit_id).to eq(SeedRepo::Commit::ID) }
+      it { expect(file.name).to eq('.gitignore') }
+    end
+
+    describe :readme do
+      let(:file) { tree.select(&:readme?).first }
+
+      it { expect(file).to be_kind_of Gitlab::Git::Tree }
+      it { expect(file.name).to eq('README.md') }
+    end
+
+    describe :contributing do
+      let(:file) { tree.select(&:contributing?).first }
+
+      it { expect(file).to be_kind_of Gitlab::Git::Tree }
+      it { expect(file.name).to eq('CONTRIBUTING.md') }
+    end
+
+    describe :submodule do
+      let(:submodule) { tree.select(&:submodule?).first }
+
+      it { expect(submodule).to be_kind_of Gitlab::Git::Tree }
+      it { expect(submodule.id).to eq('79bceae69cb5750d6567b223597999bfa91cb3b9') }
+      it { expect(submodule.commit_id).to eq('570e7b2abdd848b95f2f578043fc23bd6f6fd24d') }
+      it { expect(submodule.name).to eq('gitlab-shell') }
+    end
+  end
+end

spec/lib/gitlab/git/util_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Git::Util do
+  describe :count_lines do
+    [
+      ["", 0],
+      ["foo", 1],
+      ["foo\n", 1],
+      ["foo\n\n", 2],
+    ].each do |string, line_count|
+      it "counts #{line_count} lines in #{string.inspect}" do
+        expect(Gitlab::Git::Util.count_lines(string)).to eq(line_count)
+      end
+    end
+  end
+end

spec/support/matchers/be_valid_commit.rb

+RSpec::Matchers.define :be_valid_commit do
+  match do |actual|
+    actual &&
+      actual.id == SeedRepo::Commit::ID &&
+      actual.message == SeedRepo::Commit::MESSAGE &&
+      actual.author_name == SeedRepo::Commit::AUTHOR_FULL_NAME
+  end
+end

spec/support/seed_helper.rb

+# This file is specific to specs in spec/lib/gitlab/git/
+
+SEED_REPOSITORY_PATH   = File.expand_path('../../tmp/repositories', __dir__)
+TEST_REPO_PATH         = File.join(SEED_REPOSITORY_PATH, 'gitlab-git-test.git')
+TEST_NORMAL_REPO_PATH  = File.join(SEED_REPOSITORY_PATH, "not-bare-repo.git")
+TEST_MUTABLE_REPO_PATH = File.join(SEED_REPOSITORY_PATH, "mutable-repo.git")
+TEST_BROKEN_REPO_PATH  = File.join(SEED_REPOSITORY_PATH, "broken-repo.git")
+
+module SeedHelper
+  GITLAB_URL = "https://gitlab.com/gitlab-org/gitlab-git-test.git"
+
+  def ensure_seeds
+    if File.exist?(SEED_REPOSITORY_PATH)
+      FileUtils.rm_r(SEED_REPOSITORY_PATH)
+    end
+
+    FileUtils.mkdir_p(SEED_REPOSITORY_PATH)
+
+    create_bare_seeds
+    create_normal_seeds
+    create_mutable_seeds
+    create_broken_seeds
+    create_git_attributes
+    create_invalid_git_attributes
+  end
+
+  def create_bare_seeds
+    system(git_env, *%W(git clone --bare #{GITLAB_URL}),
+           chdir: SEED_REPOSITORY_PATH,
+           out:   '/dev/null',
+           err:   '/dev/null')
+  end
+
+  def create_normal_seeds
+    system(git_env, *%W(git clone #{TEST_REPO_PATH} #{TEST_NORMAL_REPO_PATH}),
+           out: '/dev/null',
+           err: '/dev/null')
+  end
+
+  def create_mutable_seeds
+    system(git_env, *%W(git clone #{TEST_REPO_PATH} #{TEST_MUTABLE_REPO_PATH}),
+           out: '/dev/null',
+           err: '/dev/null')
+
+    system(git_env, *%w(git branch -t feature origin/feature),
+           chdir: TEST_MUTABLE_REPO_PATH, out: '/dev/null', err: '/dev/null')
+
+    system(git_env, *%W(git remote add expendable #{GITLAB_URL}),
+           chdir: TEST_MUTABLE_REPO_PATH, out: '/dev/null', err: '/dev/null')
+  end
+
+  def create_broken_seeds
+    system(git_env, *%W(git clone --bare #{TEST_REPO_PATH} #{TEST_BROKEN_REPO_PATH}),
+           out: '/dev/null',
+           err: '/dev/null')
+
+    refs_path = File.join(TEST_BROKEN_REPO_PATH, 'refs')
+
+    FileUtils.rm_r(refs_path)
+  end
+
+  def create_git_attributes
+    dir = File.join(SEED_REPOSITORY_PATH, 'with-git-attributes.git', 'info')
+
+    FileUtils.mkdir_p(dir)
+
+    File.open(File.join(dir, 'attributes'), 'w') do |handle|
+      handle.write <<-EOF.strip
+# This is a comment, it should be ignored.
+
+*.txt     text
+*.jpg     -text
+*.sh      eol=lf gitlab-language=shell
+*.haml.*  gitlab-language=haml
+foo/bar.* foo
+*.cgi     key=value?p1=v1&p2=v2
+/*.png    gitlab-language=png
+*.binary  binary
+
+# This uses a tab instead of spaces to ensure the parser also supports this.
+*.md\tgitlab-language=markdown
+bla/bla.txt
+      EOF
+    end
+  end
+
+  def create_invalid_git_attributes
+    dir = File.join(SEED_REPOSITORY_PATH, 'with-invalid-git-attributes.git', 'info')
+
+    FileUtils.mkdir_p(dir)
+
+    enc = Encoding::UTF_16
+
+    File.open(File.join(dir, 'attributes'), 'w', encoding: enc) do |handle|
+      handle.write('# hello'.encode(enc))
+    end
+  end
+
+  # Prevent developer git configurations from being persisted to test
+  # repositories
+  def git_env
+    { 'GIT_TEMPLATE_DIR' => '' }
+  end
+end
+
+RSpec.configure do |config|
+  config.include SeedHelper, :seed_helper
+
+  config.before(:all, :seed_helper) do
+    ensure_seeds
+  end
+end

spec/support/seed_repo.rb

+# Seed repo:
+# 0e50ec4d3c7ce42ab74dda1d422cb2cbffe1e326 Merge branch 'lfs_pointers' into 'master'
+# 33bcff41c232a11727ac6d660bd4b0c2ba86d63d Add valid and invalid lfs pointers
+# 732401c65e924df81435deb12891ef570167d2e2 Update year in license file
+# b0e52af38d7ea43cf41d8a6f2471351ac036d6c9 Empty commit
+# 40f4a7a617393735a95a0bb67b08385bc1e7c66d Add ISO-8859-encoded file
+# 66028349a123e695b589e09a36634d976edcc5e8 Merge branch 'add-comments-to-gitmodules' into 'master'
+# de5714f34c4e34f1d50b9a61a2e6c9132fe2b5fd Add comments to the end of .gitmodules to test parsing
+# fa1b1e6c004a68b7d8763b86455da9e6b23e36d6 Merge branch 'add-files' into 'master'
+# eb49186cfa5c4338011f5f590fac11bd66c5c631 Add submodules nested deeper than the root
+# 18d9c205d0d22fdf62bc2f899443b83aafbf941f Add executables and links files
+# 5937ac0a7beb003549fc5fd26fc247adbce4a52e Add submodule from gitlab.com
+# 570e7b2abdd848b95f2f578043fc23bd6f6fd24d Change some files
+# 6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9 More submodules
+# d14d6c0abdd253381df51a723d58691b2ee1ab08 Remove ds_store files
+# c1acaa58bbcbc3eafe538cb8274ba387047b69f8 Ignore DS files
+# ae73cb07c9eeaf35924a10f713b364d32b2dd34f Binary file added
+# 874797c3a73b60d2187ed6e2fcabd289ff75171e Ruby files modified
+# 2f63565e7aac07bcdadb654e253078b727143ec4 Modified image
+# 33f3729a45c02fc67d00adb1b8bca394b0e761d9 Image added
+# 913c66a37b4a45b9769037c55c2d238bd0942d2e Files, encoding and much more
+# cfe32cf61b73a0d5e9f13e774abde7ff789b1660 Add submodule
+# 6d394385cf567f80a8fd85055db1ab4c5295806f Added contributing guide
+# 1a0b36b3cdad1d2ee32457c102a8c0b7056fa863 Initial commit
+
+module SeedRepo
+  module BigCommit
+    ID               = "913c66a37b4a45b9769037c55c2d238bd0942d2e"
+    PARENT_ID        = "cfe32cf61b73a0d5e9f13e774abde7ff789b1660"
+    MESSAGE          = "Files, encoding and much more"
+    AUTHOR_FULL_NAME = "Dmitriy Zaporozhets"
+    FILES_COUNT      = 2
+  end
+
+  module Commit
+    ID               = "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
+    PARENT_ID        = "6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9"
+    MESSAGE          = "Change some files\n\nSigned-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>\n"
+    AUTHOR_FULL_NAME = "Dmitriy Zaporozhets"
+    FILES            = ["files/ruby/popen.rb", "files/ruby/regex.rb"]
+    FILES_COUNT      = 2
+    C_FILE_PATH      = "files/ruby"
+    C_FILES          = ["popen.rb", "regex.rb", "version_info.rb"]
+    BLOB_FILE        = %{%h3= @key.title\n%hr\n%pre= @key.key\n.actions\n  = link_to 'Remove', @key, :confirm => 'Are you sure?', :method => :delete, :class => \"btn danger delete-key\"\n\n\n}
+    BLOB_FILE_PATH   = "app/views/keys/show.html.haml"
+  end
+
+  module EmptyCommit
+    ID               = "b0e52af38d7ea43cf41d8a6f2471351ac036d6c9"
+    PARENT_ID        = "40f4a7a617393735a95a0bb67b08385bc1e7c66d"
+    MESSAGE          = "Empty commit"
+    AUTHOR_FULL_NAME = "Rémy Coutable"
+    FILES            = []
+    FILES_COUNT      = FILES.count
+  end
+
+  module EncodingCommit
+    ID               = "40f4a7a617393735a95a0bb67b08385bc1e7c66d"
+    PARENT_ID        = "66028349a123e695b589e09a36634d976edcc5e8"
+    MESSAGE          = "Add ISO-8859-encoded file"
+    AUTHOR_FULL_NAME = "Stan Hu"
+    FILES            = ["encoding/iso8859.txt"]
+    FILES_COUNT      = FILES.count
+  end
+
+  module FirstCommit
+    ID               = "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863"
+    PARENT_ID        = nil
+    MESSAGE          = "Initial commit"
+    AUTHOR_FULL_NAME = "Dmitriy Zaporozhets"
+    FILES            = ["LICENSE", ".gitignore", "README.md"]
+    FILES_COUNT      = 3
+  end
+
+  module LastCommit
+    ID               = "4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6"
+    PARENT_ID        = "0e1b353b348f8477bdbec1ef47087171c5032cd9"
+    MESSAGE          = "Merge branch 'master' into 'master'"
+    AUTHOR_FULL_NAME = "Stan Hu"
+    FILES            = ["bin/executable"]
+    FILES_COUNT      = FILES.count
+  end
+
+  module Repo
+    HEAD = "master"
+    BRANCHES = %w[
+      feature
+      fix
+      fix-blob-path
+      fix-existing-submodule-dir
+      fix-mode
+      gitattributes
+      gitattributes-updated
+      master
+      merge-test
+    ]
+    TAGS = %w[v1.0.0 v1.1.0 v1.2.0 v1.2.1]
+  end
+
+  module RubyBlob
+    ID = "7e3e39ebb9b2bf433b4ad17313770fbe4051649c"
+    NAME = "popen.rb"
+    CONTENT = <<-eos
+require 'fileutils'
+require 'open3'
+
+module Popen
+  extend self
+
+  def popen(cmd, path=nil)
+    unless cmd.is_a?(Array)
+      raise RuntimeError, "System commands must be given as an array of strings"
+    end
+
+    path ||= Dir.pwd
+
+    vars = {
+      "PWD" => path
+    }
+
+    options = {
+      chdir: path
+    }
+
+    unless File.directory?(path)
+      FileUtils.mkdir_p(path)
+    end
+
+    @cmd_output = ""
+    @cmd_status = 0
+
+    Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+      @cmd_output << stdout.read
+      @cmd_output << stderr.read
+      @cmd_status = wait_thr.value.exitstatus
+    end
+
+    return @cmd_output, @cmd_status
+  end
+end
+    eos
+  end
+end