Convert unicode emojis to images.

CHANGELOG

@@ -14,6 +14,7 @@ v 8.13.0 (unreleased)
   - AbstractReferenceFilter caches project_refs on RequestStore when active
   - Replaced the check sign to arrow in the show build view. !6501
   - Add a /wip slash command to toggle the Work In Progress status of a merge request. !6259 (tbalthazar)
+  - Fix Error 500 when viewing old merge requests with bad diff data
   - Speed-up group milestones show page
   - Fix inconsistent options dropdown caret on mobile viewports (ClemMakesApps)
   - Don't include archived projects when creating group milestones. !4940 (Jeroen Jacobs)

app/models/merge_request_diff.rb

@@ -6,6 +6,9 @@ class MergeRequestDiff < ActiveRecord::Base
   # Prevent store of diff if commits amount more then 500
   COMMITS_SAFE_SIZE = 100
 
+  # Valid types of serialized diffs allowed by Gitlab::Git::Diff
+  VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta]
+
   belongs_to :merge_request
 
   state_machine :state, initial: :empty do
@@ -170,6 +173,15 @@ class MergeRequestDiff < ActiveRecord::Base
 
   private
 
+  # Old GitLab implementations may have generated diffs as ["--broken-diff"].
+  # Avoid an error 500 by ignoring bad elements. See:
+  # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776
+  def valid_raw_diff?(raw)
+    return false unless raw.respond_to?(:each)
+
+    raw.any? { |element| VALID_CLASSES.include?(element.class) }
+  end
+
   def dump_commits(commits)
     commits.map(&:to_hash)
   end
@@ -200,7 +212,7 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def load_diffs(raw, options)
-    if raw.respond_to?(:each)
+    if valid_raw_diff?(raw)
       if paths = options[:paths]
         raw = raw.select do |diff|
           paths.include?(diff[:old_path]) || paths.include?(diff[:new_path])

doc/ci/pipelines.md

@@ -31,6 +31,8 @@ project.
 ## Seeing build status
 
 Clicking on a pipeline will show the builds that were run for that pipeline.
+Clicking on an individual build will show you its build trace, and allow you to
+cancel the build, retry it,  or erase the build trace.
 
 ## Badges
 

doc/ci/variables/README.md

@@ -48,6 +48,7 @@ The `API_TOKEN` will take the Secure Variable value: `SECURE`.
 | **CI_RUNNER_ID**        | 8.10   | 0.5    | The unique id of runner being used |
 | **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5    | The description of the runner as saved in GitLab |
 | **CI_RUNNER_TAGS**      | 8.10   | 0.5    | The defined runner tags |
+| **CI_DEBUG_TRACE**      | all    | 1.7    | Whether [debug tracing](#debug-tracing) is enabled |
 | **GITLAB_USER_ID**      | 8.12   | all    | The id of the user who started the build |
 | **GITLAB_USER_EMAIL**   | 8.12   | all    | The email of the user who started the build |
 
@@ -105,6 +106,39 @@ Variables can be defined at a global level, but also at a job level.
 
 More information about Docker integration can be found in [Using Docker Images](../docker/using_docker_images.md).
 
+#### Debug tracing
+
+> **WARNING:** Enabling debug tracing can have severe security implications. The
+  output **will** contain the content of all your secure variables and any other
+  secrets! The output **will** be uploaded to the GitLab server and made visible
+  in build traces!
+
+By default, GitLab Runner hides most of the details of what it is doing when
+processing a job. This behaviour keeps build traces short, and prevents secrets
+from being leaked into the trace unless your script writes them to the screen.
+
+If a job isn't working as expected, this can make the problem difficult to
+investigate; in these cases, you can enable debug tracing in `.gitlab-ci.yml`.
+Available on GitLab Runner v1.7+, this feature enables the shell's execution
+trace, resulting in a verbose build trace listing all commands that were run,
+variables that were set, etc.
+
+Before enabling this, you should ensure builds are visible to
+[team members only](../../../user/permissions.md#project-features). You should
+also [erase](../pipelines.md#seeing-build-traces) all generated build traces
+before making them visible again.
+
+To enable debug traces, set the `CI_DEBUG_TRACE` variable to `true`:
+
+```yaml
+job1:
+  variables:
+    CI_DEBUG_TRACE: "true"
+```
+
+The [example project](https://gitlab.com/gitlab-examples/ci-debug-trace)
+demonstrates a working configuration, including build trace examples.
+
 ### User-defined variables (Secure Variables)
 **This feature requires GitLab Runner 0.4.0 or higher**
 

lib/banzai/filter/emoji_filter.rb

 module Banzai
   module Filter
-    # HTML filter that replaces :emoji: with images.
+    # HTML filter that replaces :emoji: and unicode with images.
     #
     # Based on HTML::Pipeline::EmojiFilter
     #
@@ -13,14 +13,14 @@ module Banzai
       def call
         search_text_nodes(doc).each do |node|
           content = node.to_html
-          next unless content.include?(':')
           next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS)
+          if content.include?(':') || node.text.match(emoji_unicode_pattern)
+            html = emoji_name_image_filter(content)
+            html = emoji_unicode_image_filter(html)
+            next if html == content
+            node.replace(html)
+          end
 
-          html = emoji_image_filter(content)
-
-          next if html == content
-
-          node.replace(html)
         end
 
         doc
@@ -31,18 +31,34 @@ module Banzai
       # text - String text to replace :emoji: in.
       #
       # Returns a String with :emoji: replaced with images.
-      def emoji_image_filter(text)
+      def emoji_name_image_filter(text)
         text.gsub(emoji_pattern) do |match|
           name = $1
           "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />"
         end
       end
 
+
+      # Replace unicode emojis with corresponding images if they exist.
+      #
+      # text - String text to replace unicode emojis in.
+      #
+      # Returns a String with unicode emojis replaced with images.
+
+      def emoji_unicode_image_filter(text)
+        text.gsub(emoji_unicode_pattern) do |moji|
+          "<img class='emoji' title=':#{Gitlab::Emoji.emojis_by_moji[moji]['name']}:' alt=':#{Gitlab::Emoji.emojis_by_moji[moji]['name']}:' src='#{emoji_unicode_url(moji)}' height='20' width='20' align='absmiddle' />"
+        end
+      end
       # Build a regexp that matches all valid :emoji: names.
       def self.emoji_pattern
         @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/
       end
+      # Build a regexp that matches all valid unicode emojis names.
+      def self.emoji_unicode_pattern
+        @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/
 
+      end
       private
 
       def emoji_url(name)
@@ -60,6 +76,21 @@ module Banzai
         end
       end
 
+      def emoji_unicode_url(moji)
+        emoji_unicode_path = emoji_unicode_filename(moji)
+
+        if context[:asset_host]
+          # Asset host is specified.
+          url_to_image(emoji_unicode_path)
+        elsif context[:asset_root]
+          # Gitlab url is specified
+          File.join(context[:asset_root], url_to_image(emoji_unicode_path))
+        else
+          # All other cases
+          url_to_image(emoji_unicode_path)
+        end
+      end
+
       def url_to_image(image)
         ActionController::Base.helpers.url_to_image(image)
       end
@@ -71,6 +102,13 @@ module Banzai
       def emoji_filename(name)
         "#{Gitlab::Emoji.emoji_filename(name)}.png"
       end
+      def emoji_unicode_pattern
+        self.class.emoji_unicode_pattern
+      end
+
+      def emoji_unicode_filename(name)
+        "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png"
+      end
     end
   end
 end

lib/gitlab/emoji.rb

@@ -9,7 +9,9 @@ module Gitlab
     def emojis_by_moji
       Gemojione.index.instance_variable_get(:@emoji_by_moji)
     end
-
+    def emojis_unicodes
+      emojis_by_moji.keys.sort
+    end
     def emojis_names
       emojis.keys.sort
     end
@@ -17,5 +19,8 @@ module Gitlab
     def emoji_filename(name)
       emojis[name]["unicode"]
     end
+    def emoji_unicode_filename(moji)
+      emojis_by_moji[moji]["unicode"]
+    end
   end
 end

spec/lib/banzai/filter/emoji_filter_spec.rb

@@ -12,11 +12,14 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     ActionController::Base.asset_host = @original_asset_host
   end
 
-  it 'replaces supported emoji' do
+  it 'replaces supported name emoji' do
     doc = filter('<p>:heart:</p>')
     expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/2764.png'
   end
-
+  it 'replaces supported unicode emoji' do
+    doc = filter('<p>❤️</p>')
+    expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/2764.png'
+  end
   it 'ignores unsupported emoji' do
     exp = act = '<p>:foo:</p>'
     doc = filter(act)
@@ -28,46 +31,96 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/1F44D.png'
   end
 
+  it 'correctly encodes unicode to the URL' do
+    doc = filter('<p>👍</p>')
+    expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/1F44D.png'
+  end
+
   it 'matches at the start of a string' do
     doc = filter(':+1:')
     expect(doc.css('img').size).to eq 1
   end
 
+  it 'unicode matches at the start of a string' do
+    doc = filter("'👍'")
+    expect(doc.css('img').size).to eq 1
+  end
+
   it 'matches at the end of a string' do
     doc = filter('This gets a :-1:')
     expect(doc.css('img').size).to eq 1
   end
 
+  it 'unicode matches at the end of a string' do
+    doc = filter('This gets a 👍')
+    expect(doc.css('img').size).to eq 1
+  end
+
   it 'matches with adjacent text' do
     doc = filter('+1 (:+1:)')
     expect(doc.css('img').size).to eq 1
   end
 
+  it 'unicode matches with adjacent text' do
+    doc = filter('+1 (👍)')
+    expect(doc.css('img').size).to eq 1
+  end
+
   it 'matches multiple emoji in a row' do
     doc = filter(':see_no_evil::hear_no_evil::speak_no_evil:')
     expect(doc.css('img').size).to eq 3
   end
 
+  it 'unicode matches multiple emoji in a row' do
+    doc = filter("'🙈🙉🙊'")
+    expect(doc.css('img').size).to eq 3
+  end
+
+  it 'mixed matches multiple emoji in a row' do
+    doc = filter("'🙈:see_no_evil:🙉:hear_no_evil:🙊:speak_no_evil:'")
+    expect(doc.css('img').size).to eq 6
+  end
+
   it 'has a title attribute' do
     doc = filter(':-1:')
     expect(doc.css('img').first.attr('title')).to eq ':-1:'
   end
 
+  it 'unicode has a title attribute' do
+    doc = filter("'👎'")
+    expect(doc.css('img').first.attr('title')).to eq ':thumbsdown:'
+  end
+
   it 'has an alt attribute' do
     doc = filter(':-1:')
     expect(doc.css('img').first.attr('alt')).to eq ':-1:'
   end
 
+  it 'unicode has an alt attribute' do
+    doc = filter("'👎'")
+    expect(doc.css('img').first.attr('alt')).to eq ':thumbsdown:'
+  end
+
   it 'has an align attribute' do
     doc = filter(':8ball:')
     expect(doc.css('img').first.attr('align')).to eq 'absmiddle'
   end
 
+  it 'unicode has an align attribute' do
+    doc = filter("'🎱'")
+    expect(doc.css('img').first.attr('align')).to eq 'absmiddle'
+  end
+
   it 'has an emoji class' do
     doc = filter(':cat:')
     expect(doc.css('img').first.attr('class')).to eq 'emoji'
   end
 
+  it 'unicode has an emoji class' do
+    doc = filter("'🐱'")
+    expect(doc.css('img').first.attr('class')).to eq 'emoji'
+  end
+
   it 'has height and width attributes' do
     doc = filter(':dog:')
     img = doc.css('img').first
@@ -76,12 +129,26 @@ describe Banzai::Filter::EmojiFilter, lib: true do
     expect(img.attr('height')).to eq '20'
   end
 
+  it 'unicode has height and width attributes' do
+    doc = filter("'🐶'")
+    img = doc.css('img').first
+
+    expect(img.attr('width')).to eq '20'
+    expect(img.attr('height')).to eq '20'
+  end
+
   it 'keeps whitespace intact' do
     doc = filter('This deserves a :+1:, big time.')
 
     expect(doc.to_html).to match(/^This deserves a <img.+>, big time\.\z/)
   end
 
+  it 'unicode keeps whitespace intact' do
+    doc = filter('This deserves a 🎱, big time.')
+
+    expect(doc.to_html).to match(/^This deserves a <img.+>, big time\.\z/)
+  end
+
   it 'uses a custom asset_root context' do
     root = Gitlab.config.gitlab.url + 'gitlab/root'
 

spec/models/merge_request_diff_spec.rb

@@ -44,6 +44,16 @@ describe MergeRequestDiff, models: true do
       end
     end
 
+    context 'when the raw diffs have invalid content' do
+      before { mr_diff.update_attributes(st_diffs: ["--broken-diff"]) }
+
+      it 'returns an empty DiffCollection' do
+        expect(mr_diff.raw_diffs.to_a).to be_empty
+        expect(mr_diff.raw_diffs).to be_a(Gitlab::Git::DiffCollection)
+        expect(mr_diff.raw_diffs).to be_empty
+      end
+    end
+
     context 'when the raw diffs exist' do
       it 'returns the diffs' do
         expect(mr_diff.raw_diffs).to be_a(Gitlab::Git::DiffCollection)