Add a lexeme for a string pipeline expression

lib/gitlab/ci/pipeline/expression/lexer.rb

@@ -2,32 +2,34 @@ module Gitlab
   module Ci
     module Pipeline
       module Expression
-        LEXEMES = [
-          Expression::Variable
-        ]
+        class Lexer
+          LEXEMES = [
+            Expression::Variable,
+            Expression::String
+          ]
 
-        MAX_CYCLES = 5
+          MAX_CYCLES = 5
+          SyntaxError = Class.new(StandardError)
 
-        class Lexer
           def initialize(statement)
             @scanner = StringScanner.new(statement)
             @tokens = []
           end
 
           def tokenize
-            @tokens.tap do
-              MAX_CYCLES.times do
-                LEXEMES.each do |lexeme|
-                  @scanner.scan(/\s+/) # ignore whitespace
-
-                  lexeme.scan(@scanner).tap do |token|
-                    @tokens.push(token) if token.present?
-                  end
+            MAX_CYCLES.times do
+              LEXEMES.each do |lexeme|
+                @scanner.scan(/\s+/) # ignore whitespace
 
-                  return @tokens if @scanner.eos?
+                lexeme.scan(@scanner).tap do |token|
+                  @tokens.push(token) if token.present?
                 end
+
+                return @tokens if @scanner.eos?
               end
             end
+
+            raise Lexer::SyntaxError unless @scanner.eos?
           end
         end
       end

lib/gitlab/ci/pipeline/expression/string.rb

@@ -3,7 +3,7 @@ module Gitlab
     module Pipeline
       module Expression
         class String < Expression::Lexeme
-          PATTERN = /("|')(?<value>.+)('|")/.freeze
+          PATTERN = /"(?<string>.+?)"/.freeze
 
           def initialize(value)
             @value = value

lib/gitlab/ci/pipeline/expression/token.rb

@@ -3,6 +3,8 @@ module Gitlab
     module Pipeline
       module Expression
         class Token
+          attr_reader :value, :type
+
           def initialize(value, type)
             @value = value
             @type = type

spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb

@@ -26,5 +26,21 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
       expect(tokens.size).to eq 2
       expect(tokens).to all(be_an_instance_of(token_class))
     end
+
+    it 'tokenizes multiple values with different tokens' do
+      tokens = described_class.new('$VARIABLE "text" "value"').tokenize
+
+      expect(tokens.size).to eq 3
+      expect(tokens.first.value).to eq '$VARIABLE'
+      expect(tokens.second.value).to eq '"text"'
+      expect(tokens.third.value).to eq '"value"'
+    end
+
+    it 'limits statement to 5 tokens' do
+      lexer = described_class.new("$V1 $V2 $V3 $V4 $V5 $V6")
+
+      expect { lexer.tokenize }
+        .to raise_error described_class::SyntaxError
+    end
   end
 end