Extract pipeline expressions parser to a separate class

lib/gitlab/ci/pipeline/expression/lexer.rb

@@ -17,7 +17,9 @@ module Gitlab
             @tokens = []
           end
 
-          def tokenize
+          def tokens
+            return @tokens if @tokens.any?
+
             MAX_CYCLES.times do
               LEXEMES.each do |lexeme|
                 @scanner.skip(/\s+/) # ignore whitespace
@@ -32,6 +34,10 @@ module Gitlab
 
             raise Lexer::SyntaxError unless @scanner.eos?
           end
+
+          def lexemes
+            tokens.map(&:to_lexeme)
+          end
         end
       end
     end

lib/gitlab/ci/pipeline/expression/parser.rb

+module Gitlab
+  module Ci
+    module Pipeline
+      module Expression
+        class Parser
+          def initialize(syntax)
+            if syntax.is_a?(Expression::Lexer)
+              @tokens = syntax.tokens
+            else
+              @tokens = syntax.to_a
+            end
+          end
+
+          def tree
+            if @tokens.many?
+              Expression::Equals.new(@tokens.first.build, @tokens.last.build)
+            else
+              @tokens.first.build
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/expression/statement.rb

@@ -23,31 +23,14 @@ module Gitlab
             end
           end
 
-          def tokens
-            @tokens ||= @lexer.tokenize
-          end
-
-          def lexemes
-            @lexemes ||= tokens.map(&:to_lexeme)
-          end
-
-          ##
-          # Our syntax is very simple, so we don't yet need to implement a
-          # recursive parser, we can use the most simple approach to create
-          # a reverse descent parse tree "by hand".
-          #
           def parse_tree
-            raise StatementError if lexemes.empty?
+            raise StatementError if @lexer.lexemes.empty?
 
-            unless GRAMMAR.find { |syntax| syntax == lexemes }
+            unless GRAMMAR.find { |syntax| syntax == @lexer.lexemes }
               raise StatementError, 'Unknown pipeline expression!'
             end
 
-            if tokens.many?
-              Expression::Equals.new(tokens.first.build, tokens.last.build)
-            else
-              tokens.first.build
-            end
+            Expression::Parser.new(@lexer).tree
           end
 
           def evaluate

spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb

@@ -5,30 +5,30 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
     Gitlab::Ci::Pipeline::Expression::Token
   end
 
-  describe '#tokenize' do
-    it 'tokenizes single value' do
-      tokens = described_class.new('$VARIABLE').tokenize
+  describe '#tokens' do
+    it 'tokenss single value' do
+      tokens = described_class.new('$VARIABLE').tokens
 
       expect(tokens).to be_one
       expect(tokens).to all(be_an_instance_of(token_class))
     end
 
     it 'does ignore whitespace characters' do
-      tokens = described_class.new("\t$VARIABLE ").tokenize
+      tokens = described_class.new("\t$VARIABLE ").tokens
 
       expect(tokens).to be_one
       expect(tokens).to all(be_an_instance_of(token_class))
     end
 
-    it 'tokenizes multiple values of the same token' do
-      tokens = described_class.new("$VARIABLE1 $VARIABLE2").tokenize
+    it 'tokenss multiple values of the same token' do
+      tokens = described_class.new("$VARIABLE1 $VARIABLE2").tokens
 
       expect(tokens.size).to eq 2
       expect(tokens).to all(be_an_instance_of(token_class))
     end
 
-    it 'tokenizes multiple values with different tokens' do
-      tokens = described_class.new('$VARIABLE "text" "value"').tokenize
+    it 'tokenss multiple values with different tokens' do
+      tokens = described_class.new('$VARIABLE "text" "value"').tokens
 
       expect(tokens.size).to eq 3
       expect(tokens.first.value).to eq '$VARIABLE'
@@ -36,8 +36,8 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
       expect(tokens.third.value).to eq '"value"'
     end
 
-    it 'tokenizes tokens and operators' do
-      tokens = described_class.new('$VARIABLE == "text"').tokenize
+    it 'tokenss tokens and operators' do
+      tokens = described_class.new('$VARIABLE == "text"').tokens
 
       expect(tokens.size).to eq 3
       expect(tokens.first.value).to eq '$VARIABLE'
@@ -48,15 +48,24 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
     it 'limits statement to 5 tokens' do
       lexer = described_class.new("$V1 $V2 $V3 $V4 $V5 $V6")
 
-      expect { lexer.tokenize }
+      expect { lexer.tokens }
         .to raise_error described_class::SyntaxError
     end
 
     it 'raises syntax error in case of finding unknown tokens' do
       lexer = described_class.new('$V1 123 $V2')
 
-      expect { lexer.tokenize }
+      expect { lexer.tokens }
         .to raise_error described_class::SyntaxError
     end
   end
+
+  describe '#lexemes' do
+    it 'returns an array of syntax lexemes' do
+      lexer = described_class.new('$VAR "text"')
+
+      expect(lexer.lexemes).to eq %w[variable string]
+    end
+  end
+
 end

spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Expression::Parser do
+  describe '#tree' do
+  end
+end

spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb

@@ -12,18 +12,6 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
     pipeline.variables.build([key: 'VARIABLE', value: 'my variable'])
   end
 
-  describe '#tokens' do
-    it 'returns raw tokens' do
-      expect(subject.tokens.size).to eq 2
-    end
-  end
-
-  describe '#lexemes' do
-    it 'returns an array of syntax lexemes' do
-      expect(subject.lexemes).to eq %w[variable string]
-    end
-  end
-
   describe '#parse_tree' do
     context 'when expression is empty' do
       let(:text) { '' }