TokenFactory和Tokenizer基础

tests/Tokenizer/TokenizerTest.php

@@ -0,0 +1,50 @@
+<?php
+/**
+ * @filename TokenizerTest.php
+ * @author Jerry Yan <792602257@qq.com>
+ * @date 2021/1/22 13:33
+ */
+
+namespace JerryYan\DSL\Test\Tokenizer;
+
+use JerryYan\DSL\Reader\StringReader;
+use JerryYan\DSL\Token\Factory\DefaultFactory;
+use JerryYan\DSL\Token\TokenAnd;
+use JerryYan\DSL\Token\TokenInterface;
+use JerryYan\DSL\Token\TokenOr;
+use JerryYan\DSL\Token\TokenVar;
+use JerryYan\DSL\Tokenizer\Tokenizer;
+use PHPUnit\Framework\TestCase;
+
+class TokenizerTest extends TestCase
+{
+    protected $tokenizer;
+    protected $reader;
+    private $text = "这个 和 那个 或者 那个 和 这个";
+    private $textTokenType = [
+        TokenVar::class,
+        TokenAnd::class,
+        TokenVar::class,
+        TokenOr::class,
+        TokenVar::class,
+        TokenAnd::class,
+        TokenVar::class,
+    ];
+    protected function setUp(): void
+    {
+        $this->tokenizer = new Tokenizer(new DefaultFactory());
+        $this->reader = new StringReader($this->text);
+    }
+
+    public function testTokenize()
+    {
+        $tokens = $this->tokenizer->tokenize($this->reader);
+        $this->assertInstanceOf(TokenInterface::class, $tokens);
+        $index = 0;
+        do {
+            $this->assertInstanceOf($this->textTokenType[$index], $tokens);
+            $tokens = $tokens->getNextToken();
+            $index ++;
+        } while ($tokens->hasNextToken());
+    }
+}