TokenizerLinter - fix lack of linting when code is cached

src/Linter/TokenizerLinter.php

@@ -12,6 +12,7 @@
 
 namespace PhpCsFixer\Linter;
 
+use PhpCsFixer\Tokenizer\CodeHasher;
 use PhpCsFixer\Tokenizer\Tokens;
 
 /**
@@ -52,8 +53,12 @@ final class TokenizerLinter implements LinterInterface
     public function lintSource($source)
     {
         try {
-            // it will throw ParseError on syntax error
-            // if not, it will cache the tokenized version of code, which is great for Runner
+            // To lint, we will parse the source into Tokens.
+            // During that process, it might throw ParseError.
+            // If it won't, cache of tokenized version of source will be kept, which is great for Runner.
+            // Yet, first we need to clear already existing cache to not hit it and lint the code indeed.
+            $codeHash = CodeHasher::calculateCodeHash($source);
+            Tokens::clearCache($codeHash);
             Tokens::fromCode($source);
 
             return new TokenizerLintingResult();

src/Tokenizer/CodeHasher.php

@@ -0,0 +1,38 @@
+<?php
+
+/*
+ * This file is part of PHP CS Fixer.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *     Dariusz Rumiński <dariusz.ruminski@gmail.com>
+ *
+ * This source file is subject to the MIT license that is bundled
+ * with this source code in the file LICENSE.
+ */
+
+namespace PhpCsFixer\Tokenizer;
+
+/**
+ * @author Dariusz Rumiński <dariusz.ruminski@gmail.com>
+ *
+ * @internal
+ */
+final class CodeHasher
+{
+    private function __construct()
+    {
+        // cannot create instance of util. class
+    }
+
+    /**
+     * Calculate hash for code.
+     *
+     * @param string $code
+     *
+     * @return string
+     */
+    public static function calculateCodeHash($code)
+    {
+        return (string) crc32($code);
+    }
+}

src/Tokenizer/Tokens.php

@@ -1195,7 +1195,7 @@ class Tokens extends \SplFixedArray
      */
     private static function calculateCodeHash($code)
     {
-        return (string) crc32($code);
+        return CodeHasher::calculateCodeHash($code);
     }
 
     /**

tests/Linter/AbstractLinterTestCase.php

@@ -13,6 +13,8 @@
 namespace PhpCsFixer\Tests\Linter;
 
 use PhpCsFixer\Linter\LinterInterface;
+use PhpCsFixer\Tokenizer\Token;
+use PhpCsFixer\Tokenizer\Tokens;
 use PHPUnit\Framework\TestCase;
 
 /**
@@ -24,6 +26,17 @@ abstract class AbstractLinterTestCase extends TestCase
 {
     abstract public function testIsAsync();
 
+    public function testLintingAfterTokenManipulation()
+    {
+        $linter = $this->createLinter();
+
+        $tokens = Tokens::fromCode("<?php \n#EOF\n");
+        $tokens->insertAt(1, new Token(array(T_NS_SEPARATOR, '\\')));
+
+        $this->setExpectedException('\PhpCsFixer\Linter\LintingException');
+        $linter->lintSource($tokens->generateCode())->check();
+    }
+
     /**
      * @param string      $file
      * @param null|string $errorRegExp

tests/Tokenizer/CodeHasherTest.php

@@ -0,0 +1,32 @@
+<?php
+
+/*
+ * This file is part of PHP CS Fixer.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *     Dariusz Rumiński <dariusz.ruminski@gmail.com>
+ *
+ * This source file is subject to the MIT license that is bundled
+ * with this source code in the file LICENSE.
+ */
+
+namespace PhpCsFixer\Tests\Tokenizer;
+
+use PhpCsFixer\Tokenizer\CodeHasher;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @author SpacePossum
+ *
+ * @internal
+ *
+ * @covers \PhpCsFixer\Tokenizer\CodeHasher
+ */
+final class CodeHasherTest extends TestCase
+{
+    public function testCodeHasher()
+    {
+        $this->assertSame('322920910', CodeHasher::calculateCodeHash('<?php echo 1;'));
+        $this->assertSame('322920910', CodeHasher::calculateCodeHash('<?php echo 1;')); // calling twice, hashes should always be the same when the input doesn't change.
+    }
+}