Tokenizer - remove deprecations and legacy mode

.travis.yml

@@ -69,7 +69,7 @@ jobs:
             <<: *STANDARD_TEST_JOB
             stage: Test
             php: 7.1
-            env: SYMFONY_DEPRECATIONS_HELPER=weak PHP_CS_FIXER_TEST_USE_LEGACY_TOKENIZER=1 SYMFONY_VERSION="^4.0" MIN_STABILITY=dev
+            env: SYMFONY_DEPRECATIONS_HELPER=weak SYMFONY_VERSION="^4.0" MIN_STABILITY=dev
 
         -
             <<: *STANDARD_TEST_JOB

phpunit.xml.dist

@@ -49,6 +49,5 @@
         <ini name="zend.enable_gc" value="0"/>
         <ini name="memory_limit" value="1G"/>
         <env name="SKIP_LINT_TEST_CASES" value="0"/>
-        <env name="PHP_CS_FIXER_TEST_USE_LEGACY_TOKENIZER" value="0"/>
     </php>
 </phpunit>

src/Runner/Runner.php

@@ -164,8 +164,6 @@ final class Runner
         }
 
         $old = file_get_contents($file->getRealPath());
-
-        Tokens::setLegacyMode(false);
         $tokens = Tokens::fromCode($old);
         $oldHash = $tokens->getCodeHash();
 

src/Tokenizer/Token.php

@@ -114,36 +114,6 @@ class Token
         return $classTokens;
     }
 
-    /**
-     * Clear token at given index.
-     *
-     * Clearing means override token by empty string.
-     *
-     * @deprecated since 2.4
-     */
-    public function clear()
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0.', E_USER_DEPRECATED);
-        Tokens::setLegacyMode(true);
-
-        $this->content = '';
-        $this->id = null;
-        $this->isArray = false;
-    }
-
-    /**
-     * Clear internal flag if token was changed.
-     *
-     * @deprecated since 2.4
-     */
-    public function clearChanged()
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0.', E_USER_DEPRECATED);
-        Tokens::setLegacyMode(true);
-
-        $this->changed = false;
-    }
-
     /**
      * Check if token is equals to given one.
      *
@@ -371,20 +341,6 @@ class Token
         return $this->isGivenKind(self::getCastTokenKinds());
     }
 
-    /**
-     * Check if token was changed.
-     *
-     * @return bool
-     *
-     * @deprecated since 2.4
-     */
-    public function isChanged()
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0.', E_USER_DEPRECATED);
-
-        return $this->changed;
-    }
-
     /**
      * Check if token is one of classy tokens: T_CLASS, T_INTERFACE or T_TRAIT.
      *
@@ -407,20 +363,6 @@ class Token
         return $this->isGivenKind($commentTokens);
     }
 
-    /**
-     * Check if token is empty, e.g. because of clearing.
-     *
-     * @return bool
-     *
-     * @deprecated since 2.4
-     */
-    public function isEmpty()
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0.', E_USER_DEPRECATED);
-
-        return null === $this->id && ('' === $this->content || null === $this->content);
-    }
-
     /**
      * Check if token is one of given kind.
      *
@@ -491,66 +433,6 @@ class Token
         return '' === trim($this->content, $whitespaces);
     }
 
-    /**
-     * Override token.
-     *
-     * If called on Token inside Tokens collection please use `Tokens::overrideAt` instead.
-     *
-     * @param array|string|Token $other token prototype
-     *
-     * @deprecated since 2.4
-     */
-    public function override($other)
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0.', E_USER_DEPRECATED);
-        Tokens::setLegacyMode(true);
-
-        $prototype = $other instanceof self ? $other->getPrototype() : $other;
-
-        if ($this->equals($prototype)) {
-            return;
-        }
-
-        $this->changed = true;
-
-        if (is_array($prototype)) {
-            $this->isArray = true;
-            $this->id = $prototype[0];
-            $this->content = $prototype[1];
-
-            return;
-        }
-
-        $this->isArray = false;
-        $this->id = null;
-        $this->content = $prototype;
-    }
-
-    /**
-     * @param string $content
-     *
-     * @deprecated since 2.4
-     */
-    public function setContent($content)
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0.', E_USER_DEPRECATED);
-        Tokens::setLegacyMode(true);
-
-        if ($this->content === $content) {
-            return;
-        }
-
-        $this->changed = true;
-        $this->content = $content;
-
-        // setting empty content is clearing the token
-        if ('' === $content) {
-            @trigger_error(__METHOD__.' shall not be used to clear token, use Tokens::clearAt instead.', E_USER_DEPRECATED);
-            $this->id = null;
-            $this->isArray = false;
-        }
-    }
-
     public function toArray()
     {
         return [

src/Tokenizer/Tokens.php

@@ -80,13 +80,6 @@ class Tokens extends \SplFixedArray
      */
     private $foundTokenKinds = [];
 
-    /**
-     * @var bool
-     *
-     * @todo remove at 3.0
-     */
-    private static $isLegacyMode = false;
-
     /**
      * Clone tokens collection.
      */
@@ -97,34 +90,6 @@ class Tokens extends \SplFixedArray
         }
     }
 
-    /**
-     * @return bool
-     *
-     * @internal
-     *
-     * @todo remove at 3.0
-     */
-    public static function isLegacyMode()
-    {
-        return self::$isLegacyMode;
-    }
-
-    /**
-     * @param bool $isLegacy
-     *
-     * @internal
-     *
-     * @todo remove at 3.0
-     */
-    public static function setLegacyMode($isLegacy)
-    {
-        if (getenv('PHP_CS_FIXER_FUTURE_MODE') && $isLegacy) {
-            throw new \RuntimeException('Cannot enable `legacy mode` when using `future mode`. This check was performed as `PHP_CS_FIXER_FUTURE_MODE` env var is set.');
-        }
-
-        self::$isLegacyMode = $isLegacy;
-    }
-
     /**
      * Clear cache - one position or all of them.
      *
@@ -328,12 +293,6 @@ class Tokens extends \SplFixedArray
     public function clearChanged()
     {
         $this->changed = false;
-
-        if (self::isLegacyMode()) {
-            foreach ($this as $token) {
-                $token->clearChanged();
-            }
-        }
     }
 
     /**
@@ -446,7 +405,7 @@ class Tokens extends \SplFixedArray
             throw new \InvalidArgumentException(sprintf('Invalid param type: %s.', $type));
         }
 
-        if (!self::isLegacyMode() && isset($this->blockEndCache[$searchIndex])) {
+        if (isset($this->blockEndCache[$searchIndex])) {
             return $this->blockEndCache[$searchIndex];
         }
 
@@ -519,11 +478,9 @@ class Tokens extends \SplFixedArray
             $elements[$kind] = [];
         }
 
-        if (!self::isLegacyMode()) {
-            $possibleKinds = array_filter($possibleKinds, function ($kind) {
-                return $this->isTokenKindFound($kind);
-            });
-        }
+        $possibleKinds = array_filter($possibleKinds, function ($kind) {
+            return $this->isTokenKindFound($kind);
+        });
 
         if (count($possibleKinds)) {
             for ($i = $start; $i < $end; ++$i) {
@@ -676,11 +633,9 @@ class Tokens extends \SplFixedArray
      */
     public function getTokenOfKindSibling($index, $direction, array $tokens = [], $caseSensitive = true)
     {
-        if (!self::isLegacyMode()) {
-            $tokens = array_filter($tokens, function ($token) {
-                return $this->isTokenKindFound($this->extractTokenKind($token));
-            });
-        }
+        $tokens = array_filter($tokens, function ($token) {
+            return $this->isTokenKindFound($this->extractTokenKind($token));
+        });
 
         if (!count($tokens)) {
             return null;
@@ -838,11 +793,9 @@ class Tokens extends \SplFixedArray
             }
         }
 
-        if (!self::isLegacyMode()) {
-            foreach ($sequence as $token) {
-                if (!$this->isTokenKindFound($this->extractTokenKind($token))) {
-                    return null;
-                }
+        foreach ($sequence as $token) {
+            if (!$this->isTokenKindFound($this->extractTokenKind($token))) {
+                return null;
             }
         }
 
@@ -938,14 +891,6 @@ class Tokens extends \SplFixedArray
             return true;
         }
 
-        if (self::isLegacyMode()) {
-            foreach ($this as $token) {
-                if ($token->isChanged()) {
-                    return true;
-                }
-            }
-        }
-
         return false;
     }
 
@@ -966,23 +911,6 @@ class Tokens extends \SplFixedArray
         $this[$index] = new Token('');
     }
 
-    /**
-     * Override token at given index and register it.
-     *
-     * @param int                $index
-     * @param array|string|Token $token token prototype
-     *
-     * @deprecated since 2.4, use offsetSet instead
-     */
-    public function overrideAt($index, $token)
-    {
-        @trigger_error(__METHOD__.' is deprecated and will be removed in 3.0, use offsetSet instead.', E_USER_DEPRECATED);
-        self::$isLegacyMode = true;
-
-        $this[$index]->override($token);
-        $this->registerFoundToken($token);
-    }
-
     /**
      * Override tokens at given range.
      *
@@ -1165,10 +1093,6 @@ class Tokens extends \SplFixedArray
      */
     public function countTokenKind($tokenKind)
     {
-        if (self::isLegacyMode()) {
-            throw new \RuntimeException(sprintf('%s is not available in legacy mode.', __METHOD__));
-        }
-
         return isset($this->foundTokenKinds[$tokenKind]) ? $this->foundTokenKinds[$tokenKind] : 0;
     }
 
@@ -1201,21 +1125,6 @@ class Tokens extends \SplFixedArray
             return false;
         }
 
-        if (self::isLegacyMode()) {
-            // If code is not monolithic there is a great chance that first or last token is `T_INLINE_HTML`:
-            if ($this[0]->isGivenKind(T_INLINE_HTML) || $this[$size - 1]->isGivenKind(T_INLINE_HTML)) {
-                return false;
-            }
-
-            for ($index = 1; $index < $size; ++$index) {
-                if ($this[$index]->isGivenKind([T_INLINE_HTML, T_OPEN_TAG, T_OPEN_TAG_WITH_ECHO])) {
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
         if ($this->isTokenKindFound(T_INLINE_HTML)) {
             return false;
         }

tests/Test/AbstractFixerTestCase.php

@@ -55,19 +55,6 @@ abstract class AbstractFixerTestCase extends TestCase
 
         $this->linter = $this->getLinter();
         $this->fixer = $this->createFixer();
-
-        // @todo remove at 3.0 together with env var itself
-        if (getenv('PHP_CS_FIXER_TEST_USE_LEGACY_TOKENIZER')) {
-            Tokens::setLegacyMode(true);
-        }
-    }
-
-    protected function tearDown()
-    {
-        parent::tearDown();
-
-        // @todo remove at 3.0
-        Tokens::setLegacyMode(false);
     }
 
     /**

tests/Test/AbstractIntegrationTestCase.php

@@ -105,19 +105,6 @@ abstract class AbstractIntegrationTestCase extends TestCase
         parent::setUp();
 
         $this->linter = $this->getLinter();
-
-        // @todo remove at 3.0 together with env var itself
-        if (getenv('PHP_CS_FIXER_TEST_USE_LEGACY_TOKENIZER')) {
-            Tokens::setLegacyMode(true);
-        }
-    }
-
-    protected function tearDown()
-    {
-        parent::tearDown();
-
-        // @todo remove at 3.0
-        Tokens::setLegacyMode(false);
     }
 
     /**

tests/Test/AbstractTransformerTestCase.php

@@ -23,24 +23,6 @@ use PHPUnit\Framework\TestCase;
  */
 abstract class AbstractTransformerTestCase extends TestCase
 {
-    protected function setUp()
-    {
-        parent::setUp();
-
-        // @todo remove at 3.0 together with env var itself
-        if (getenv('PHP_CS_FIXER_TEST_USE_LEGACY_TOKENIZER')) {
-            Tokens::setLegacyMode(true);
-        }
-    }
-
-    protected function tearDown()
-    {
-        parent::tearDown();
-
-        // @todo remove at 3.0
-        Tokens::setLegacyMode(false);
-    }
-
     protected function doTest($source, array $expectedTokens = [], array $observedKindsOrPrototypes = [])
     {
         $tokens = Tokens::fromCode($source);

tests/Tokenizer/TokenTest.php

@@ -14,7 +14,6 @@ namespace PhpCsFixer\Tests\Tokenizer;
 
 use PhpCsFixer\Tokenizer\CT;
 use PhpCsFixer\Tokenizer\Token;
-use PhpCsFixer\Tokenizer\Tokens;
 use PHPUnit\Framework\TestCase;
 
 /**
@@ -76,22 +75,6 @@ final class TokenTest extends TestCase
         ];
     }
 
-    /**
-     * @group legacy
-     * @expectedDeprecation PhpCsFixer\Tokenizer\Token::clear is deprecated and will be removed in 3.0.
-     */
-    public function testClear()
-    {
-        $token = $this->getForeachToken();
-        $token->clear();
-
-        Tokens::setLegacyMode(false);
-
-        $this->assertSame('', $token->getContent());
-        $this->assertNull($token->getId());
-        $this->assertFalse($token->isArray());
-    }
-
     public function testGetPrototype()
     {
         $this->assertSame($this->getBraceTokenPrototype(), $this->getBraceToken()->getPrototype());
@@ -175,22 +158,6 @@ final class TokenTest extends TestCase
         ];
     }
 
-    /**
-     * @group legacy
-     * @expectedDeprecation PhpCsFixer\Tokenizer\Token::isEmpty is deprecated and will be removed in 3.0.
-     */
-    public function testIsEmpty()
-    {
-        $braceToken = $this->getBraceToken();
-        $this->assertFalse($braceToken->isEmpty());
-
-        $emptyToken = new Token('');
-        $this->assertTrue($emptyToken->isEmpty());
-
-        $whitespaceToken = new Token([T_WHITESPACE, ' ']);
-        $this->assertFalse($whitespaceToken->isEmpty());
-    }
-
     public function testIsGivenKind()
     {
         $braceToken = $this->getBraceToken();
@@ -471,16 +438,6 @@ final class TokenTest extends TestCase
         ];
     }
 
-    /**
-     * @group legacy
-     * @expectedDeprecation PhpCsFixer\Tokenizer\Token::isChanged is deprecated and will be removed in 3.0.
-     */
-    public function testIsChanged()
-    {
-        $token = new Token([T_WHITESPACE, ' ']);
-        $this->assertFalse($token->isChanged());
-    }
-
     /**
      * @param null|string $expected
      * @param int         $id