* Dariusz Rumiński * * This source file is subject to the MIT license that is bundled * with this source code in the file LICENSE. */ namespace PhpCsFixer\Tests\Tokenizer; use PhpCsFixer\Tests\Test\Assert\AssertTokensTrait; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\Tokenizer\Analyzer\Analysis\NamespaceAnalysis; use PhpCsFixer\Tokenizer\CT; use PhpCsFixer\Tokenizer\Token; use PhpCsFixer\Tokenizer\Tokens; /** * @author Dariusz Rumiński * * @internal * * @covers \PhpCsFixer\Tokenizer\Tokens */ final class TokensTest extends TestCase { use AssertTokensTrait; public function testReadFromCacheAfterClearing(): void { $code = 'count(); for ($i = 0; $i < $countBefore; ++$i) { $tokens->clearAt($i); } $tokens = Tokens::fromCode($code); self::assertCount($countBefore, $tokens); } /** * @param null|array $expected * @param list $sequence * @param bool|list $caseSensitive * * @dataProvider provideFindSequenceCases */ public function testFindSequence( string $source, ?array $expected, array $sequence, int $start = 0, ?int $end = null, $caseSensitive = true ): void { $tokens = Tokens::fromCode($source); self::assertEqualsTokensArray( $expected, $tokens->findSequence( $sequence, $start, $end, $caseSensitive ) ); } public static function provideFindSequenceCases(): iterable { yield [ ' new Token([T_OPEN_TAG, ' new Token([T_VARIABLE, '$x']), ], [ [T_OPEN_TAG], [T_VARIABLE, '$x'], ], ]; yield [ ' new Token('='), 5 => new Token([T_LNUMBER, '4']), 6 => new Token(';'), ], [ '=', [T_LNUMBER, '4'], ';', ], ]; yield [ ' new Token([T_OPEN_TAG, ' new Token([T_VARIABLE, '$x']), ], [ [T_OPEN_TAG], [T_VARIABLE, '$x'], ], 0, ]; yield [ ' new Token('='), 5 => new Token([T_LNUMBER, '7']), 6 => new Token(';'), ], [ '=', [T_LNUMBER, '7'], ';', ], 3, 6, ]; yield [ ' new Token([T_OPEN_TAG, ' new Token([T_VARIABLE, '$x']), ], [ [T_OPEN_TAG], [T_VARIABLE, '$x'], ], 0, 1, true, ]; yield [ ' true], ]; yield [ ' new Token([T_OPEN_TAG, ' new Token([T_VARIABLE, '$x']), ], [ [T_OPEN_TAG], [T_VARIABLE, '$X'], ], 0, 1, false, ]; yield [ ' new Token([T_OPEN_TAG, ' new Token([T_VARIABLE, '$x']), ], [ [T_OPEN_TAG], [T_VARIABLE, '$X'], ], 0, 1, [1 => false], ]; yield [ ' new Token([T_OPEN_TAG, ' new Token([T_VARIABLE, '$x']), ], [ [T_OPEN_TAG], [T_VARIABLE, '$X'], ], 0, 1, [1 => false], ]; yield [ ' false], ]; yield [ ' $sequence sequence of token prototypes * * @dataProvider provideFindSequenceExceptionCases */ public function testFindSequenceException(string $message, array $sequence): void { $tokens = Tokens::fromCode('expectException(\InvalidArgumentException::class); $this->expectExceptionMessage($message); $tokens->findSequence($sequence); } public static function provideFindSequenceExceptionCases(): iterable { $emptyToken = new Token(''); yield ['Invalid sequence.', []]; yield [ 'Non-meaningful token at position: "0".', [[T_WHITESPACE, ' ']], ]; yield [ 'Non-meaningful token at position: "1".', ['{', [T_COMMENT, '// Foo'], '}'], ]; yield [ 'Non-meaningful (empty) token at position: "2".', ['{', '!', $emptyToken, '}'], ]; } public function testClearRange(): void { $source = <<<'PHP' findGivenKind(T_PUBLIC)); $tokens->clearRange($fooIndex, $barIndex - 1); $newPublicIndexes = array_keys($tokens->findGivenKind(T_PUBLIC)); self::assertSame($barIndex, reset($newPublicIndexes)); for ($i = $fooIndex; $i < $barIndex; ++$i) { self::assertTrue($tokens[$i]->isWhitespace()); } } /** * @dataProvider provideMonolithicPhpDetectionCases */ public function testMonolithicPhpDetection(bool $isMonolithic, string $source): void { $tokens = Tokens::fromCode($source); self::assertSame($isMonolithic, $tokens->isMonolithicPhp()); } /** * @return iterable */ public static function provideMonolithicPhpDetectionCases(): iterable { yield [true, ""]; yield [false, "#!\n "]; yield [false, "']; yield [false, ' World!']; // short open tag yield [(bool) \ini_get('short_open_tag'), ""]; yield [false, " "]; yield [false, ""]; yield [false, " "]; yield [false, "isTokenKindFound(T_CLASS)); self::assertTrue($tokens->isTokenKindFound(T_RETURN)); self::assertFalse($tokens->isTokenKindFound(T_INTERFACE)); self::assertFalse($tokens->isTokenKindFound(T_ARRAY)); self::assertTrue($tokens->isAllTokenKindsFound([T_CLASS, T_RETURN])); self::assertFalse($tokens->isAllTokenKindsFound([T_CLASS, T_INTERFACE])); self::assertTrue($tokens->isAnyTokenKindsFound([T_CLASS, T_RETURN])); self::assertTrue($tokens->isAnyTokenKindsFound([T_CLASS, T_INTERFACE])); self::assertFalse($tokens->isAnyTokenKindsFound([T_INTERFACE, T_ARRAY])); } public function testFindGivenKind(): void { $source = <<<'PHP' $found */ $found = $tokens->findGivenKind(T_CLASS); self::assertCount(1, $found); self::assertArrayHasKey(1, $found); self::assertSame(T_CLASS, $found[1]->getId()); $found = $tokens->findGivenKind([T_CLASS, T_FUNCTION]); self::assertCount(2, $found); self::assertArrayHasKey(T_CLASS, $found); self::assertIsArray($found[T_CLASS]); self::assertCount(1, $found[T_CLASS]); self::assertArrayHasKey(1, $found[T_CLASS]); self::assertSame(T_CLASS, $found[T_CLASS][1]->getId()); self::assertArrayHasKey(T_FUNCTION, $found); self::assertIsArray($found[T_FUNCTION]); self::assertCount(2, $found[T_FUNCTION]); self::assertArrayHasKey(9, $found[T_FUNCTION]); self::assertSame(T_FUNCTION, $found[T_FUNCTION][9]->getId()); self::assertArrayHasKey(26, $found[T_FUNCTION]); self::assertSame(T_FUNCTION, $found[T_FUNCTION][26]->getId()); // test offset and limits of the search $found = $tokens->findGivenKind([T_CLASS, T_FUNCTION], 10); self::assertArrayHasKey(T_CLASS, $found); self::assertCount(0, $found[T_CLASS]); self::assertArrayHasKey(T_FUNCTION, $found); self::assertCount(1, $found[T_FUNCTION]); self::assertArrayHasKey(26, $found[T_FUNCTION]); $found = $tokens->findGivenKind([T_CLASS, T_FUNCTION], 2, 10); self::assertArrayHasKey(T_CLASS, $found); self::assertCount(0, $found[T_CLASS]); self::assertArrayHasKey(T_FUNCTION, $found); self::assertCount(1, $found[T_FUNCTION]); self::assertArrayHasKey(9, $found[T_FUNCTION]); } /** * @param list $indexes to clear * @param list $expected tokens * * @dataProvider provideClearTokenAndMergeSurroundingWhitespaceCases */ public function testClearTokenAndMergeSurroundingWhitespace(string $source, array $indexes, array $expected): void { $this->doTestClearTokens($source, $indexes, $expected); if (\count($indexes) > 1) { $this->doTestClearTokens($source, array_reverse($indexes), $expected); } } public static function provideClearTokenAndMergeSurroundingWhitespaceCases(): iterable { $clearToken = new Token(''); yield [ ' $findTokens * * @dataProvider provideTokenOfKindSiblingCases */ public function testTokenOfKindSibling( ?int $expectedIndex, int $direction, int $index, array $findTokens, bool $caseSensitive = true ): void { $source = 'getNextTokenOfKind($index, $findTokens, $caseSensitive)); } else { self::assertSame($expectedIndex, $tokens->getPrevTokenOfKind($index, $findTokens, $caseSensitive)); } self::assertSame($expectedIndex, $tokens->getTokenOfKindSibling($index, $direction, $findTokens, $caseSensitive)); } public static function provideTokenOfKindSiblingCases(): iterable { // find next cases yield [ 35, 1, 34, [';'], ]; yield [ 14, 1, 0, [[T_RETURN]], ]; yield [ 32, 1, 14, [[T_RETURN]], ]; yield [ 6, 1, 0, [[T_RETURN], [T_FUNCTION]], ]; // find previous cases yield [ 14, -1, 32, [[T_RETURN], [T_FUNCTION]], ]; yield [ 6, -1, 7, [[T_FUNCTION]], ]; yield [ null, -1, 6, [[T_FUNCTION]], ]; } /** * @dataProvider provideFindBlockEndCases * * @param Tokens::BLOCK_TYPE_* $type */ public function testFindBlockEnd(int $expectedIndex, string $source, int $type, int $searchIndex): void { self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex); } /** * @return iterable */ public static function provideFindBlockEndCases(): iterable { yield [4, '{$bar};', Tokens::BLOCK_TYPE_DYNAMIC_PROP_BRACE, 3]; yield [4, '', Tokens::BLOCK_TYPE_CURLY_BRACE, 5]; yield [11, '{"set_{$name}"}(42);', Tokens::BLOCK_TYPE_DYNAMIC_PROP_BRACE, 3]; yield [19, ' */ public static function provideFindBlockEnd80Cases(): iterable { yield [ 9, ' */ public static function provideFindBlockEnd82Cases(): iterable { yield [ 11, ' 11, 19 => 23, 27 => 35] as $openIndex => $closeIndex) { yield [ $closeIndex, ' */ public static function provideFindBlockEnd83Cases(): iterable { yield 'simple dynamic class constant fetch' => [ 7, ' [ $startEnd[1], "", Tokens::BLOCK_TYPE_DYNAMIC_CLASS_CONSTANT_FETCH_CURLY_BRACE, $startEnd[0], ]; } } /** * @param Tokens::BLOCK_TYPE_* $type * * @dataProvider provideFindBlockEndPre84Cases * * @requires PHP <8.4 */ public function testFindBlockEndPre84(int $expectedIndex, string $source, int $type, int $searchIndex): void { self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex); } /** * @return iterable */ public static function provideFindBlockEndPre84Cases(): iterable { yield [4, 'expectException(\InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Invalid param type: "-1"\.$/'); // @phpstan-ignore-next-line $tokens->findBlockEnd(-1, 0); } public function testFindBlockEndInvalidStart(): void { Tokens::clearCache(); $tokens = Tokens::fromCode('expectException(\InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Invalid param \$startIndex - not a proper block "start"\.$/'); $tokens->findBlockEnd(Tokens::BLOCK_TYPE_DYNAMIC_VAR_BRACE, 0); } public function testFindBlockEndCalledMultipleTimes(): void { Tokens::clearCache(); $tokens = Tokens::fromCode('findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 2)); $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Invalid param \$startIndex - not a proper block "start"\.$/'); $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 7); } public function testFindBlockStartEdgeCalledMultipleTimes(): void { Tokens::clearCache(); $tokens = Tokens::fromCode('findBlockStart(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 7)); $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Invalid param \$startIndex - not a proper block "end"\.$/'); $tokens->findBlockStart(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 2); } public function testEmptyTokens(): void { $code = ''; $tokens = Tokens::fromCode($code); self::assertCount(0, $tokens); self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG)); } public function testEmptyTokensMultiple(): void { $code = ''; $tokens = Tokens::fromCode($code); self::assertFalse($tokens->isChanged()); $tokens->insertAt(0, new Token([T_WHITESPACE, ' '])); self::assertCount(1, $tokens); self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG)); self::assertTrue($tokens->isChanged()); $tokens2 = Tokens::fromCode($code); self::assertCount(0, $tokens2); self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG)); } public function testFromArray(): void { $code = 'toArray()); self::assertTrue($tokens1->isTokenKindFound(T_OPEN_TAG)); self::assertTrue($tokens2->isTokenKindFound(T_OPEN_TAG)); self::assertSame($tokens1->getCodeHash(), $tokens2->getCodeHash()); } public function testFromArrayEmpty(): void { $tokens = Tokens::fromArray([]); self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG)); } /** * @dataProvider provideIsEmptyCases */ public function testIsEmpty(Token $token, bool $isEmpty): void { $tokens = Tokens::fromArray([$token]); Tokens::clearCache(); self::assertSame($isEmpty, $tokens->isEmptyAt(0), $token->toJson()); } /** * @return iterable */ public static function provideIsEmptyCases(): iterable { yield [new Token(''), true]; yield [new Token('('), false]; yield [new Token([T_WHITESPACE, ' ']), false]; } public function testClone(): void { $code = 'isTokenKindFound(T_OPEN_TAG)); self::assertTrue($tokensClone->isTokenKindFound(T_OPEN_TAG)); $count = \count($tokens); self::assertCount($count, $tokensClone); for ($i = 0; $i < $count; ++$i) { self::assertTrue($tokens[$i]->equals($tokensClone[$i])); self::assertNotSame($tokens[$i], $tokensClone[$i]); } } /** * @dataProvider provideEnsureWhitespaceAtIndexCases */ public function testEnsureWhitespaceAtIndex(string $expected, string $input, int $index, int $offset, string $whiteSpace): void { $tokens = Tokens::fromCode($input); $tokens->ensureWhitespaceAtIndex($index, $offset, $whiteSpace); $tokens->clearEmptyTokens(); self::assertTokens(Tokens::fromCode($expected), $tokens); } /** * @return iterable */ public static function provideEnsureWhitespaceAtIndexCases(): iterable { yield [ 'name = $name; } }'; $tokens = Tokens::fromCode(\sprintf($template, '')); $commentIndex = $tokens->getNextTokenOfKind(0, [[T_COMMENT]]); $tokens->insertAt( $commentIndex, [ new Token([T_PRIVATE, 'private']), new Token([T_WHITESPACE, ' ']), new Token([T_VARIABLE, '$name']), new Token(';'), ] ); self::assertTrue($tokens->isChanged()); $expected = Tokens::fromCode(\sprintf($template, 'private $name;')); self::assertFalse($expected->isChanged()); self::assertTokens($expected, $tokens); } /** * @dataProvider provideRemoveLeadingWhitespaceCases */ public function testRemoveLeadingWhitespace(int $index, ?string $whitespaces, string $expected, ?string $input = null): void { Tokens::clearCache(); $tokens = Tokens::fromCode($input ?? $expected); $tokens->removeLeadingWhitespace($index, $whitespaces); self::assertSame($expected, $tokens->generateCode()); } /** * @return iterable */ public static function provideRemoveLeadingWhitespaceCases(): iterable { yield [ 7, null, "removeTrailingWhitespace($index, $whitespaces); self::assertSame($expected, $tokens->generateCode()); } /** * @return iterable */ public static function provideRemoveTrailingWhitespaceCases(): iterable { $leadingCases = self::provideRemoveLeadingWhitespaceCases(); foreach ($leadingCases as $leadingCase) { $leadingCase[0] -= 2; yield $leadingCase; } } public function testRemovingLeadingWhitespaceWithEmptyTokenInCollection(): void { $code = "clearAt(2); $tokens->removeLeadingWhitespace(3); $tokens->clearEmptyTokens(); self::assertTokens(Tokens::fromCode("clearAt(2); $tokens->removeTrailingWhitespace(1); $tokens->clearEmptyTokens(); self::assertTokens(Tokens::fromCode("count(); $tokens->removeLeadingWhitespace(4); self::assertCount($originalCount, $tokens); self::assertSame( 'generateCode() ); } /** * Action that begins with the word "remove" should not change the size of collection. */ public function testRemovingTrailingWhitespaceWillNotIncreaseTokensCount(): void { $tokens = Tokens::fromCode('count(); $tokens->removeTrailingWhitespace(2); self::assertCount($originalCount, $tokens); self::assertSame( 'generateCode() ); } /** * @param null|array{type: Tokens::BLOCK_TYPE_*, isStart: bool} $expected * * @dataProvider provideDetectBlockTypeCases */ public function testDetectBlockType(?array $expected, string $code, int $index): void { $tokens = Tokens::fromCode($code); self::assertSame($expected, Tokens::detectBlockType($tokens[$index])); } public static function provideDetectBlockTypeCases(): iterable { yield [ [ 'type' => Tokens::BLOCK_TYPE_CURLY_BRACE, 'isStart' => true, ], 'overrideRange($indexStart, $indexEnd, $items); $tokens->clearEmptyTokens(); self::assertTokens(Tokens::fromArray($expected), $tokens); } /** * @param list $expected * @param array $items * * @dataProvider provideOverrideRangeCases */ public function testOverrideRange(array $expected, string $code, int $indexStart, int $indexEnd, array $items): void { $tokens = Tokens::fromCode($code); $tokens->overrideRange($indexStart, $indexEnd, $items); $tokens->clearEmptyTokens(); self::assertTokens(Tokens::fromArray($expected), $tokens); } public static function provideOverrideRangeCases(): iterable { // typically done by transformers, here we test the reverse yield 'override different tokens but same content' => [ [ new Token([T_OPEN_TAG, ' [ [ new Token([T_OPEN_TAG, "isChanged()); $tokens = Tokens::fromArray( [ new Token([T_OPEN_TAG, "isChanged()); } /** * @param -1|1 $direction * * @dataProvider provideGetMeaningfulTokenSiblingCases */ public function testGetMeaningfulTokenSibling(?int $expectIndex, int $index, int $direction, string $source): void { Tokens::clearCache(); $tokens = Tokens::fromCode($source); self::assertSame($expectIndex, $tokens->getMeaningfulTokenSibling($index, $direction)); } /** * @return iterable */ public static function provideGetMeaningfulTokenSiblingCases(): iterable { yield [null, 0, 1, ''.$i => [3, $i, 1, '>' => [4, 3, 1, ' [null, 6, 1, ' [null, 888, 1, ' $slices */ public function testInsertSlicesAtMultiplePlaces(string $expected, array $slices): void { $input = <<<'EOF' insertSlices([ 16 => $slices, 6 => $slices, ]); self::assertTokens(Tokens::fromCode($expected), $tokens); } public static function provideInsertSlicesAtMultiplePlacesCases(): iterable { yield 'one slice count' => [ <<<'EOF' [ <<<'EOF' [ <<<'EOF' isChanged()); self::assertFalse($tokens->isTokenKindFound(T_COMMENT)); self::assertSame(5, $tokens->getSize()); $tokens->insertSlices([1 => new Token([T_COMMENT, '/* comment */'])]); self::assertTrue($tokens->isChanged()); self::assertTrue($tokens->isTokenKindFound(T_COMMENT)); self::assertSame(6, $tokens->getSize()); } /** * @param array|Token|Tokens> $slices * * @dataProvider provideInsertSlicesCases */ public function testInsertSlices(Tokens $expected, Tokens $tokens, array $slices): void { $tokens->insertSlices($slices); self::assertTokens($expected, $tokens); } public static function provideInsertSlicesCases(): iterable { // basic insert of single token at 3 different locations including appending as new token $template = " [ Tokens::fromCode(\sprintf($template, $commentContent, '', '')), clone $from, [1 => $commentToken], ]; yield 'single insert @ 3' => [ Tokens::fromCode(\sprintf($template, '', $commentContent, '')), clone $from, [3 => Tokens::fromArray([$commentToken])], ]; yield 'single insert @ 9' => [ Tokens::fromCode(\sprintf($template, '', '', $commentContent)), clone $from, [9 => [$commentToken]], ]; // basic tests for single token, array of that token and tokens object with that token $openTagToken = new Token([T_OPEN_TAG, " $openTagToken], [0 => [clone $openTagToken]], [0 => clone Tokens::fromArray([$openTagToken])], ]; foreach ($slices as $i => $slice) { yield 'insert open tag @ 0 into empty collection '.$i => [$expected, new Tokens(), $slice]; } // test insert lists of tokens, index out of order $setOne = [ new Token([T_ECHO, 'echo']), new Token([T_WHITESPACE, ' ']), new Token([T_CONSTANT_ENCAPSED_STRING, '"new"']), new Token(';'), ]; $setTwo = [ new Token([T_WHITESPACE, ' ']), new Token([T_COMMENT, '/* new comment */']), ]; $setThree = Tokens::fromArray([ new Token([T_VARIABLE, '$new']), new Token([T_WHITESPACE, ' ']), new Token('='), new Token([T_WHITESPACE, ' ']), new Token([T_LNUMBER, '8899']), new Token(';'), new Token([T_WHITESPACE, "\n"]), ]); $template = " [$expected, $from, [9 => $setThree, 1 => $setOne, 3 => $setTwo]]; $sets = []; for ($j = 0; $j < 4; ++$j) { $set = ['tokens' => [], 'content' => '']; for ($i = 0; $i < 10; ++$i) { $content = \sprintf('/* new %d|%s */', $j, $i); $set['tokens'][] = new Token([T_COMMENT, $content]); $set['content'] .= $content; } $sets[$j] = $set; } yield 'overlapping inserts of bunch of comments' => [ Tokens::fromCode(\sprintf(" $sets[0]['tokens'], 3 => $sets[1]['tokens'], 5 => $sets[2]['tokens'], 6 => $sets[3]['tokens']], ]; } public function testBlockEdgeCachingOffsetSet(): void { $tokens = $this->getBlockEdgeCachingTestTokens(); $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); self::assertSame(9, $endIndex); $tokens->offsetSet(5, new Token('(')); $tokens->offsetSet(9, new Token('(')); $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessage('Invalid param $startIndex - not a proper block "start".'); $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); } public function testBlockEdgeCachingOffsetSetPruneEvenIfTokenEquals(): void { $tokens = Tokens::fromArray([ new Token([T_OPEN_TAG, 'findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 4)); self::assertSame(4, $tokens->findBlockStart(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 6)); $tokens->overrideRange(3, 6, [ new Token([CT::T_ARRAY_SQUARE_BRACE_OPEN, '[']), $tokens[4], new Token([CT::T_ARRAY_SQUARE_BRACE_CLOSE, ']']), $tokens[6], ]); self::assertSame(5, $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 4)); self::assertSame(4, $tokens->findBlockStart(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5)); self::assertSame(6, $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 3)); self::assertSame(3, $tokens->findBlockStart(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 6)); } public function testBlockEdgeCachingClearAt(): void { $tokens = $this->getBlockEdgeCachingTestTokens(); $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); self::assertSame(9, $endIndex); $tokens->clearAt(7); // note: offsetUnset doesn't work here $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); self::assertSame(9, $endIndex); $tokens->clearEmptyTokens(); $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); self::assertSame(8, $endIndex); } public function testBlockEdgeCachingInsertSlices(): void { $tokens = $this->getBlockEdgeCachingTestTokens(); $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); self::assertSame(9, $endIndex); $tokens->insertSlices([6 => [new Token([T_COMMENT, '/* A */'])], new Token([T_COMMENT, '/* B */'])]); $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5); self::assertSame(11, $endIndex); } public function testNamespaceDeclarations(): void { $code = 'getNamespaceDeclarations()) ); $newNS = 'insertAt(2, Tokens::fromCode($newNS)); self::assertSame( serialize([ new NamespaceAnalysis( 'Foo\Bar', 'Bar', 3, 8, 3, 8 ), ]), serialize($tokens->getNamespaceDeclarations()) ); } public function testFindingToken(): void { $tokens = Tokens::fromCode('isTokenKindFound(T_VARIABLE)); $tokens->offsetUnset(1); $tokens->offsetUnset(1); // 2nd unset of the same index should not crash anything self::assertFalse($tokens->isTokenKindFound(T_VARIABLE)); $tokens[1] = new Token([T_VARIABLE, '$x']); self::assertTrue($tokens->isTokenKindFound(T_VARIABLE)); } public function testSettingSizeThrowsException(): void { $tokens = new Tokens(); $this->expectException(\RuntimeException::class); $this->expectExceptionMessage('Changing tokens collection size explicitly is not allowed.'); $tokens->setSize(3); } public function testSettingSizeInTryCatchBlockDoesNotChangeSize(): void { $tokens = Tokens::fromCode('getSize(); try { $tokens->setSize(5); } catch (\RuntimeException $exception) { self::assertSame('Changing tokens collection size explicitly is not allowed.', $exception->getMessage()); } self::assertSame($size, $tokens->getSize()); } private function getBlockEdgeCachingTestTokens(): Tokens { Tokens::clearCache(); return Tokens::fromArray([ new Token([T_OPEN_TAG, 'findBlockEnd($type, $searchIndex)); self::assertSame($searchIndex, $tokens->findBlockStart($type, $expectedIndex)); $detectedType = Tokens::detectBlockType($tokens[$searchIndex]); self::assertIsArray($detectedType); self::assertArrayHasKey('type', $detectedType); self::assertArrayHasKey('isStart', $detectedType); self::assertSame($type, $detectedType['type']); self::assertTrue($detectedType['isStart']); $detectedType = Tokens::detectBlockType($tokens[$expectedIndex]); self::assertIsArray($detectedType); self::assertArrayHasKey('type', $detectedType); self::assertArrayHasKey('isStart', $detectedType); self::assertSame($type, $detectedType['type']); self::assertFalse($detectedType['isStart']); } /** * @param null|array $expected * @param null|array $input */ private static function assertEqualsTokensArray(?array $expected = null, ?array $input = null): void { if (null === $expected) { self::assertNull($input); return; } if (null === $input) { self::fail('While "input" is , "expected" is not.'); } self::assertSame(array_keys($expected), array_keys($input), 'Both arrays need to have same keys.'); foreach ($expected as $index => $expectedToken) { self::assertTrue( $expectedToken->equals($input[$index]), \sprintf('The token at index %d should be %s, got %s', $index, $expectedToken->toJson(), $input[$index]->toJson()) ); } } /** * @param list $indexes * @param list $expected */ private function doTestClearTokens(string $source, array $indexes, array $expected): void { Tokens::clearCache(); $tokens = Tokens::fromCode($source); foreach ($indexes as $index) { $tokens->clearTokenAndMergeSurroundingWhitespace($index); } self::assertSameSize($expected, $tokens); foreach ($expected as $index => $expectedToken) { $token = $tokens[$index]; $expectedPrototype = $expectedToken->getPrototype(); self::assertTrue($token->equals($expectedPrototype), \sprintf('The token at index %d should be %s, got %s', $index, json_encode($expectedPrototype, JSON_THROW_ON_ERROR), $token->toJson())); } } }