Skip to content

Commit f66a32e

Browse files
nikicnikic
committed
Emulate PHP 8 attribute syntax
Perform emulation by replacing #[ with %[, then patching % back to # and coalescing #[ into T_ATTRIBUTE if it is a freestanding token.
1 parent 75abbbd commit f66a32e

File tree

8 files changed

+182
-63
lines changed

8 files changed

+182
-63
lines changed

‎grammar/tokens.y

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,4 +109,5 @@
109109
%token T_ELLIPSIS
110110
%token T_NAME_FULLY_QUALIFIED
111111
%token T_NAME_QUALIFIED
112-
%token T_NAME_RELATIVE
112+
%token T_NAME_RELATIVE
113+
%token T_ATTRIBUTE

‎lib/PhpParser/Lexer.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,7 @@ private function defineCompatibilityTokens() {
421421
'T_NAME_RELATIVE',
422422
'T_MATCH',
423423
'T_NULLSAFE_OBJECT_OPERATOR',
424+
'T_ATTRIBUTE',
424425
];
425426

426427
// PHP-Parser might be used together with another library that also emulates some or all
@@ -510,6 +511,7 @@ protected function createTokenMap() : array {
510511
$tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
511512
$tokenMap[\T_MATCH] = Tokens::T_MATCH;
512513
$tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
514+
$tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
513515

514516
return $tokenMap;
515517
}

‎lib/PhpParser/Lexer/Emulative.php

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
use PhpParser\Error;
66
use PhpParser\ErrorHandler;
77
use PhpParser\Lexer;
8+
use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
89
use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
910
use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
1011
use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
@@ -49,6 +50,7 @@ public function __construct(array $options = [])
4950
new CoaleseEqualTokenEmulator(),
5051
new NumericLiteralSeparatorEmulator(),
5152
new NullsafeTokenEmulator(),
53+
new AttributeEmulator(),
5254
];
5355

5456
// Collect emulators that are relevant for the PHP version we're running
@@ -81,6 +83,7 @@ public function startLexing(string $code, ErrorHandler $errorHandler = null) {
8183

8284
$collector = new ErrorHandler\Collecting();
8385
parent::startLexing($code, $collector);
86+
$this->sortPatches();
8487
$this->fixupTokens();
8588

8689
$errors = $collector->getErrors();
@@ -106,6 +109,15 @@ private function isReverseEmulationNeeded(string $emulatorPhpVersion): bool {
106109
&& version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<');
107110
}
108111

112+
private function sortPatches()
113+
{
114+
// Patches may be contributed by different emulators.
115+
// Make sure they are sorted by increasing patch position.
116+
usort($this->patches, function($p1, $p2) {
117+
return $p1[0] <=> $p2[0];
118+
});
119+
}
120+
109121
private function fixupTokens()
110122
{
111123
if (\count($this->patches) === 0) {
@@ -122,7 +134,20 @@ private function fixupTokens()
122134
for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
123135
$token = $this->tokens[$i];
124136
if (\is_string($token)) {
125-
// We assume that patches don't apply to string tokens
137+
if ($patchPos === $pos) {
138+
// Only support replacement for string tokens.
139+
assert($patchType === 'replace');
140+
$this->tokens[$i] = $patchText;
141+
142+
// Fetch the next patch
143+
$patchIdx++;
144+
if ($patchIdx >= \count($this->patches)) {
145+
// No more patches, we're done
146+
return;
147+
}
148+
list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
149+
}
150+
126151
$pos += \strlen($token);
127152
continue;
128153
}
@@ -150,6 +175,11 @@ private function fixupTokens()
150175
$token[1], $patchText, $patchPos - $pos + $posDelta, 0
151176
);
152177
$posDelta += $patchTextLen;
178+
} else if ($patchType === 'replace') {
179+
// Replace inside the token string
180+
$this->tokens[$i][1] = substr_replace(
181+
$token[1], $patchText, $patchPos - $pos + $posDelta, $patchTextLen
182+
);
153183
} else {
154184
assert(false);
155185
}
@@ -196,7 +226,7 @@ private function fixupErrors(array $errors) {
196226
if ($patchType === 'add') {
197227
$posDelta += strlen($patchText);
198228
$lineDelta += substr_count($patchText, "\n");
199-
} else {
229+
} else if ($patchType === 'remove') {
200230
$posDelta -= strlen($patchText);
201231
$lineDelta -= substr_count($patchText, "\n");
202232
}

‎lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
<?php declare(strict_types=1);
2+
3+
namespace PhpParser\Lexer\TokenEmulator;
4+
5+
use PhpParser\Lexer\Emulative;
6+
7+
final class AttributeEmulator extends TokenEmulator
8+
{
9+
public function getPhpVersion(): string
10+
{
11+
return Emulative::PHP_8_0;
12+
}
13+
14+
public function isEmulationNeeded(string $code) : bool
15+
{
16+
return strpos($code, '#[') !== false;
17+
}
18+
19+
public function emulate(string $code, array $tokens): array
20+
{
21+
// We need to manually iterate and manage a count because we'll change
22+
// the tokens array on the way.
23+
$line = 1;
24+
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
25+
if ($tokens[$i] === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1] === '[') {
26+
array_splice($tokens, $i, 2, [
27+
[\T_ATTRIBUTE, '#[', $line]
28+
]);
29+
$c--;
30+
continue;
31+
}
32+
if (\is_array($tokens[$i])) {
33+
$line += substr_count($tokens[$i][1], "\n");
34+
}
35+
}
36+
37+
return $tokens;
38+
}
39+
40+
public function reverseEmulate(string $code, array $tokens): array
41+
{
42+
// TODO
43+
return $tokens;
44+
}
45+
46+
public function preprocessCode(string $code, array &$patches): string {
47+
$pos = 0;
48+
while (false !== $pos = strpos($code, '#[', $pos)) {
49+
// Replace #[ with %[
50+
$code[$pos] = '%';
51+
$patches[] = [$pos, 'replace', '#'];
52+
$pos += 2;
53+
}
54+
return $code;
55+
}
56+
}

‎lib/PhpParser/Parser/Php5.php

Lines changed: 31 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
*/
1818
class Php5 extends \PhpParser\ParserAbstract
1919
{
20-
protected $tokenToSymbolMapSize = 391;
20+
protected $tokenToSymbolMapSize = 392;
2121
protected $actionTableSize = 1069;
2222
protected $gotoTableSize = 580;
2323

24-
protected $invalidSymbol = 164;
24+
protected $invalidSymbol = 165;
2525
protected $errorSymbol = 1;
2626
protected $defaultAction = -32766;
2727
protected $unexpectedTokenRule = 32767;
@@ -193,36 +193,37 @@ class Php5 extends \PhpParser\ParserAbstract
193193
"'`'",
194194
"']'",
195195
"'\"'",
196-
"T_NULLSAFE_OBJECT_OPERATOR"
196+
"T_NULLSAFE_OBJECT_OPERATOR",
197+
"T_ATTRIBUTE"
197198
);
198199

199200
protected $tokenToSymbol = array(
200-
0, 164, 164, 164, 164, 164, 164, 164, 164, 164,
201-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
202-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
203-
164, 164, 164, 55, 162, 164, 159, 54, 37, 164,
204-
157, 158, 52, 49, 8, 50, 51, 53, 164, 164,
205-
164, 164, 164, 164, 164, 164, 164, 164, 31, 154,
206-
43, 16, 45, 30, 67, 164, 164, 164, 164, 164,
207-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
208-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
209-
164, 69, 164, 161, 36, 164, 160, 164, 164, 164,
210-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
211-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
212-
164, 164, 164, 155, 35, 156, 57, 164, 164, 164,
213-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
214-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
215-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
216-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
217-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
218-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
219-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
220-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
221-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
222-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
223-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
224-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
225-
164, 164, 164, 164, 164, 164, 1, 2, 3, 4,
201+
0, 165, 165, 165, 165, 165, 165, 165, 165, 165,
202+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
203+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
204+
165, 165, 165, 55, 162, 165, 159, 54, 37, 165,
205+
157, 158, 52, 49, 8, 50, 51, 53, 165, 165,
206+
165, 165, 165, 165, 165, 165, 165, 165, 31, 154,
207+
43, 16, 45, 30, 67, 165, 165, 165, 165, 165,
208+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
209+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
210+
165, 69, 165, 161, 36, 165, 160, 165, 165, 165,
211+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
212+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
213+
165, 165, 165, 155, 35, 156, 57, 165, 165, 165,
214+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
215+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
216+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
217+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
218+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
219+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
220+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
221+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
222+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
223+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
224+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
225+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
226+
165, 165, 165, 165, 165, 165, 1, 2, 3, 4,
226227
5, 6, 7, 9, 10, 11, 12, 13, 14, 15,
227228
17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
228229
27, 28, 29, 32, 33, 34, 38, 39, 40, 41,
@@ -236,7 +237,7 @@ class Php5 extends \PhpParser\ParserAbstract
236237
124, 125, 126, 127, 128, 129, 130, 131, 163, 132,
237238
133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
238239
143, 144, 145, 146, 147, 148, 149, 150, 151, 152,
239-
153
240+
153, 164
240241
);
241242

242243
protected $action = array(

‎lib/PhpParser/Parser/Php7.php

Lines changed: 31 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
*/
1818
class Php7 extends \PhpParser\ParserAbstract
1919
{
20-
protected $tokenToSymbolMapSize = 391;
20+
protected $tokenToSymbolMapSize = 392;
2121
protected $actionTableSize = 1124;
2222
protected $gotoTableSize = 498;
2323

24-
protected $invalidSymbol = 164;
24+
protected $invalidSymbol = 165;
2525
protected $errorSymbol = 1;
2626
protected $defaultAction = -32766;
2727
protected $unexpectedTokenRule = 32767;
@@ -193,36 +193,37 @@ class Php7 extends \PhpParser\ParserAbstract
193193
"'`'",
194194
"']'",
195195
"'\"'",
196-
"'$'"
196+
"'$'",
197+
"T_ATTRIBUTE"
197198
);
198199

199200
protected $tokenToSymbol = array(
200-
0, 164, 164, 164, 164, 164, 164, 164, 164, 164,
201-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
202-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
203-
164, 164, 164, 55, 162, 164, 163, 54, 37, 164,
204-
158, 159, 52, 49, 8, 50, 51, 53, 164, 164,
205-
164, 164, 164, 164, 164, 164, 164, 164, 31, 155,
206-
43, 16, 45, 30, 67, 164, 164, 164, 164, 164,
207-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
208-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
209-
164, 69, 164, 161, 36, 164, 160, 164, 164, 164,
210-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
211-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
212-
164, 164, 164, 156, 35, 157, 57, 164, 164, 164,
213-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
214-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
215-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
216-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
217-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
218-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
219-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
220-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
221-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
222-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
223-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
224-
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
225-
164, 164, 164, 164, 164, 164, 1, 2, 3, 4,
201+
0, 165, 165, 165, 165, 165, 165, 165, 165, 165,
202+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
203+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
204+
165, 165, 165, 55, 162, 165, 163, 54, 37, 165,
205+
158, 159, 52, 49, 8, 50, 51, 53, 165, 165,
206+
165, 165, 165, 165, 165, 165, 165, 165, 31, 155,
207+
43, 16, 45, 30, 67, 165, 165, 165, 165, 165,
208+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
209+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
210+
165, 69, 165, 161, 36, 165, 160, 165, 165, 165,
211+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
212+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
213+
165, 165, 165, 156, 35, 157, 57, 165, 165, 165,
214+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
215+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
216+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
217+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
218+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
219+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
220+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
221+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
222+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
223+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
224+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
225+
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
226+
165, 165, 165, 165, 165, 165, 1, 2, 3, 4,
226227
5, 6, 7, 9, 10, 11, 12, 13, 14, 15,
227228
17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
228229
27, 28, 29, 32, 33, 34, 38, 39, 40, 41,
@@ -236,7 +237,7 @@ class Php7 extends \PhpParser\ParserAbstract
236237
124, 125, 126, 127, 128, 129, 130, 131, 132, 133,
237238
134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
238239
144, 145, 146, 147, 148, 149, 150, 151, 152, 153,
239-
154
240+
154, 164
240241
);
241242

242243
protected $action = array(

‎lib/PhpParser/Parser/Tokens.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,4 +140,5 @@ final class Tokens
140140
const T_NAME_FULLY_QUALIFIED = 388;
141141
const T_NAME_QUALIFIED = 389;
142142
const T_NAME_RELATIVE = 390;
143+
const T_ATTRIBUTE = 391;
143144
}

‎test/PhpParser/Lexer/EmulativeTest.php

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,33 @@ public function provideTestLexNewFeatures() {
275275
['?->', [
276276
[Tokens::T_NULLSAFE_OBJECT_OPERATOR, '?->'],
277277
]],
278+
['#[Attr]', [
279+
[Tokens::T_ATTRIBUTE, '#['],
280+
[Tokens::T_STRING, 'Attr'],
281+
[ord(']'), ']'],
282+
]],
283+
["#[\nAttr\n]", [
284+
[Tokens::T_ATTRIBUTE, '#['],
285+
[Tokens::T_STRING, 'Attr'],
286+
[ord(']'), ']'],
287+
]],
288+
// Test interaction of two patch-based emulators
289+
["<<<LABEL\n LABEL, #[Attr]", [
290+
[Tokens::T_START_HEREDOC, "<<<LABEL\n"],
291+
[Tokens::T_END_HEREDOC, " LABEL"],
292+
[ord(','), ','],
293+
[Tokens::T_ATTRIBUTE, '#['],
294+
[Tokens::T_STRING, 'Attr'],
295+
[ord(']'), ']'],
296+
]],
297+
["#[Attr] <<<LABEL\n LABEL,", [
298+
[Tokens::T_ATTRIBUTE, '#['],
299+
[Tokens::T_STRING, 'Attr'],
300+
[ord(']'), ']'],
301+
[Tokens::T_START_HEREDOC, "<<<LABEL\n"],
302+
[Tokens::T_END_HEREDOC, " LABEL"],
303+
[ord(','), ','],
304+
]],
278305
];
279306
}
280307

0 commit comments

Comments
 (0)