Add USE_* and MAGIC_* flag constants

This drops the effective dependency on tokenizer.

Also replace token numbers with %d as these are unstable.

Author: nikic

README.md

@@ -245,7 +245,7 @@ ast\flags\BINARY_IS_GREATER          // since version 20
 ast\flags\BINARY_IS_GREATER_OR_EQUAL // since version 20
 ast\flags\BINARY_SPACESHIP
 
-// Used by ast\AST_ASSIGN_OP in version 10 (exclusive)
+// Used by ast\AST_ASSIGN_OP in versions before 20 (exclusive)
 ast\flags\ASSIGN_BITWISE_OR
 ast\flags\ASSIGN_BITWISE_AND
 ast\flags\ASSIGN_BITWISE_XOR
@@ -260,21 +260,19 @@ ast\flags\ASSIGN_SHIFT_LEFT
 ast\flags\ASSIGN_SHIFT_RIGHT
 
 // Used by ast\AST_MAGIC_CONST (exclusive)
-// (Constants defined by ext\tokenizer)
-T_LINE
-T_FILE
-T_DIR
-T_TRAIT_C
-T_METHOD_C
-T_FUNC_C
-T_NS_C
-T_CLASS_C
+ast\flags\MAGIC_LINE
+ast\flags\MAGIC_FILE
+ast\flags\MAGIC_DIR
+ast\flags\MAGIC_NAMESPACE
+ast\flags\MAGIC_FUNCTION
+ast\flags\MAGIC_METHOD
+ast\flags\MAGIC_CLASS
+ast\flags\MAGIC_TRAIT
 
 // Used by ast\AST_USE, ast\AST_GROUP_USE and ast\AST_USE_ELEM (exclusive)
-// (Constants defined by ext\tokenizer)
-T_CLASS
-T_FUNCTION
-T_CONST
+ast\flags\USE_NORMAL
+ast\flags\USE_FUNCTION
+ast\flags\USE_CONST
 
 // Used by ast\AST_INCLUDE_OR_EVAL (exclusive)
 ast\flags\EXEC_EVAL

ast.c

@@ -10,6 +10,7 @@
 
 #include "zend_language_scanner.h"
 #include "zend_language_scanner_defs.h"
+#include "zend_language_parser.h"
 #include "zend_exceptions.h"
 #include "zend_smart_str.h"
 
@@ -616,6 +617,19 @@ PHP_MINIT_FUNCTION(ast) {
 	ast_register_flag_constant("EXEC_REQUIRE", ZEND_REQUIRE);
 	ast_register_flag_constant("EXEC_REQUIRE_ONCE", ZEND_REQUIRE_ONCE);
 
+	ast_register_flag_constant("USE_NORMAL", T_CLASS);
+	ast_register_flag_constant("USE_FUNCTION", T_FUNCTION);
+	ast_register_flag_constant("USE_CONST", T_CONST);
+
+	ast_register_flag_constant("MAGIC_LINE", T_LINE);
+	ast_register_flag_constant("MAGIC_FILE", T_FILE);
+	ast_register_flag_constant("MAGIC_DIR", T_DIR);
+	ast_register_flag_constant("MAGIC_NAMESPACE", T_NS_C);
+	ast_register_flag_constant("MAGIC_FUNCTION", T_FUNC_C);
+	ast_register_flag_constant("MAGIC_METHOD", T_METHOD_C);
+	ast_register_flag_constant("MAGIC_CLASS", T_CLASS_C);
+	ast_register_flag_constant("MAGIC_TRAIT", T_TRAIT_C);
+
 	INIT_CLASS_ENTRY(tmp_ce, "ast\\Node", NULL);
 	ast_node_ce = zend_register_internal_class(&tmp_ce);
 

tests/magic_constants.phpt

@@ -0,0 +1,40 @@
+--TEST--
+Magic constants
+--FILE--
+<?php
+
+require __DIR__ . '/../util.php';
+
+$code = <<<'PHP'
+<?php
+__LINE__;
+__FILE__;
+__DIR__;
+__NAMESPACE__;
+__FUNCTION__;
+__METHOD__;
+__CLASS__;
+__TRAIT__;
+PHP;
+
+echo ast_dump(ast\parse_code($code, $version=15));
+
+?>
+--EXPECTF--
+AST_STMT_LIST
+    0: AST_MAGIC_CONST
+        flags: MAGIC_LINE (%d)
+    1: AST_MAGIC_CONST
+        flags: MAGIC_FILE (%d)
+    2: AST_MAGIC_CONST
+        flags: MAGIC_DIR (%d)
+    3: AST_MAGIC_CONST
+        flags: MAGIC_NAMESPACE (%d)
+    4: AST_MAGIC_CONST
+        flags: MAGIC_FUNCTION (%d)
+    5: AST_MAGIC_CONST
+        flags: MAGIC_METHOD (%d)
+    6: AST_MAGIC_CONST
+        flags: MAGIC_CLASS (%d)
+    7: AST_MAGIC_CONST
+        flags: MAGIC_TRAIT (%d)

tests/use_declarations.phpt

@@ -16,16 +16,16 @@ PHP;
 echo ast_dump(ast\parse_code($code, $version=15));
 
 ?>
---EXPECT--
+--EXPECTF--
 AST_STMT_LIST
     0: AST_USE
-        flags: T_CLASS (361)
+        flags: USE_NORMAL (%d)
         0: AST_USE_ELEM
             flags: 0
             0: "Foo\Bar"
             1: "Baz"
     1: AST_USE
-        flags: T_FUNCTION (346)
+        flags: USE_FUNCTION (%d)
         0: AST_USE_ELEM
             flags: 0
             0: "foo\bar"
@@ -36,15 +36,15 @@ AST_STMT_LIST
         1: AST_USE
             flags: 0
             0: AST_USE_ELEM
-                flags: T_CLASS (361)
+                flags: USE_NORMAL (%d)
                 0: "Bar"
                 1: null
             1: AST_USE_ELEM
-                flags: T_FUNCTION (346)
+                flags: USE_FUNCTION (%d)
                 0: "bar"
                 1: null
     3: AST_GROUP_USE
-        flags: T_FUNCTION (346)
+        flags: USE_FUNCTION (%d)
         0: "foo"
         1: AST_USE
             flags: 0

util.php

@@ -30,9 +30,9 @@ function get_flag_info() : array {
         flags\TYPE_CALLABLE => 'TYPE_CALLABLE',
     ];
     $useTypes = [
-        T_CLASS => 'T_CLASS',
-        T_FUNCTION => 'T_FUNCTION',
-        T_CONST => 'T_CONST',
+        flags\USE_NORMAL => 'USE_NORMAL',
+        flags\USE_FUNCTION => 'USE_FUNCTION',
+        flags\USE_CONST => 'USE_CONST',
     ];
     $sharedBinaryOps = [
         flags\BINARY_BITWISE_OR => 'BINARY_BITWISE_OR',
@@ -105,14 +105,14 @@ function get_flag_info() : array {
             flags\ASSIGN_SHIFT_RIGHT => 'ASSIGN_SHIFT_RIGHT',
         ],
         ast\AST_MAGIC_CONST => [
-            T_LINE => 'T_LINE',
-            T_FILE => 'T_FILE',
-            T_DIR => 'T_DIR',
-            T_TRAIT_C => 'T_TRAIT_C',
-            T_METHOD_C => 'T_METHOD_C',
-            T_FUNC_C => 'T_FUNC_C',
-            T_NS_C => 'T_NS_C',
-            T_CLASS_C => 'T_CLASS_C',
+            flags\MAGIC_LINE => 'MAGIC_LINE',
+            flags\MAGIC_FILE => 'MAGIC_FILE',
+            flags\MAGIC_DIR => 'MAGIC_DIR',
+            flags\MAGIC_NAMESPACE => 'MAGIC_NAMESPACE',
+            flags\MAGIC_FUNCTION => 'MAGIC_FUNCTION',
+            flags\MAGIC_METHOD => 'MAGIC_METHOD',
+            flags\MAGIC_CLASS => 'MAGIC_CLASS',
+            flags\MAGIC_TRAIT => 'MAGIC_TRAIT',
         ],
         ast\AST_USE => $useTypes,
         ast\AST_GROUP_USE => $useTypes,