Skip to content

Commit c8bd22c

Browse files
staabmstaabm
committed
Refactored token logic into CodeCoverage_Token utilising lazy-loading for tname.
1 parent 49b7338 commit c8bd22c

File tree

1 file changed

+149
-145
lines changed

1 file changed

+149
-145
lines changed

‎src/CodeCoverage/Util/Tokenizer.php

Lines changed: 149 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -7,139 +7,10 @@ class PHP_CodeCoverage_Util_Tokenizer {
77
private $traits = [];
88
private $linesOfCode = array('loc' => 0, 'cloc' => 0, 'ncloc' => 0);
99

10-
/**
11-
* @var array
12-
*/
13-
protected static $customTokens = array(
14-
'(' => 'PHP_Token_OPEN_BRACKET',
15-
')' => 'PHP_Token_CLOSE_BRACKET',
16-
'[' => 'PHP_Token_OPEN_SQUARE',
17-
']' => 'PHP_Token_CLOSE_SQUARE',
18-
'{' => 'PHP_Token_OPEN_CURLY',
19-
'}' => 'PHP_Token_CLOSE_CURLY',
20-
';' => 'PHP_Token_SEMICOLON',
21-
'.' => 'PHP_Token_DOT',
22-
',' => 'PHP_Token_COMMA',
23-
'=' => 'PHP_Token_EQUAL',
24-
'<' => 'PHP_Token_LT',
25-
'>' => 'PHP_Token_GT',
26-
'+' => 'PHP_Token_PLUS',
27-
'-' => 'PHP_Token_MINUS',
28-
'*' => 'PHP_Token_MULT',
29-
'/' => 'PHP_Token_DIV',
30-
'?' => 'PHP_Token_QUESTION_MARK',
31-
'!' => 'PHP_Token_EXCLAMATION_MARK',
32-
':' => 'PHP_Token_COLON',
33-
'"' => 'PHP_Token_DOUBLE_QUOTES',
34-
'@' => 'PHP_Token_AT',
35-
'&' => 'PHP_Token_AMPERSAND',
36-
'%' => 'PHP_Token_PERCENT',
37-
'|' => 'PHP_Token_PIPE',
38-
'$' => 'PHP_Token_DOLLAR',
39-
'^' => 'PHP_Token_CARET',
40-
'~' => 'PHP_Token_TILDE',
41-
'`' => 'PHP_Token_BACKTICK'
42-
);
43-
4410
public function __construct($filename) {
4511
$this->filename = $filename;
4612
}
4713

48-
49-
private function tconst($token) {
50-
if (is_array($token)) {
51-
return $token[0];
52-
}
53-
return null;
54-
}
55-
56-
private function tclass($token) {
57-
if (is_array($token)) {
58-
$name = substr(token_name($token[0]), 2);
59-
$text = $token[1];
60-
61-
return 'PHP_Token_' . $name;
62-
} else {
63-
$text = $token;
64-
return self::$customTokens[$token];
65-
}
66-
}
67-
68-
private function tline($idx) {
69-
return $this->tlines[$idx];
70-
}
71-
72-
private function tname(array $tokens, $idx) {
73-
$tconst = $this->tconst($tokens[$idx]);
74-
75-
if ($tconst === T_REQUIRE || $tconst === T_REQUIRE_ONCE || $tconst === T_INCLUDE || $tconst === T_INCLUDE_ONCE) {
76-
if ($this->tconst(tokens[$idx+2]) === T_CONSTANT_ENCAPSED_STRING) {
77-
return trim($this->tstring($tokens[$idx+2]), "'\"");
78-
}
79-
return null;
80-
}
81-
82-
if ($tconst === T_FUNCTION) {
83-
for ($i = $idx + 1; $i < count($tokens); $i++) {
84-
$token = $tokens[$i];
85-
$tconst = $this->tconst($token);
86-
$tclass = $this->tclass($token);
87-
88-
if ($tconst === T_STRING) {
89-
$name = $this->tstring($token);
90-
break;
91-
} elseif ($tclass === 'PHP_Token_AMPERSAND' && $this->tconst($tokens[$i+1]) === T_STRING) {
92-
$name = $this->tstring($tokens[$i+1]);
93-
break;
94-
} elseif ($tclass === 'PHP_Token_OPEN_BRACKET') {
95-
$name = 'anonymous function';
96-
break;
97-
}
98-
}
99-
100-
if ($name != 'anonymous function') {
101-
for ($i = $idx; $i; --$i) {
102-
$tconst = $this->tconst($tokens[$i]);
103-
if ($tconst === T_NAMESPACE) {
104-
$name = $this->tname($tokens, $i) . '\\' . $name;
105-
break;
106-
}
107-
108-
if ($tconst === T_INTERFACE || $tconst === T_CLASS || $tconst === T_TRAIT) {
109-
break;
110-
}
111-
}
112-
}
113-
114-
return $name;
115-
}
116-
117-
if ($tconst === T_INTERFACE || $tconst === T_CLASS || $tconst === T_TRAIT) {
118-
return $this->tstring($tokens[$idx + 2]);
119-
}
120-
121-
if ($tconst === T_NAMESPACE) {
122-
$namespace = $this->tstring($tokens[$idx+2]);
123-
124-
for ($i = $idx + 3;; $i += 2) {
125-
if (isset($tokens[$i]) && $this->tconst($tokens[$i]) === T_NS_SEPARATOR) {
126-
$namespace .= '\\' . $this->tstring($tokens[$i+1]);
127-
} else {
128-
break;
129-
}
130-
}
131-
132-
return $namespace;
133-
}
134-
}
135-
136-
private function tstring($token) {
137-
if (is_array($token)) {
138-
return $token[1];
139-
}
140-
return $token;
141-
}
142-
14314
/**
14415
* @return string
14516
*/
@@ -441,20 +312,21 @@ public function tokenize() {
441312
$tokens = token_get_all($sourceCode);
442313
$numTokens = count($tokens);
443314

444-
// precalculate in which line the tokens reside, for later lookaheads
445315
$line = 1;
446316
$ccTokens = array();
447317
for ($i = 0; $i < $numTokens; ++$i) {
448-
$preToken = $tokens[$i];
449-
$text = $this->tstring($preToken);
450-
451-
$ccTokens[] = new CodeCoverage_Token(
452-
$this->tname($tokens, $i),
453-
$this->tconst($preToken),
454-
$this->tclass($preToken),
455-
$text,
318+
$nativeToken = $tokens[$i];
319+
unset($tokens[$i]);
320+
321+
$ccToken = new CodeCoverage_Token(
322+
$nativeToken,
323+
$i,
456324
$line
457325
);
326+
$ccToken->allTokens =& $ccTokens;
327+
$ccTokens[$i] = $ccToken;
328+
329+
$text = $ccToken->tstring;
458330

459331
$lines = substr_count($text, "\n");
460332
$line += $lines;
@@ -590,18 +462,150 @@ public function getFunctions() {
590462
}
591463
}
592464

465+
/**
466+
* @property tname
467+
*/
593468
class CodeCoverage_Token {
594-
public $tname;
595-
public $tconst;
469+
/**
470+
* @var CodeCoverage_Token[] $allTokens
471+
*/
472+
public $allTokens;
473+
/**
474+
* PHP native token provided by token_get_all().
475+
* @var string|array
476+
*/
477+
private $nativeToken;
478+
private $idx;
479+
596480
public $tclass;
481+
public $tconst;
597482
public $tstring;
598483
public $tline;
599484

600-
public function __construct($tname, $tconst, $tclass, $tstring, $tline) {
601-
$this->tname = $tname;
602-
$this->tconst = $tconst;
603-
$this->tclass = $tclass;
604-
$this->tstring = $tstring;
485+
/**
486+
* @var array
487+
*/
488+
protected static $customTokens = array(
489+
'(' => 'PHP_Token_OPEN_BRACKET',
490+
')' => 'PHP_Token_CLOSE_BRACKET',
491+
'[' => 'PHP_Token_OPEN_SQUARE',
492+
']' => 'PHP_Token_CLOSE_SQUARE',
493+
'{' => 'PHP_Token_OPEN_CURLY',
494+
'}' => 'PHP_Token_CLOSE_CURLY',
495+
';' => 'PHP_Token_SEMICOLON',
496+
'.' => 'PHP_Token_DOT',
497+
',' => 'PHP_Token_COMMA',
498+
'=' => 'PHP_Token_EQUAL',
499+
'<' => 'PHP_Token_LT',
500+
'>' => 'PHP_Token_GT',
501+
'+' => 'PHP_Token_PLUS',
502+
'-' => 'PHP_Token_MINUS',
503+
'*' => 'PHP_Token_MULT',
504+
'/' => 'PHP_Token_DIV',
505+
'?' => 'PHP_Token_QUESTION_MARK',
506+
'!' => 'PHP_Token_EXCLAMATION_MARK',
507+
':' => 'PHP_Token_COLON',
508+
'"' => 'PHP_Token_DOUBLE_QUOTES',
509+
'@' => 'PHP_Token_AT',
510+
'&' => 'PHP_Token_AMPERSAND',
511+
'%' => 'PHP_Token_PERCENT',
512+
'|' => 'PHP_Token_PIPE',
513+
'$' => 'PHP_Token_DOLLAR',
514+
'^' => 'PHP_Token_CARET',
515+
'~' => 'PHP_Token_TILDE',
516+
'`' => 'PHP_Token_BACKTICK'
517+
);
518+
519+
public function __construct($nativeToken, $idx, $tline) {
520+
$this->nativeToken = $nativeToken;
521+
$this->idx = $idx;
605522
$this->tline = $tline;
523+
$this->tclass = is_array($nativeToken) ? 'PHP_Token_' . substr(token_name($nativeToken[0]), 2) : self::$customTokens[$nativeToken];
524+
$this->tconst = is_array($nativeToken) ? $nativeToken[0] : $nativeToken;
525+
$this->tstring = is_array($nativeToken) ? $nativeToken[1] : $nativeToken;
526+
}
527+
528+
public function __destruct() {
529+
$this->nativeToken = null;
530+
}
531+
532+
public function __get($key) {
533+
// lazy init properties which are not that likely to be required
534+
// and cost us too much to preload
535+
switch($key) {
536+
case "tname":
537+
$val = $this->$key($this->allTokens, $this->idx);
538+
// free this maybe big array as soon as possible
539+
$this->nativeToken = null;
540+
break;
541+
default: throw new Exception("undefined property ". $key);
542+
}
543+
544+
$this->$key = $val;
545+
return $val;
546+
}
547+
548+
private function tname(array $tokens, $idx) {
549+
$tconst = $this->tconst;
550+
551+
if ($tconst === T_REQUIRE || $tconst === T_REQUIRE_ONCE || $tconst === T_INCLUDE || $tconst === T_INCLUDE_ONCE) {
552+
if ($tokens[$idx+2]->tconst === T_CONSTANT_ENCAPSED_STRING) {
553+
return trim($tokens[$idx+2]->tstring, "'\"");
554+
}
555+
return null;
556+
}
557+
558+
if ($tconst === T_FUNCTION) {
559+
for ($i = $idx + 1; $i < count($tokens); $i++) {
560+
$token = $tokens[$i];
561+
$tconst = $token->tconst;
562+
$tclass = $token->tclass;
563+
564+
if ($tconst === T_STRING) {
565+
$name = $token->tstring;
566+
break;
567+
} elseif ($tclass === 'PHP_Token_AMPERSAND' && $tokens[$i+1]->tconst === T_STRING) {
568+
$name = $tokens[$i+1]->tstring;
569+
break;
570+
} elseif ($tclass === 'PHP_Token_OPEN_BRACKET') {
571+
$name = 'anonymous function';
572+
break;
573+
}
574+
}
575+
576+
if ($name != 'anonymous function') {
577+
for ($i = $idx; $i; --$i) {
578+
$tconst = $tokens[$i]->tconst;
579+
if ($tconst === T_NAMESPACE) {
580+
$name = $tokens[$i]->tname . '\\' . $name;
581+
break;
582+
}
583+
584+
if ($tconst === T_INTERFACE || $tconst === T_CLASS || $tconst === T_TRAIT) {
585+
break;
586+
}
587+
}
588+
}
589+
590+
return $name;
591+
}
592+
593+
if ($tconst === T_INTERFACE || $tconst === T_CLASS || $tconst === T_TRAIT) {
594+
return $tokens[$idx + 2]->tstring;
595+
}
596+
597+
if ($tconst === T_NAMESPACE) {
598+
$namespace = $tokens[$idx+2]->tstring;
599+
600+
for ($i = $idx + 3;; $i += 2) {
601+
if (isset($tokens[$i]) && $tokens[$i]->tconst === T_NS_SEPARATOR) {
602+
$namespace .= '\\' . $tokens[$i+1]->tstring;
603+
} else {
604+
break;
605+
}
606+
}
607+
608+
return $namespace;
609+
}
606610
}
607611
}

0 commit comments

Comments
 (0)