Skip to content

Commit ec72178

Browse files
staabmstaabm
committed
WIP: inline token scanning into Tokenizer class
1 parent 2782b4c commit ec72178

File tree

1 file changed

+160
-3
lines changed

1 file changed

+160
-3
lines changed

‎src/CodeCoverage/Util/Tokenizer.php

Lines changed: 160 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,54 @@
11
<?php
22

33
class Tokenizer {
4-
private $tokens;
4+
private $filename;
55
private $linesOfCode = array('loc' => 0, 'cloc' => 0, 'ncloc' => 0);
66

7+
/**
8+
* @var array
9+
*/
10+
protected static $customTokens = array(
11+
'(' => 'PHP_Token_OPEN_BRACKET',
12+
')' => 'PHP_Token_CLOSE_BRACKET',
13+
'[' => 'PHP_Token_OPEN_SQUARE',
14+
']' => 'PHP_Token_CLOSE_SQUARE',
15+
'{' => 'PHP_Token_OPEN_CURLY',
16+
'}' => 'PHP_Token_CLOSE_CURLY',
17+
';' => 'PHP_Token_SEMICOLON',
18+
'.' => 'PHP_Token_DOT',
19+
',' => 'PHP_Token_COMMA',
20+
'=' => 'PHP_Token_EQUAL',
21+
'<' => 'PHP_Token_LT',
22+
'>' => 'PHP_Token_GT',
23+
'+' => 'PHP_Token_PLUS',
24+
'-' => 'PHP_Token_MINUS',
25+
'*' => 'PHP_Token_MULT',
26+
'/' => 'PHP_Token_DIV',
27+
'?' => 'PHP_Token_QUESTION_MARK',
28+
'!' => 'PHP_Token_EXCLAMATION_MARK',
29+
':' => 'PHP_Token_COLON',
30+
'"' => 'PHP_Token_DOUBLE_QUOTES',
31+
'@' => 'PHP_Token_AT',
32+
'&' => 'PHP_Token_AMPERSAND',
33+
'%' => 'PHP_Token_PERCENT',
34+
'|' => 'PHP_Token_PIPE',
35+
'$' => 'PHP_Token_DOLLAR',
36+
'^' => 'PHP_Token_CARET',
37+
'~' => 'PHP_Token_TILDE',
38+
'`' => 'PHP_Token_BACKTICK'
39+
);
40+
741
public function __construct($filename) {
8-
$this->tokens = token_get_all($filename);
42+
$this->filename = $filename;
943
}
1044

1145
public function next() {
12-
$numTokens = count($this->tokens);
46+
$sourceCode = file_get_contents($this->filename);
47+
$tokens = token_get_all($sourceCode);
48+
$numTokens = count($tokens);
1349

1450
for ($i = 0; $i < $numTokens; ++$i) {
51+
$token = $tokens[$i];
1552
if (is_array($token)) {
1653
$name = substr(token_name($token[0]), 2);
1754
$text = $token[1];
@@ -27,6 +64,122 @@ public function next() {
2764
$tokenClass = self::$customTokens[$token];
2865
}
2966

67+
$token = new $tokenClass($text, $line, $this, $i);
68+
switch ($tokenClass) {
69+
case 'PHP_Token_HALT_COMPILER':
70+
break;
71+
72+
case 'PHP_Token_INTERFACE':
73+
$interface = $token->getName();
74+
$interfaceEndLine = $token->getEndLine();
75+
76+
$this->interfaces[$interface] = array(
77+
'methods' => array(),
78+
'parent' => $token->getParent(),
79+
'keywords' => $token->getKeywords(),
80+
'docblock' => $token->getDocblock(),
81+
'startLine' => $token->getLine(),
82+
'endLine' => $interfaceEndLine,
83+
'package' => $token->getPackage(),
84+
'file' => $this->filename
85+
);
86+
break;
87+
88+
case 'PHP_Token_CLASS':
89+
case 'PHP_Token_TRAIT':
90+
$tmp = array(
91+
'methods' => array(),
92+
'parent' => $token->getParent(),
93+
'interfaces'=> $token->getInterfaces(),
94+
'keywords' => $token->getKeywords(),
95+
'docblock' => $token->getDocblock(),
96+
'startLine' => $token->getLine(),
97+
'endLine' => $token->getEndLine(),
98+
'package' => $token->getPackage(),
99+
'file' => $this->filename
100+
);
101+
102+
if ($token instanceof PHP_Token_CLASS) {
103+
$class = $token->getName();
104+
$classEndLine = $token->getEndLine();
105+
$this->classes[$class] = $tmp;
106+
} else {
107+
$trait = $token->getName();
108+
$traitEndLine = $token->getEndLine();
109+
$this->traits[$trait] = $tmp;
110+
}
111+
break;
112+
113+
case 'PHP_Token_FUNCTION':
114+
$name = $token->getName();
115+
$tmp = array(
116+
'docblock' => $token->getDocblock(),
117+
'keywords' => $token->getKeywords(),
118+
'visibility'=> $token->getVisibility(),
119+
'signature' => $token->getSignature(),
120+
'startLine' => $token->getLine(),
121+
'endLine' => $token->getEndLine(),
122+
'ccn' => $token->getCCN(),
123+
'file' => $this->filename
124+
);
125+
126+
if ($class === false &&
127+
$trait === false &&
128+
$interface === false) {
129+
$this->functions[$name] = $tmp;
130+
131+
for ($line = $tmp['startLine']; $line <= $tmp['endLine']; $line++) {
132+
$this->lineToFunctionMap[$line] = $name;
133+
}
134+
} elseif ($class !== false) {
135+
$this->classes[$class]['methods'][$name] = $tmp;
136+
137+
for ($line = $tmp['startLine']; $line <= $tmp['endLine']; $line++) {
138+
$this->lineToFunctionMap[$line] = $class . '::' . $name;
139+
}
140+
} elseif ($trait !== false) {
141+
$this->traits[$trait]['methods'][$name] = $tmp;
142+
143+
for ($line = $tmp['startLine']; $line <= $tmp['endLine']; $line++) {
144+
$this->lineToFunctionMap[$line] = $trait . '::' . $name;
145+
}
146+
} else {
147+
$this->interfaces[$interface]['methods'][$name] = $tmp;
148+
}
149+
break;
150+
151+
case 'PHP_Token_CLOSE_CURLY':
152+
if ($classEndLine !== false &&
153+
$classEndLine == $token->getLine()) {
154+
$class = false;
155+
$classEndLine = false;
156+
} elseif ($traitEndLine !== false &&
157+
$traitEndLine == $token->getLine()) {
158+
$trait = false;
159+
$traitEndLine = false;
160+
} elseif ($interfaceEndLine !== false &&
161+
$interfaceEndLine == $token->getLine()) {
162+
$interface = false;
163+
$interfaceEndLine = false;
164+
}
165+
break;
166+
167+
case 'PHP_Token_COMMENT':
168+
// fall through
169+
case 'PHP_Token_DOC_COMMENT':
170+
$this->linesOfCode['cloc'] += $lines + 1;
171+
break;
172+
}
173+
174+
$lines = substr_count($text, "\n");
175+
$line += $lines;
176+
177+
if ($name == 'DOUBLE_COLON') {
178+
$lastNonWhitespaceTokenWasDoubleColon = true;
179+
} elseif ($name != 'WHITESPACE') {
180+
$lastNonWhitespaceTokenWasDoubleColon = false;
181+
}
182+
30183
if (false) {
31184
yield "class" => [];
32185
} else if (false) {
@@ -35,6 +188,10 @@ public function next() {
35188
yield "function" => [];
36189
}
37190
}
191+
192+
$this->linesOfCode['loc'] = substr_count($sourceCode, "\n");
193+
$this->linesOfCode['ncloc'] = $this->linesOfCode['loc'] -
194+
$this->linesOfCode['cloc'];
38195
}
39196

40197
public function getLinesOfCode() {

0 commit comments

Comments
 (0)