WIP added Tokenizer instead of PHP_Token_stream

Author: staabm

src/CodeCoverage/Report/Node/File.php

@@ -351,17 +351,23 @@ public function getNumTestedFunctions()
      */
     protected function calculateStatistics()
     {
-        if ($this->cacheTokens) {
-            $tokens = PHP_Token_Stream_CachingFactory::get($this->getPath());
-        } else {
-            $tokens = new PHP_Token_Stream($this->getPath());
+        $tokenizer = new Tokenizer($this->getPath());
+        $classes = $traits = $functions = array();
+        foreach($tokenizer as $tokenName => $token) {
+            switch ($tokenName) {
+                case "class":
+                    $classes[] = $token;
+                    break;
+                case "trait":
+                    $traits[] = $token;
+                    break;
+                case "function":
+                    $functions[] = $token;
+                    break;
+            }
         }
-
-        $this->processClasses($tokens->getClasses());
-        $this->processTraits($tokens->getTraits());
-        $this->processFunctions($tokens->getFunctions());
-        $this->linesOfCode = $tokens->getLinesOfCode();
-        unset($tokens);
+        $this->linesOfCode = $tokenizer->getLinesOfCode();
+        unset($tokenizer);
 
         for ($lineNumber = 1; $lineNumber <= $this->linesOfCode['loc']; $lineNumber++) {
             if (isset($this->startLines[$lineNumber])) {

src/CodeCoverage/Util/Tokenizer.php

@@ -0,0 +1,43 @@
+<?php
+
+class Tokenizer {
+    private $tokens;
+    private $linesOfCode = array('loc' => 0, 'cloc' => 0, 'ncloc' => 0);
+
+    public function __construct($filename) {
+        $this->tokens = token_get_all($filename);
+    }
+
+    public function next() {
+        $numTokens = count($this->tokens);
+
+        for ($i = 0; $i < $numTokens; ++$i) {
+            if (is_array($token)) {
+                $name = substr(token_name($token[0]), 2);
+                $text = $token[1];
+
+                if ($lastNonWhitespaceTokenWasDoubleColon && $name == 'CLASS') {
+                    $name = 'CLASS_NAME_CONSTANT';
+                }
+
+                $tokenClass = 'PHP_Token_' . $name;
+
+            } else {
+                $text       = $token;
+                $tokenClass = self::$customTokens[$token];
+            }
+
+            if (false) {
+                yield "class" => [];
+            } else if (false) {
+                yield "trait" => [];
+            } else if (false) {
+                yield "function" => [];
+            }
+        }
+    }
+
+    public function getLinesOfCode() {
+        return $this->linesOfCode;
+    }
+}