Merge pull request github#3212 from RasmusWL/python-fix-tests-filter

Python: Fix (some) shortcomings of tests filter

Author: tausbn

python/ql/src/Statements/AssertLiteralConstant.ql

@@ -17,7 +17,7 @@ import semmle.python.filters.Tests
 from Assert a, string value
 where
     /* Exclude asserts inside test cases */
-    not a.getScope() instanceof Test and
+    not a.getScope().getScope*() instanceof TestScope and
     exists(Expr test | test = a.getTest() |
         value = test.(IntegerLiteral).getN()
         or

python/ql/src/semmle/python/filters/Tests.qll

@@ -5,18 +5,17 @@ abstract class TestScope extends Scope { }
 // don't extend Class directly to avoid ambiguous method warnings
 class UnitTestClass extends TestScope {
     UnitTestClass() {
-        exists(ClassObject c | this = c.getPyClass() |
-            c.getASuperType() = theUnitTestPackage().attr(_)
+        exists(ClassValue cls | this = cls.getScope() |
+            cls.getABaseType+() = Module::named("unittest").attr(_)
             or
-            c.getASuperType().getName().toLowerCase() = "testcase"
+            cls.getABaseType+().getName().toLowerCase() = "testcase"
         )
     }
 }
 
-PackageObject theUnitTestPackage() { result.getName() = "unittest" }
-
 abstract class Test extends TestScope { }
 
+/** Class of test function that uses the `unittest` framework */
 class UnitTestFunction extends Test {
     UnitTestFunction() {
         this.getScope+() instanceof UnitTestClass and
@@ -37,3 +36,11 @@ class NoseTestFunction extends Test {
         this.(Function).getName().matches("test%")
     }
 }
+
+/** Class of functions that are clearly tests, but don't belong to a specific framework */
+class UnknownTestFunction extends Test {
+    UnknownTestFunction() {
+        this.(Function).getName().matches("test%") and
+        this.getEnclosingModule().getFile().getShortName().matches("test_%.py")
+    }
+}

python/ql/test/library-tests/filters/tests/Filter.expected

@@ -1,3 +1,6 @@
-| Class MyTest |
-| Function test_1 |
-| Function test_2 |
+| test.py:4:1:4:23 | Class MyTest |
+| test.py:6:5:6:21 | Function test_1 |
+| test.py:9:5:9:21 | Function test_2 |
+| test_foo.py:3:1:3:15 | Function test_foo |
+| unittest_test.py:3:1:3:33 | Class FooTest |
+| unittest_test.py:4:5:4:25 | Function test_valid |

python/ql/test/library-tests/filters/tests/Filter.ql

@@ -2,4 +2,4 @@ import python
 import semmle.python.filters.Tests
 
 from TestScope t
-select t.toString()
+select t

python/ql/test/library-tests/filters/tests/test.py

@@ -1,5 +1,3 @@
-
-
 class TestCase:
     pass
 

python/ql/test/library-tests/filters/tests/test_foo.py

@@ -0,0 +1,4 @@
+# This is running under some unknown framework, but is clearly a test!
+
+def test_foo():
+    assert True

python/ql/test/library-tests/filters/tests/unittest_test.py

@@ -0,0 +1,5 @@
+import unittest
+
+class FooTest(unittest.TestCase):
+    def test_valid(self):
+        pass