diff --git a/CMakeLists.txt b/CMakeLists.txt index cc973a1..fb6985e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,9 @@ find_package(Clang REQUIRED CONFIG) find_package(Swift REQUIRED CONFIG) find_package(SwiftSyntax REQUIRED CONFIG) +set(THREADS_PREFER_PTHREAD_FLAG YES) +include(FindThreads) + message("Using LLVM_CONFIG: ${Swift_CONFIG}") message("Using Clang_CONFIG: ${LLVM_CONFIG}") message("Using Swift_CONFIG: ${Clang_CONFIG}") @@ -13,6 +16,14 @@ message("Using Swift_CONFIG: ${Clang_CONFIG}") add_executable(codeql-swift-artifacts empty.cpp) target_link_libraries(codeql-swift-artifacts PRIVATE LLVMSupport swiftFrontendTool swiftCompilerModules) +set (CODEQL_SWIFT_VERSION_MAJOR ${SWIFT_VERSION_MAJOR}) +set (CODEQL_SWIFT_VERSION_MINOR ${SWIFT_VERSION_MINOR}) +set (CODEQL_SWIFT_VERSION_PATCH ${SWIFT_VERSION_PATCH}) + +# TODO: these are now upstreamed +configure_file(${CMAKE_SOURCE_DIR}/CodeQLSwiftVersion.h.in + ${SWIFT_BINARY_DIR}/include/swift/CodeQLSwiftVersion.h) + if(APPLE) execute_process( COMMAND xcrun -find swiftc diff --git a/CodeQLSwiftVersion.h.in b/CodeQLSwiftVersion.h.in new file mode 100644 index 0000000..535107a --- /dev/null +++ b/CodeQLSwiftVersion.h.in @@ -0,0 +1,9 @@ +#ifndef CODEQL_SWIFT_VERSION_H +#define CODEQL_SWIFT_VERSION_H + +#cmakedefine CODEQL_SWIFT_VERSION_MAJOR @CODEQL_SWIFT_VERSION_MAJOR@ +#cmakedefine CODEQL_SWIFT_VERSION_MINOR @CODEQL_SWIFT_VERSION_MINOR@ +#cmakedefine CODEQL_SWIFT_VERSION_PATCH @CODEQL_SWIFT_VERSION_PATCH@ + +#endif // CODEQL_SWIFT_VERSION_H + diff --git a/patches/swift-experimental-string-processing/add-type-signature.patch b/patches/swift-experimental-string-processing/add-type-signature.patch deleted file mode 100644 index ab06857..0000000 --- a/patches/swift-experimental-string-processing/add-type-signature.patch +++ /dev/null @@ -1,15 +0,0 @@ -Swift 5.5.2/Xcode 13.2 refuses to compile this piece as it is "unable to infer complex closure return type." - -diff --git a/Sources/_RegexParser/Utility/TypeConstruction.swift b/Sources/_RegexParser/Utility/TypeConstruction.swift -index 4d1765e..39b4595 100644 ---- a/Sources/_RegexParser/Utility/TypeConstruction.swift -+++ b/Sources/_RegexParser/Utility/TypeConstruction.swift -@@ -60,7 +60,7 @@ public enum TypeConstruction { - flags |= 0x10000 - } - -- let result = elementTypes.withContiguousStorageIfAvailable { elementTypesBuffer in -+ let result = elementTypes.withContiguousStorageIfAvailable { elementTypesBuffer -> (value: Any.Type, state: Int) in - if let labels = labels { - return labels.withCString { labelsPtr in - swift_getTupleTypeMetadata( diff --git a/patches/swift/extend-swift-observer.patch b/patches/swift/extend-swift-observer.patch new file mode 100644 index 0000000..52a5a1b --- /dev/null +++ b/patches/swift/extend-swift-observer.patch @@ -0,0 +1,61 @@ +diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h +index c47fdaae886..aa9c265d6d3 100644 +--- a/include/swift/Frontend/FrontendOptions.h ++++ b/include/swift/Frontend/FrontendOptions.h +@@ -212,6 +212,9 @@ public: + /// The path to which we should output statistics files. + std::string StatsOutputDir; + ++ /// CodeQL: Prevent ASTContext from being freed during at the frontend ++ bool KeepASTContext = false; ++ + /// Trace changes to stats to files in StatsOutputDir. + bool TraceStats = false; + +diff --git a/include/swift/FrontendTool/FrontendTool.h b/include/swift/FrontendTool/FrontendTool.h +index 184e6196918..8bc237725b5 100644 +--- a/include/swift/FrontendTool/FrontendTool.h ++++ b/include/swift/FrontendTool/FrontendTool.h +@@ -46,6 +46,9 @@ public: + /// The frontend has performed semantic analysis. + virtual void performedSemanticAnalysis(CompilerInstance &instance); + ++ /// CodeQL: The frontend has performed compilation. ++ virtual void performedCompilation(CompilerInstance &instance); ++ + /// The frontend has performed basic SIL generation. + /// SIL diagnostic passes have not yet been applied. + virtual void performedSILGeneration(SILModule &module); +diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp +index 47b8883f7e2..4080c02d6b0 100644 +--- a/lib/FrontendTool/FrontendTool.cpp ++++ b/lib/FrontendTool/FrontendTool.cpp +@@ -1572,6 +1572,11 @@ static bool validateTBDIfNeeded(const CompilerInvocation &Invocation, + } + + static void freeASTContextIfPossible(CompilerInstance &Instance) { ++ // CodeQL: keep ASTContext until we are done with the extraction ++ if (Instance.getInvocation().getFrontendOptions().KeepASTContext) { ++ return; ++ } ++ + // If the stats reporter is installed, we need the ASTContext to live through + // the entire compilation process. + if (Instance.getASTContext().Stats) { +@@ -2321,6 +2326,10 @@ int swift::performFrontend(ArrayRef Args, + + int ReturnValue = 0; + bool HadError = performCompile(*Instance, ReturnValue, observer); ++ // Compilation happened ++ if (observer) { ++ observer->performedCompilation(*Instance); ++ } + + if (verifierEnabled) { + DiagnosticEngine &diags = Instance->getDiags(); +@@ -2348,3 +2357,5 @@ void FrontendObserver::configuredCompiler(CompilerInstance &instance) {} + void FrontendObserver::performedSemanticAnalysis(CompilerInstance &instance) {} + void FrontendObserver::performedSILGeneration(SILModule &module) {} + void FrontendObserver::performedSILProcessing(SILModule &module) {} ++// CodeQL: Add another hook right after compilation so that we can run the extraction ++void FrontendObserver::performedCompilation(CompilerInstance &instance) {} diff --git a/patches/swift/handle-empty-pattern-list.patch b/patches/swift/handle-empty-pattern-list.patch deleted file mode 100644 index 2010ee8..0000000 --- a/patches/swift/handle-empty-pattern-list.patch +++ /dev/null @@ -1,19 +0,0 @@ -`PatternBindingDecl::getPatternList()` can in some cases return an empty list, which causes a -segmentation fault when getting the source range. This case needs to be handled for the extractor. - -diff --git a/lib/AST/Decl.cpp b/lib/AST/Decl.cpp -index 75b99a22e73..09115678a82 100644 ---- a/lib/AST/Decl.cpp -+++ b/lib/AST/Decl.cpp -@@ -1725,7 +1725,10 @@ StringRef PatternBindingEntry::getInitStringRepresentation( - - SourceRange PatternBindingDecl::getSourceRange() const { - SourceLoc startLoc = getStartLoc(); -- SourceLoc endLoc = getPatternList().back().getSourceRange().End; -+ SourceLoc endLoc = startLoc; -+ if (!getPatternList().empty()) { -+ endLoc = getPatternList().back().getSourceRange().End; -+ } - if (startLoc.isValid() != endLoc.isValid()) return SourceRange(); - return { startLoc, endLoc }; - } diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index bd5ea5e..c36c060 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -61,6 +61,7 @@ def configure_dummy_project(tmp, prefixes): print(script_dir) shutil.copy(script_dir / "CMakeLists.txt", tmp / "CMakeLists.txt") shutil.copy(script_dir / "empty.cpp", tmp / "empty.cpp") + shutil.copy(script_dir / "CodeQLSwiftVersion.h.in", tmp / "CodeQLSwiftVersion.h.in") tgt = tmp / "build" tgt.mkdir() prefixes = ';'.join(str(p) for p in prefixes) @@ -79,7 +80,7 @@ def get_libs(configured): ret.static.append((configured / l).absolute()) elif l.endswith(".so") or l.endswith(".tbd") or l.endswith(".dylib"): ret.shared.append((configured / l).absolute()) - elif l.startswith(("-L", "-Wl", "-l")): + elif l.startswith(("-L", "-Wl", "-l")) or l == "-pthread": ret.linker_flags.append(l) else: raise ValueError(f"cannot understand link.txt: " + l) @@ -118,43 +119,22 @@ def copy_includes(src, tgt): tgtfile.parent.mkdir(parents=True, exist_ok=True) shutil.copy(srcfile, tgtfile) - -def export_sdk(tgt, swift_source_tree, swift_build_tree): - print("assembling sdk") - srcdir = swift_build_tree / "lib" / "swift" - tgtdir = tgt / "usr" / "lib" / "swift" - if get_platform() == "linux": - srcdir /= "linux" - tgtdir /= "linux/x86_64" - else: - srcdir /= "macosx" - for mod in srcdir.glob("*.swiftmodule"): - shutil.copytree(mod, tgtdir / mod.name) - shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims" / "swift" / "shims", - tgt / "usr" / "lib" / "swift" / "shims", - ignore=shutil.ignore_patterns('CMakeLists.txt')) - - def export_stdlibs(exported_dir, swift_build_tree): ext = 'dylib' - platform = 'macosx' - if get_platform() == "linux": - platform = 'linux' - ext = 'so' + platform = 'linux' if get_platform() == 'linux' else 'macosx' lib_dir = swift_build_tree / 'lib/swift' / platform - stdlibs = [ - f'libswiftCore.{ext}', - 'libswiftCompatibility50.a', - 'libswiftCompatibility51.a', - 'libswiftCompatibilityConcurrency.a', - 'libswiftCompatibilityDynamicReplacements.a'] - for stdlib in stdlibs: - lib_path = lib_dir / stdlib - if lib_path.exists(): + patterns = [f'lib{dep}.*' for dep in ( + "dispatch", + "BlocksRuntime", + "swiftCore", + "swift_*", + "swiftGlibc", + "swiftCompatibility*", + )] + for pattern in patterns: + for stdlib in lib_dir.glob(pattern): print(f'Copying {stdlib}') - shutil.copy(lib_path, exported_dir) - else: - print(f'Skipping {stdlib}') + shutil.copy(stdlib, exported_dir) def export_libs(exported_dir, libs, swift_build_tree): @@ -193,16 +173,14 @@ def main(opts): os.mkdir(tmp) llvm_build_tree = next(opts.build_tree.glob("llvm-*")) swift_build_tree = next(opts.build_tree.glob("swift-*")) - earlyswiftsyntax_build_tree = next(opts.build_tree.glob("earlyswiftsyntax-*")) configured = configure_dummy_project(tmp, prefixes=[llvm_build_tree, swift_build_tree, - earlyswiftsyntax_build_tree / "cmake" / "modules"]) + swift_build_tree / 'cmake' / 'modules']) libs = get_libs(configured) exported = tmp / "exported" exported.mkdir() export_libs(exported, libs, swift_build_tree) export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree) - export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree) zip_dir(exported, opts.output) diff --git a/swift-build-presets b/swift-build-presets index fac6aba..44b67ee 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -1,10 +1,6 @@ [preset: codeql-baseline] llvm-cmake-options=-DLLVM_ENABLE_TERMINFO=OFF -DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64 -skip-ios -skip-tvos -skip-watchos - skip-test-osx skip-test-linux skip-test-swiftpm @@ -15,10 +11,13 @@ skip-test-sourcekit-lsp skip-test-playgroundsupport skip-test-skstresstester skip-test-swiftformat -skip-test-swiftevolve skip-test-toolchain-benchmarks skip-test-swift-inspect skip-test-swift +skip-build-xros +skip-test-xros +skip-test-xros-host +skip-test-xros-simulator skip-build-clang-tools-extra skip-build-benchmarks @@ -27,9 +26,17 @@ enable-experimental-string-processing swift-enable-experimental-string-processing=1 no-assertions - reconfigure +install-swift +install-swiftsyntax +install-swift-testing +install-swift-testing-macros +install-prefix=/codeql-toolchain + +swift-testing +swift-testing-macros + [preset: codeql-release] mixin-preset=codeql-baseline release