From 75dc189702d444f214ddd63bb436419702e0183e Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Wed, 19 Apr 2023 12:42:01 +0200 Subject: [PATCH 01/22] Add more swift libs --- pkg_swift_llvm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index bd5ea5e..31af96a 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -146,6 +146,8 @@ def export_stdlibs(exported_dir, swift_build_tree): f'libswiftCore.{ext}', 'libswiftCompatibility50.a', 'libswiftCompatibility51.a', + 'libswiftCompatibility56.a', + 'libswiftCompatibilityBytecodeLayouts.a', 'libswiftCompatibilityConcurrency.a', 'libswiftCompatibilityDynamicReplacements.a'] for stdlib in stdlibs: From f7ff5a9a506122dd531315642ff7eac1c2b6a1e9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 19 Apr 2023 13:39:31 +0200 Subject: [PATCH 02/22] Add regex parser lib and turn compatibility libs to a glob --- pkg_swift_llvm.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 31af96a..cc63e14 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -143,20 +143,13 @@ def export_stdlibs(exported_dir, swift_build_tree): ext = 'so' lib_dir = swift_build_tree / 'lib/swift' / platform stdlibs = [ - f'libswiftCore.{ext}', - 'libswiftCompatibility50.a', - 'libswiftCompatibility51.a', - 'libswiftCompatibility56.a', - 'libswiftCompatibilityBytecodeLayouts.a', - 'libswiftCompatibilityConcurrency.a', - 'libswiftCompatibilityDynamicReplacements.a'] + lib_dir / f'libswiftCore.{ext}', + lib_dir / f'libswift_RegexParser.{ext}', + ] + stdlibs.extend(lib_dir.glob('libswiftCompatibility*.a')) for stdlib in stdlibs: - lib_path = lib_dir / stdlib - if lib_path.exists(): - print(f'Copying {stdlib}') - shutil.copy(lib_path, exported_dir) - else: - print(f'Skipping {stdlib}') + print(f'Copying {stdlib}') + shutil.copy(stdlib, exported_dir) def export_libs(exported_dir, libs, swift_build_tree): From 4c67171bed63dd78f3e8c33896a5555bc16d9355 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 19 Apr 2023 13:54:10 +0200 Subject: [PATCH 03/22] Add Glibc on Linux --- pkg_swift_llvm.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index cc63e14..ec116c9 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -137,19 +137,18 @@ def export_sdk(tgt, swift_source_tree, swift_build_tree): def export_stdlibs(exported_dir, swift_build_tree): ext = 'dylib' - platform = 'macosx' - if get_platform() == "linux": - platform = 'linux' - ext = 'so' + platform = 'linux' if get_platform() == 'linux' else 'macosx' lib_dir = swift_build_tree / 'lib/swift' / platform - stdlibs = [ - lib_dir / f'libswiftCore.{ext}', - lib_dir / f'libswift_RegexParser.{ext}', - ] - stdlibs.extend(lib_dir.glob('libswiftCompatibility*.a')) - for stdlib in stdlibs: - print(f'Copying {stdlib}') - shutil.copy(stdlib, exported_dir) + patterns = [f'libswift{dep}.*' for dep in ( + "Core", + "_RegexParser", + "Glibc", + "Compatibility*", + )] + for pattern in patterns: + for stdlib in lib_dir.glob(pattern): + print(f'Copying {stdlib}') + shutil.copy(stdlib, exported_dir) def export_libs(exported_dir, libs, swift_build_tree): From 0a01cb83ea87c97f162df6fbcab3e1f4db898341 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 19 Apr 2023 17:25:46 +0200 Subject: [PATCH 04/22] Include more dynamic libraries --- pkg_swift_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index ec116c9..8b24e2f 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -141,7 +141,7 @@ def export_stdlibs(exported_dir, swift_build_tree): lib_dir = swift_build_tree / 'lib/swift' / platform patterns = [f'libswift{dep}.*' for dep in ( "Core", - "_RegexParser", + "_*", "Glibc", "Compatibility*", )] From 422cee13b2708a4b69b06a37b7222e6e32085de7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 20 Apr 2023 09:05:38 +0200 Subject: [PATCH 05/22] Add libdispatch.so --- pkg_swift_llvm.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 8b24e2f..a7bdec0 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -139,11 +139,12 @@ def export_stdlibs(exported_dir, swift_build_tree): ext = 'dylib' platform = 'linux' if get_platform() == 'linux' else 'macosx' lib_dir = swift_build_tree / 'lib/swift' / platform - patterns = [f'libswift{dep}.*' for dep in ( - "Core", - "_*", - "Glibc", - "Compatibility*", + patterns = [f'lib{dep}.*' for dep in ( + "dispatch", + "swiftCore", + "swift_*", + "swiftGlibc", + "swiftCompatibility*", )] for pattern in patterns: for stdlib in lib_dir.glob(pattern): From b7317b5264b748d243bddae03d501e70713e1a2a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 20 Apr 2023 17:24:48 +0200 Subject: [PATCH 06/22] Add `libBlocksRuntime.so` --- pkg_swift_llvm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index a7bdec0..fac4166 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -141,6 +141,7 @@ def export_stdlibs(exported_dir, swift_build_tree): lib_dir = swift_build_tree / 'lib/swift' / platform patterns = [f'lib{dep}.*' for dep in ( "dispatch", + "BlocksRuntime", "swiftCore", "swift_*", "swiftGlibc", From 9745ded65bf77e236677c5473108f40ffe83d140 Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Thu, 4 May 2023 15:21:51 +0200 Subject: [PATCH 07/22] Package resource-dir --- pkg_swift_llvm.py | 8 ++++++++ swift-build-presets | 7 +++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index fac4166..ab900ca 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -133,6 +133,13 @@ def export_sdk(tgt, swift_source_tree, swift_build_tree): shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims" / "swift" / "shims", tgt / "usr" / "lib" / "swift" / "shims", ignore=shutil.ignore_patterns('CMakeLists.txt')) + shutil.copytree(swift_build_tree/ "lib" / "swift", + tgt / "resource-dir") + + +def export_resource_dir(tgt, swift_build_tree): + print("assembling resource dir") + shutil.copytree(swift_build_tree/ "lib" / "swift", tgt) def export_stdlibs(exported_dir, swift_build_tree): @@ -199,6 +206,7 @@ def main(opts): export_libs(exported, libs, swift_build_tree) export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree) export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree) + export_resource_dir(exported / "resource-dir", swift_build_tree) zip_dir(exported, opts.output) diff --git a/swift-build-presets b/swift-build-presets index fac6aba..9e5b9bc 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -1,10 +1,6 @@ [preset: codeql-baseline] llvm-cmake-options=-DLLVM_ENABLE_TERMINFO=OFF -DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64 -skip-ios -skip-tvos -skip-watchos - skip-test-osx skip-test-linux skip-test-swiftpm @@ -41,6 +37,9 @@ bootstrapping=hosttools [preset: codeql-macOS] mixin-preset=codeql-release +ios +tvos +watchos bootstrapping=bootstrapping [preset: codeql-debug] From 4261066dd1fb7fe830c9405942c118b3ba3f5cc7 Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Wed, 31 May 2023 15:40:28 +0200 Subject: [PATCH 08/22] Swift: package whole toolchain instead of just resource dir --- pkg_swift_llvm.py | 11 +++++------ swift-build-presets | 4 +++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index ab900ca..ba6ebde 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -133,13 +133,11 @@ def export_sdk(tgt, swift_source_tree, swift_build_tree): shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims" / "swift" / "shims", tgt / "usr" / "lib" / "swift" / "shims", ignore=shutil.ignore_patterns('CMakeLists.txt')) - shutil.copytree(swift_build_tree/ "lib" / "swift", - tgt / "resource-dir") -def export_resource_dir(tgt, swift_build_tree): - print("assembling resource dir") - shutil.copytree(swift_build_tree/ "lib" / "swift", tgt) +def export_toolchain(tgt, toolchain_dir): + print("assembling toolchain") + shutil.copytree(toolchain_dir, tgt) def export_stdlibs(exported_dir, swift_build_tree): @@ -196,6 +194,7 @@ def main(opts): os.mkdir(tmp) llvm_build_tree = next(opts.build_tree.glob("llvm-*")) swift_build_tree = next(opts.build_tree.glob("swift-*")) + toolchain_dir = next(opts.build_tree.glob("toolchain-*/codeql-toolchain")) earlyswiftsyntax_build_tree = next(opts.build_tree.glob("earlyswiftsyntax-*")) configured = configure_dummy_project(tmp, prefixes=[llvm_build_tree, swift_build_tree, earlyswiftsyntax_build_tree / "cmake" / "modules"]) @@ -206,7 +205,7 @@ def main(opts): export_libs(exported, libs, swift_build_tree) export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree) export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree) - export_resource_dir(exported / "resource-dir", swift_build_tree) + export_toolchain(exported / "toolchain", toolchain_dir) zip_dir(exported, opts.output) diff --git a/swift-build-presets b/swift-build-presets index 9e5b9bc..65edc33 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -23,9 +23,11 @@ enable-experimental-string-processing swift-enable-experimental-string-processing=1 no-assertions - reconfigure +install-swift +install-prefix=/codeql-toolchain + [preset: codeql-release] mixin-preset=codeql-baseline release From 31aaf4d05b64e36b410462470f7c393442dab1db Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Mon, 26 Jun 2023 13:36:58 +0200 Subject: [PATCH 09/22] Extend Swift frontend observer --- patches/swift/extend-swift-observer.patch | 61 +++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 patches/swift/extend-swift-observer.patch diff --git a/patches/swift/extend-swift-observer.patch b/patches/swift/extend-swift-observer.patch new file mode 100644 index 0000000..52a5a1b --- /dev/null +++ b/patches/swift/extend-swift-observer.patch @@ -0,0 +1,61 @@ +diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h +index c47fdaae886..aa9c265d6d3 100644 +--- a/include/swift/Frontend/FrontendOptions.h ++++ b/include/swift/Frontend/FrontendOptions.h +@@ -212,6 +212,9 @@ public: + /// The path to which we should output statistics files. + std::string StatsOutputDir; + ++ /// CodeQL: Prevent ASTContext from being freed during at the frontend ++ bool KeepASTContext = false; ++ + /// Trace changes to stats to files in StatsOutputDir. + bool TraceStats = false; + +diff --git a/include/swift/FrontendTool/FrontendTool.h b/include/swift/FrontendTool/FrontendTool.h +index 184e6196918..8bc237725b5 100644 +--- a/include/swift/FrontendTool/FrontendTool.h ++++ b/include/swift/FrontendTool/FrontendTool.h +@@ -46,6 +46,9 @@ public: + /// The frontend has performed semantic analysis. + virtual void performedSemanticAnalysis(CompilerInstance &instance); + ++ /// CodeQL: The frontend has performed compilation. ++ virtual void performedCompilation(CompilerInstance &instance); ++ + /// The frontend has performed basic SIL generation. + /// SIL diagnostic passes have not yet been applied. + virtual void performedSILGeneration(SILModule &module); +diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp +index 47b8883f7e2..4080c02d6b0 100644 +--- a/lib/FrontendTool/FrontendTool.cpp ++++ b/lib/FrontendTool/FrontendTool.cpp +@@ -1572,6 +1572,11 @@ static bool validateTBDIfNeeded(const CompilerInvocation &Invocation, + } + + static void freeASTContextIfPossible(CompilerInstance &Instance) { ++ // CodeQL: keep ASTContext until we are done with the extraction ++ if (Instance.getInvocation().getFrontendOptions().KeepASTContext) { ++ return; ++ } ++ + // If the stats reporter is installed, we need the ASTContext to live through + // the entire compilation process. + if (Instance.getASTContext().Stats) { +@@ -2321,6 +2326,10 @@ int swift::performFrontend(ArrayRef Args, + + int ReturnValue = 0; + bool HadError = performCompile(*Instance, ReturnValue, observer); ++ // Compilation happened ++ if (observer) { ++ observer->performedCompilation(*Instance); ++ } + + if (verifierEnabled) { + DiagnosticEngine &diags = Instance->getDiags(); +@@ -2348,3 +2357,5 @@ void FrontendObserver::configuredCompiler(CompilerInstance &instance) {} + void FrontendObserver::performedSemanticAnalysis(CompilerInstance &instance) {} + void FrontendObserver::performedSILGeneration(SILModule &module) {} + void FrontendObserver::performedSILProcessing(SILModule &module) {} ++// CodeQL: Add another hook right after compilation so that we can run the extraction ++void FrontendObserver::performedCompilation(CompilerInstance &instance) {} From f88e51e2888e6eff70a2b785888e5b0d0d0fd34c Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Tue, 27 Jun 2023 17:04:33 +0200 Subject: [PATCH 10/22] Swift: drop old patch --- .../add-type-signature.patch | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 patches/swift-experimental-string-processing/add-type-signature.patch diff --git a/patches/swift-experimental-string-processing/add-type-signature.patch b/patches/swift-experimental-string-processing/add-type-signature.patch deleted file mode 100644 index ab06857..0000000 --- a/patches/swift-experimental-string-processing/add-type-signature.patch +++ /dev/null @@ -1,15 +0,0 @@ -Swift 5.5.2/Xcode 13.2 refuses to compile this piece as it is "unable to infer complex closure return type." - -diff --git a/Sources/_RegexParser/Utility/TypeConstruction.swift b/Sources/_RegexParser/Utility/TypeConstruction.swift -index 4d1765e..39b4595 100644 ---- a/Sources/_RegexParser/Utility/TypeConstruction.swift -+++ b/Sources/_RegexParser/Utility/TypeConstruction.swift -@@ -60,7 +60,7 @@ public enum TypeConstruction { - flags |= 0x10000 - } - -- let result = elementTypes.withContiguousStorageIfAvailable { elementTypesBuffer in -+ let result = elementTypes.withContiguousStorageIfAvailable { elementTypesBuffer -> (value: Any.Type, state: Int) in - if let labels = labels { - return labels.withCString { labelsPtr in - swift_getTupleTypeMetadata( From af59f081a383b311f9c1e6defe0b260e9f8c92da Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Thu, 29 Jun 2023 13:29:59 +0200 Subject: [PATCH 11/22] Swift: enable maccatalyst --- swift-build-presets | 1 + 1 file changed, 1 insertion(+) diff --git a/swift-build-presets b/swift-build-presets index 65edc33..6393dd2 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -42,6 +42,7 @@ mixin-preset=codeql-release ios tvos watchos +maccatalyst bootstrapping=bootstrapping [preset: codeql-debug] From d79eb4d9e40cd734d30abf166d71edf8fa2137ec Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Thu, 6 Jul 2023 11:52:05 +0200 Subject: [PATCH 12/22] Swift: expose Swift version --- CMakeLists.txt | 7 +++++++ CodeQLSwiftVersion.h.in | 9 +++++++++ patches/swift/expose-swift-version.patch | 15 +++++++++++++++ pkg_swift_llvm.py | 1 + 4 files changed, 32 insertions(+) create mode 100644 CodeQLSwiftVersion.h.in create mode 100644 patches/swift/expose-swift-version.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index cc973a1..482df3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,13 @@ message("Using Swift_CONFIG: ${Clang_CONFIG}") add_executable(codeql-swift-artifacts empty.cpp) target_link_libraries(codeql-swift-artifacts PRIVATE LLVMSupport swiftFrontendTool swiftCompilerModules) +set (CODEQL_SWIFT_VERSION_MAJOR ${SWIFT_VERSION_MAJOR}) +set (CODEQL_SWIFT_VERSION_MINOR ${SWIFT_VERSION_MINOR}) +set (CODEQL_SWIFT_VERSION_PATCH ${SWIFT_VERSION_PATCH}) + +configure_file(${CMAKE_SOURCE_DIR}/CodeQLSwiftVersion.h.in + ${SWIFT_BINARY_DIR}/include/swift/CodeQLSwiftVersion.h) + if(APPLE) execute_process( COMMAND xcrun -find swiftc diff --git a/CodeQLSwiftVersion.h.in b/CodeQLSwiftVersion.h.in new file mode 100644 index 0000000..535107a --- /dev/null +++ b/CodeQLSwiftVersion.h.in @@ -0,0 +1,9 @@ +#ifndef CODEQL_SWIFT_VERSION_H +#define CODEQL_SWIFT_VERSION_H + +#cmakedefine CODEQL_SWIFT_VERSION_MAJOR @CODEQL_SWIFT_VERSION_MAJOR@ +#cmakedefine CODEQL_SWIFT_VERSION_MINOR @CODEQL_SWIFT_VERSION_MINOR@ +#cmakedefine CODEQL_SWIFT_VERSION_PATCH @CODEQL_SWIFT_VERSION_PATCH@ + +#endif // CODEQL_SWIFT_VERSION_H + diff --git a/patches/swift/expose-swift-version.patch b/patches/swift/expose-swift-version.patch new file mode 100644 index 0000000..928654e --- /dev/null +++ b/patches/swift/expose-swift-version.patch @@ -0,0 +1,15 @@ +diff --git a/cmake/modules/SwiftConfig.cmake.in b/cmake/modules/SwiftConfig.cmake.in +index 262ed49d8fc..e13988f1513 100644 +--- a/cmake/modules/SwiftConfig.cmake.in ++++ b/cmake/modules/SwiftConfig.cmake.in +@@ -2,6 +2,10 @@ + + @SWIFT_CONFIG_CODE@ + ++set(SWIFT_VERSION_MAJOR @SWIFT_VERSION_MAJOR@) ++set(SWIFT_VERSION_MINOR @SWIFT_VERSION_MINOR@) ++set(SWIFT_VERSION_PATCH @SWIFT_VERSION_PATCHLEVEL@) ++ + set(SWIFT_VERSION @SWIFT_VERSION@) + set(SWIFT_MAIN_SRC_DIR @SWIFT_SOURCE_DIR@) + diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index ba6ebde..18543bc 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -61,6 +61,7 @@ def configure_dummy_project(tmp, prefixes): print(script_dir) shutil.copy(script_dir / "CMakeLists.txt", tmp / "CMakeLists.txt") shutil.copy(script_dir / "empty.cpp", tmp / "empty.cpp") + shutil.copy(script_dir / "CodeQLSwiftVersion.h.in", tmp / "CodeQLSwiftVersion.h.in") tgt = tmp / "build" tgt.mkdir() prefixes = ';'.join(str(p) for p in prefixes) From ea6fae3cb6f5ce01ce853f815e114e5421009f38 Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Thu, 14 Sep 2023 11:33:10 +0200 Subject: [PATCH 13/22] Swift 5.9: drop unused patch --- patches/swift/handle-empty-pattern-list.patch | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 patches/swift/handle-empty-pattern-list.patch diff --git a/patches/swift/handle-empty-pattern-list.patch b/patches/swift/handle-empty-pattern-list.patch deleted file mode 100644 index 2010ee8..0000000 --- a/patches/swift/handle-empty-pattern-list.patch +++ /dev/null @@ -1,19 +0,0 @@ -`PatternBindingDecl::getPatternList()` can in some cases return an empty list, which causes a -segmentation fault when getting the source range. This case needs to be handled for the extractor. - -diff --git a/lib/AST/Decl.cpp b/lib/AST/Decl.cpp -index 75b99a22e73..09115678a82 100644 ---- a/lib/AST/Decl.cpp -+++ b/lib/AST/Decl.cpp -@@ -1725,7 +1725,10 @@ StringRef PatternBindingEntry::getInitStringRepresentation( - - SourceRange PatternBindingDecl::getSourceRange() const { - SourceLoc startLoc = getStartLoc(); -- SourceLoc endLoc = getPatternList().back().getSourceRange().End; -+ SourceLoc endLoc = startLoc; -+ if (!getPatternList().empty()) { -+ endLoc = getPatternList().back().getSourceRange().End; -+ } - if (startLoc.isValid() != endLoc.isValid()) return SourceRange(); - return { startLoc, endLoc }; - } From 6ece629ed21a3390a27ba1dce1a64392a2e8f30b Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Fri, 3 Nov 2023 13:15:21 +0100 Subject: [PATCH 14/22] Do not export toolchain --- pkg_swift_llvm.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 18543bc..a6f8e51 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -136,11 +136,6 @@ def export_sdk(tgt, swift_source_tree, swift_build_tree): ignore=shutil.ignore_patterns('CMakeLists.txt')) -def export_toolchain(tgt, toolchain_dir): - print("assembling toolchain") - shutil.copytree(toolchain_dir, tgt) - - def export_stdlibs(exported_dir, swift_build_tree): ext = 'dylib' platform = 'linux' if get_platform() == 'linux' else 'macosx' @@ -195,7 +190,6 @@ def main(opts): os.mkdir(tmp) llvm_build_tree = next(opts.build_tree.glob("llvm-*")) swift_build_tree = next(opts.build_tree.glob("swift-*")) - toolchain_dir = next(opts.build_tree.glob("toolchain-*/codeql-toolchain")) earlyswiftsyntax_build_tree = next(opts.build_tree.glob("earlyswiftsyntax-*")) configured = configure_dummy_project(tmp, prefixes=[llvm_build_tree, swift_build_tree, earlyswiftsyntax_build_tree / "cmake" / "modules"]) @@ -206,7 +200,6 @@ def main(opts): export_libs(exported, libs, swift_build_tree) export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree) export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree) - export_toolchain(exported / "toolchain", toolchain_dir) zip_dir(exported, opts.output) From 6b1015bde882911044182f292cdba507e85420b2 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 13 Feb 2024 06:39:57 +0100 Subject: [PATCH 15/22] Remove unneeded toolchain options --- swift-build-presets | 4 ---- 1 file changed, 4 deletions(-) diff --git a/swift-build-presets b/swift-build-presets index 6393dd2..ec0e6a1 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -39,10 +39,6 @@ bootstrapping=hosttools [preset: codeql-macOS] mixin-preset=codeql-release -ios -tvos -watchos -maccatalyst bootstrapping=bootstrapping [preset: codeql-debug] From ac5e8bd6a22ccfeea2b94a50cbcacb4f6fc8e8f0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 19 Feb 2024 13:27:15 +0100 Subject: [PATCH 16/22] Remove unneeded sdk --- pkg_swift_llvm.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index a6f8e51..5d95c7c 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -119,23 +119,6 @@ def copy_includes(src, tgt): tgtfile.parent.mkdir(parents=True, exist_ok=True) shutil.copy(srcfile, tgtfile) - -def export_sdk(tgt, swift_source_tree, swift_build_tree): - print("assembling sdk") - srcdir = swift_build_tree / "lib" / "swift" - tgtdir = tgt / "usr" / "lib" / "swift" - if get_platform() == "linux": - srcdir /= "linux" - tgtdir /= "linux/x86_64" - else: - srcdir /= "macosx" - for mod in srcdir.glob("*.swiftmodule"): - shutil.copytree(mod, tgtdir / mod.name) - shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims" / "swift" / "shims", - tgt / "usr" / "lib" / "swift" / "shims", - ignore=shutil.ignore_patterns('CMakeLists.txt')) - - def export_stdlibs(exported_dir, swift_build_tree): ext = 'dylib' platform = 'linux' if get_platform() == 'linux' else 'macosx' @@ -199,7 +182,6 @@ def main(opts): exported.mkdir() export_libs(exported, libs, swift_build_tree) export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree) - export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree) zip_dir(exported, opts.output) From 4f48027fbff59575e0a16ea0235ba3b67de73a69 Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Tue, 19 Mar 2024 14:05:14 +0100 Subject: [PATCH 17/22] Swift: this patch was upstreamed --- patches/swift/expose-swift-version.patch | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 patches/swift/expose-swift-version.patch diff --git a/patches/swift/expose-swift-version.patch b/patches/swift/expose-swift-version.patch deleted file mode 100644 index 928654e..0000000 --- a/patches/swift/expose-swift-version.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/cmake/modules/SwiftConfig.cmake.in b/cmake/modules/SwiftConfig.cmake.in -index 262ed49d8fc..e13988f1513 100644 ---- a/cmake/modules/SwiftConfig.cmake.in -+++ b/cmake/modules/SwiftConfig.cmake.in -@@ -2,6 +2,10 @@ - - @SWIFT_CONFIG_CODE@ - -+set(SWIFT_VERSION_MAJOR @SWIFT_VERSION_MAJOR@) -+set(SWIFT_VERSION_MINOR @SWIFT_VERSION_MINOR@) -+set(SWIFT_VERSION_PATCH @SWIFT_VERSION_PATCHLEVEL@) -+ - set(SWIFT_VERSION @SWIFT_VERSION@) - set(SWIFT_MAIN_SRC_DIR @SWIFT_SOURCE_DIR@) - From ca25dc3ff51f37d986e7120a8c9c294441640d5a Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Tue, 19 Mar 2024 14:30:45 +0100 Subject: [PATCH 18/22] Swift: remove unsupported argument --- swift-build-presets | 1 - 1 file changed, 1 deletion(-) diff --git a/swift-build-presets b/swift-build-presets index ec0e6a1..fd132c3 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -11,7 +11,6 @@ skip-test-sourcekit-lsp skip-test-playgroundsupport skip-test-skstresstester skip-test-swiftformat -skip-test-swiftevolve skip-test-toolchain-benchmarks skip-test-swift-inspect skip-test-swift From b3495a5b302fd8a62ccb82e617c8de0c0f88fbbc Mon Sep 17 00:00:00 2001 From: Alex Denisov Date: Tue, 19 Mar 2024 19:26:40 +0100 Subject: [PATCH 19/22] Fix CMake setup --- CMakeLists.txt | 4 ++++ pkg_swift_llvm.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 482df3d..fb6985e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,9 @@ find_package(Clang REQUIRED CONFIG) find_package(Swift REQUIRED CONFIG) find_package(SwiftSyntax REQUIRED CONFIG) +set(THREADS_PREFER_PTHREAD_FLAG YES) +include(FindThreads) + message("Using LLVM_CONFIG: ${Swift_CONFIG}") message("Using Clang_CONFIG: ${LLVM_CONFIG}") message("Using Swift_CONFIG: ${Clang_CONFIG}") @@ -17,6 +20,7 @@ set (CODEQL_SWIFT_VERSION_MAJOR ${SWIFT_VERSION_MAJOR}) set (CODEQL_SWIFT_VERSION_MINOR ${SWIFT_VERSION_MINOR}) set (CODEQL_SWIFT_VERSION_PATCH ${SWIFT_VERSION_PATCH}) +# TODO: these are now upstreamed configure_file(${CMAKE_SOURCE_DIR}/CodeQLSwiftVersion.h.in ${SWIFT_BINARY_DIR}/include/swift/CodeQLSwiftVersion.h) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 5d95c7c..0dd63b9 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -173,9 +173,8 @@ def main(opts): os.mkdir(tmp) llvm_build_tree = next(opts.build_tree.glob("llvm-*")) swift_build_tree = next(opts.build_tree.glob("swift-*")) - earlyswiftsyntax_build_tree = next(opts.build_tree.glob("earlyswiftsyntax-*")) configured = configure_dummy_project(tmp, prefixes=[llvm_build_tree, swift_build_tree, - earlyswiftsyntax_build_tree / "cmake" / "modules"]) + swift_build_tree / 'cmake' / 'modules']) libs = get_libs(configured) exported = tmp / "exported" From 602dde56169972448668c01cc72ee5ff71672e5a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Oct 2024 15:45:28 +0200 Subject: [PATCH 20/22] Allow `-pthread` in `link.txt` --- pkg_swift_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 0dd63b9..c36c060 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -80,7 +80,7 @@ def get_libs(configured): ret.static.append((configured / l).absolute()) elif l.endswith(".so") or l.endswith(".tbd") or l.endswith(".dylib"): ret.shared.append((configured / l).absolute()) - elif l.startswith(("-L", "-Wl", "-l")): + elif l.startswith(("-L", "-Wl", "-l")) or l == "-pthread": ret.linker_flags.append(l) else: raise ValueError(f"cannot understand link.txt: " + l) From f5788f75ce1fb752ed7714ad5e1d44c2c1c04950 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Oct 2024 15:50:17 +0200 Subject: [PATCH 21/22] Use hosttools bootstrapping for mac as well --- swift-build-presets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-build-presets b/swift-build-presets index fd132c3..7d2bffb 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -38,7 +38,7 @@ bootstrapping=hosttools [preset: codeql-macOS] mixin-preset=codeql-release -bootstrapping=bootstrapping +bootstrapping=hosttools [preset: codeql-debug] mixin-preset=codeql-baseline From 12a2975206bb254d6003beb872839b35ebaaacc0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 14 Oct 2024 09:50:50 +0200 Subject: [PATCH 22/22] Revert using hosttools for macOS, add other options --- swift-build-presets | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/swift-build-presets b/swift-build-presets index 7d2bffb..44b67ee 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -14,6 +14,10 @@ skip-test-swiftformat skip-test-toolchain-benchmarks skip-test-swift-inspect skip-test-swift +skip-build-xros +skip-test-xros +skip-test-xros-host +skip-test-xros-simulator skip-build-clang-tools-extra skip-build-benchmarks @@ -25,8 +29,14 @@ no-assertions reconfigure install-swift +install-swiftsyntax +install-swift-testing +install-swift-testing-macros install-prefix=/codeql-toolchain +swift-testing +swift-testing-macros + [preset: codeql-release] mixin-preset=codeql-baseline release @@ -38,7 +48,7 @@ bootstrapping=hosttools [preset: codeql-macOS] mixin-preset=codeql-release -bootstrapping=hosttools +bootstrapping=bootstrapping [preset: codeql-debug] mixin-preset=codeql-baseline