diff --git a/CMakeLists.txt b/CMakeLists.txt index 67a5c85..cc973a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ project(codeql-swift-artifacts C CXX) find_package(LLVM REQUIRED CONFIG) find_package(Clang REQUIRED CONFIG) find_package(Swift REQUIRED CONFIG) +find_package(SwiftSyntax REQUIRED CONFIG) message("Using LLVM_CONFIG: ${Swift_CONFIG}") message("Using Clang_CONFIG: ${LLVM_CONFIG}") @@ -31,4 +32,3 @@ if(APPLE) target_link_directories(codeql-swift-artifacts PUBLIC ${TOOLCHAIN_LIB_DIR}) target_link_directories(codeql-swift-artifacts PUBLIC ${CODEQL_MACOS_SDK_PATH}/usr/lib/swift) endif() - diff --git a/patches/swift-experimental-string-processing/01-add-type-signature.patch b/patches/swift-experimental-string-processing/add-type-signature.patch similarity index 100% rename from patches/swift-experimental-string-processing/01-add-type-signature.patch rename to patches/swift-experimental-string-processing/add-type-signature.patch diff --git a/patches/swift/03-do-not-print-labels-in-function-type.patch b/patches/swift/do-not-print-labels-in-function-type.patch similarity index 100% rename from patches/swift/03-do-not-print-labels-in-function-type.patch rename to patches/swift/do-not-print-labels-in-function-type.patch diff --git a/patches/swift/fix-linux-compilation-with-swift-parser.patch b/patches/swift/fix-linux-compilation-with-swift-parser.patch new file mode 100644 index 0000000..d6a960c --- /dev/null +++ b/patches/swift/fix-linux-compilation-with-swift-parser.patch @@ -0,0 +1,52 @@ +diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake +index 58c7eb1bd4f..4f4d5b7edea 100644 +--- a/cmake/modules/AddSwift.cmake ++++ b/cmake/modules/AddSwift.cmake +@@ -912,17 +912,22 @@ function(add_swift_host_tool executable) + endif() + + if(SWIFT_SWIFT_PARSER) +- set(extra_relative_rpath "") +- if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "") +- if (${executable} MATCHES "-bootstrapping") +- set(extra_relative_rpath "../") ++ if (SWIFT_HOST_VARIANT_SDK STREQUAL LINUX) ++ if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "" AND ${executable} MATCHES "-bootstrapping") ++ set_property(TARGET ${executable} APPEND PROPERTY INSTALL_RPATH "$ORIGIN/../../lib/swift/host") ++ else() ++ set_property(TARGET ${executable} APPEND PROPERTY INSTALL_RPATH "$ORIGIN/../lib/swift/host") ++ endif() ++ else() ++ set(extra_relative_rpath "") ++ if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "" AND ${executable} MATCHES "-bootstrapping") ++ set(extra_relative_rpath "../") + endif() ++ set_property( ++ TARGET ${executable} ++ APPEND PROPERTY INSTALL_RPATH ++ "@executable_path/../${extra_relative_rpath}lib/swift/host") + endif() +- +- set_property( +- TARGET ${executable} +- APPEND PROPERTY INSTALL_RPATH +- "@executable_path/../${extra_relative_rpath}lib/swift/host") + endif() + + if(ASHT_THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY) +diff --git a/lib/ASTGen/CMakeLists.txt b/lib/ASTGen/CMakeLists.txt +index c7334a6da0b..2a0cb80b743 100644 +--- a/lib/ASTGen/CMakeLists.txt ++++ b/lib/ASTGen/CMakeLists.txt +@@ -1,4 +1,11 @@ + if (SWIFT_SWIFT_PARSER) ++ if (SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX") ++ # including libdispatch leads to multiple module definition problems, as it seems ++ # it is already available to the swift toolchain ++ get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) ++ list(REMOVE_ITEM dirs "${SWIFT_PATH_TO_LIBDISPATCH_SOURCE}") ++ set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "${dirs}") ++ endif() + # Ensure that we do not link the _StringProcessing module. But we can + # only pass this flag for new-enough compilers that support it. + file(WRITE "${CMAKE_BINARY_DIR}/tmp/empty-check-string-processing.swift" "") diff --git a/patches/swift/02-handle-empty-pattern-list.patch b/patches/swift/handle-empty-pattern-list.patch similarity index 100% rename from patches/swift/02-handle-empty-pattern-list.patch rename to patches/swift/handle-empty-pattern-list.patch diff --git a/patches/swift/01-remove-assert-in-fallthrough-stmt.patch b/patches/swift/remove-assert-in-fallthrough-stmt.patch similarity index 100% rename from patches/swift/01-remove-assert-in-fallthrough-stmt.patch rename to patches/swift/remove-assert-in-fallthrough-stmt.patch diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 345857d..bd5ea5e 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -14,10 +14,8 @@ def getoptions(): parser = argparse.ArgumentParser(description="package swift for codeql compilation") - parser.add_argument(f"--llvm-build-tree", required=True, type=resolve, - metavar="DIR", help=f"path to LLVM build tree") - parser.add_argument(f"--swift-build-tree", required=True, type=resolve, - metavar="DIR", help=f"path to Swift build tree") + parser.add_argument(f"--build-tree", required=True, type=resolve, + metavar="DIR", help=f"path to the build tree") parser.add_argument(f"--swift-source-tree", required=True, type=resolve, metavar="DIR", help=f"path to Swift source tree") @@ -34,30 +32,30 @@ def getoptions(): return opts -Libs = namedtuple("Libs", ("archive", "static", "shared", "linker_flags")) - EXPORTED_LIB = "CodeQLSwiftFrontendTool" +Libs = namedtuple("Libs", ("static", "shared", "linker_flags")) + def resolve(p): return pathlib.Path(p).resolve() def run(prog, *, cwd, env=None, input=None): - print("running", " ".join(prog), f"(cwd={cwd})") + print("running", *prog, f"(cwd={cwd})") if env is not None: runenv = dict(os.environ) runenv.update(env) else: runenv = None - subprocess.run(prog, cwd=cwd, env=runenv, input=input, text=True) + subprocess.run(prog, cwd=cwd, env=runenv, input=input, text=True, check=True) def get_platform(): return "linux" if platform.system() == "Linux" else "macos" -def configure_dummy_project(tmp, *, llvm=None, swift=None): +def configure_dummy_project(tmp, prefixes): print("configuring dummy cmake project") script_dir = pathlib.Path(os.path.realpath(__file__)).parent print(script_dir) @@ -65,8 +63,8 @@ def configure_dummy_project(tmp, *, llvm=None, swift=None): shutil.copy(script_dir / "empty.cpp", tmp / "empty.cpp") tgt = tmp / "build" tgt.mkdir() - run(["cmake", f"-DCMAKE_PREFIX_PATH={llvm};{swift}", "-DBUILD_SHARED_LIBS=OFF", ".."], - cwd=tgt) + prefixes = ';'.join(str(p) for p in prefixes) + run(["cmake", f"-DCMAKE_PREFIX_PATH={prefixes}", "-DBUILD_SHARED_LIBS=OFF", ".."], cwd=tgt) return tgt @@ -74,17 +72,14 @@ def get_libs(configured): print("extracting linking information from dummy project") with open(configured / "CMakeFiles" / "codeql-swift-artifacts.dir" / "link.txt") as link: libs = link.read().split() - libs = libs[libs.index('codeql-swift-artifacts')+1:] # skip up to -o dummy - ret = Libs([], [], [], []) + libs = libs[libs.index('codeql-swift-artifacts') + 1:] # skip up to -o dummy + ret = Libs([], [], []) for l in libs: if l.endswith(".a"): - ret.static.append(str((configured / l).resolve())) + ret.static.append((configured / l).absolute()) elif l.endswith(".so") or l.endswith(".tbd") or l.endswith(".dylib"): - l = pathlib.Path(l).stem - ret.shared.append(f"-l{l[3:]}") # drop 'lib' prefix and '.so' suffix - elif l.startswith("-l"): - ret.shared.append(l) - elif l.startswith("-L") or l.startswith("-Wl"): + ret.shared.append((configured / l).absolute()) + elif l.startswith(("-L", "-Wl", "-l")): ret.linker_flags.append(l) else: raise ValueError(f"cannot understand link.txt: " + l) @@ -94,19 +89,18 @@ def get_libs(configured): def get_tgt(tgt, filename): if tgt.is_dir(): tgt /= filename - return tgt.resolve() + return tgt.absolute() def create_static_lib(tgt, libs): tgt = get_tgt(tgt, f"lib{EXPORTED_LIB}.a") print(f"packaging {tgt.name}") if sys.platform == 'linux': - includedlibs = "\n".join(f"addlib {l}" for l in libs.archive + libs.static) + includedlibs = "\n".join(f"addlib {l}" for l in libs.static) mriscript = f"create {tgt}\n{includedlibs}\nsave\nend" run(["ar", "-M"], cwd=tgt.parent, input=mriscript) else: libtool_args = ["libtool", "-static"] - libtool_args.extend(libs.archive) libtool_args.extend(libs.static) libtool_args.append("-o") libtool_args.append(str(tgt)) @@ -114,40 +108,8 @@ def create_static_lib(tgt, libs): return tgt -def create_shared_lib(tgt, libs): - ext = "so" - if sys.platform != 'linux': - ext = "dylib" - libname = f"lib{EXPORTED_LIB}.{ext}" - tgt = get_tgt(tgt, libname) - print(f"packaging {libname}") - compiler = os.environ.get("CC", "clang") - cmd = [compiler, "-shared"] - cmd.extend(libs.linker_flags) - - if sys.platform == 'linux': - cmd.append("-Wl,--whole-archive") - else: - cmd.append("-Wl,-all_load") - - cmd.append(f"-o{tgt}") - cmd.extend(libs.archive) - - if sys.platform == 'linux': - cmd.append("-Wl,--no-whole-archive") - else: - cmd.append("-lc++") - - cmd.extend(libs.static) - cmd.extend(libs.shared) - run(cmd, cwd=tgt.parent) - if sys.platform != "linux": - run(["install_name_tool", "-id", f"@executable_path/{libname}", libname], cwd=tgt.parent) - return tgt - - def copy_includes(src, tgt): - print("copying includes") + print(f"copying includes from {src}") for dir, exts in (("include", ("h", "def", "inc")), ("stdlib", ("h",))): srcdir = src / dir for ext in exts: @@ -159,7 +121,7 @@ def copy_includes(src, tgt): def export_sdk(tgt, swift_source_tree, swift_build_tree): print("assembling sdk") - srcdir = swift_build_tree/ "lib" / "swift" + srcdir = swift_build_tree / "lib" / "swift" tgtdir = tgt / "usr" / "lib" / "swift" if get_platform() == "linux": srcdir /= "linux" @@ -168,8 +130,8 @@ def export_sdk(tgt, swift_source_tree, swift_build_tree): srcdir /= "macosx" for mod in srcdir.glob("*.swiftmodule"): shutil.copytree(mod, tgtdir / mod.name) - shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims", - tgt / "usr" / "include" / "SwiftShims", + shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims" / "swift" / "shims", + tgt / "usr" / "lib" / "swift" / "shims", ignore=shutil.ignore_patterns('CMakeLists.txt')) @@ -181,11 +143,11 @@ def export_stdlibs(exported_dir, swift_build_tree): ext = 'so' lib_dir = swift_build_tree / 'lib/swift' / platform stdlibs = [ - f'libswiftCore.{ext}', - 'libswiftCompatibility50.a', - 'libswiftCompatibility51.a', - 'libswiftCompatibilityConcurrency.a', - 'libswiftCompatibilityDynamicReplacements.a'] + f'libswiftCore.{ext}', + 'libswiftCompatibility50.a', + 'libswiftCompatibility51.a', + 'libswiftCompatibilityConcurrency.a', + 'libswiftCompatibilityDynamicReplacements.a'] for stdlib in stdlibs: lib_path = lib_dir / stdlib if lib_path.exists(): @@ -197,13 +159,11 @@ def export_stdlibs(exported_dir, swift_build_tree): def export_libs(exported_dir, libs, swift_build_tree): print("exporting libraries") - exportedlibs = [ - create_static_lib(exported_dir, libs), - create_shared_lib(exported_dir, libs) - ] - - for l in exportedlibs: - l.rename(exported_dir / l.name) + create_static_lib(exported_dir, libs) + for lib in libs.shared: + # export libraries under the build tree (e.g. libSwiftSyntax.so) + if lib.is_relative_to(swift_build_tree.parent): + shutil.copy(lib, exported_dir) export_stdlibs(exported_dir, swift_build_tree) @@ -213,7 +173,8 @@ def export_headers(exported_dir, swift_source_tree, llvm_build_tree, swift_build llvm_source_tree = swift_source_tree.parent / 'llvm-project/llvm' clang_source_tree = swift_source_tree.parent / 'llvm-project/clang' clang_tools_build_tree = llvm_build_tree / 'tools/clang' - header_dirs = [ llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, clang_tools_build_tree ] + header_dirs = [llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, + clang_tools_build_tree] for h in header_dirs: copy_includes(h, exported_dir) @@ -230,18 +191,21 @@ def main(opts): if os.path.exists(tmp): shutil.rmtree(tmp) os.mkdir(tmp) - configured = configure_dummy_project(tmp, llvm=opts.llvm_build_tree, swift=opts.swift_build_tree) + llvm_build_tree = next(opts.build_tree.glob("llvm-*")) + swift_build_tree = next(opts.build_tree.glob("swift-*")) + earlyswiftsyntax_build_tree = next(opts.build_tree.glob("earlyswiftsyntax-*")) + configured = configure_dummy_project(tmp, prefixes=[llvm_build_tree, swift_build_tree, + earlyswiftsyntax_build_tree / "cmake" / "modules"]) libs = get_libs(configured) exported = tmp / "exported" exported.mkdir() - export_libs(exported, libs, opts.swift_build_tree) - export_headers(exported, opts.swift_source_tree, opts.llvm_build_tree, opts.swift_build_tree) - export_sdk(exported / "sdk", opts.swift_source_tree, opts.swift_build_tree) + export_libs(exported, libs, swift_build_tree) + export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree) + export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree) zip_dir(exported, opts.output) if __name__ == "__main__": main(getoptions()) - diff --git a/swift-build-presets b/swift-build-presets index d83f54f..fac6aba 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -1,6 +1,4 @@ [preset: codeql-baseline] -bootstrapping=hosttools - llvm-cmake-options=-DLLVM_ENABLE_TERMINFO=OFF -DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64 skip-ios @@ -28,13 +26,25 @@ skip-build-android enable-experimental-string-processing swift-enable-experimental-string-processing=1 -[preset: codeql] +no-assertions + +reconfigure + +[preset: codeql-release] mixin-preset=codeql-baseline release build-subdir=codeql +[preset: codeql-Linux] +mixin-preset=codeql-release +bootstrapping=hosttools + +[preset: codeql-macOS] +mixin-preset=codeql-release +bootstrapping=bootstrapping + [preset: codeql-debug] mixin-preset=codeql-baseline debug build-subdir=codeql-debug - +bootstrapping=bootstrapping