diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..67a5c85 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.12.4) +project(codeql-swift-artifacts C CXX) + +find_package(LLVM REQUIRED CONFIG) +find_package(Clang REQUIRED CONFIG) +find_package(Swift REQUIRED CONFIG) + +message("Using LLVM_CONFIG: ${Swift_CONFIG}") +message("Using Clang_CONFIG: ${LLVM_CONFIG}") +message("Using Swift_CONFIG: ${Clang_CONFIG}") + +add_executable(codeql-swift-artifacts empty.cpp) +target_link_libraries(codeql-swift-artifacts PRIVATE LLVMSupport swiftFrontendTool swiftCompilerModules) + +if(APPLE) + execute_process( + COMMAND xcrun -find swiftc + OUTPUT_VARIABLE CODEQL_SWIFT_COMPILER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND xcrun -show-sdk-path + OUTPUT_VARIABLE CODEQL_MACOS_SDK_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # Add in the toolchain directory so we can grab compatibility libraries + # Inspired by the Swift's CMakeLists + get_filename_component(TOOLCHAIN_BIN_DIR ${CODEQL_SWIFT_COMPILER} DIRECTORY) + get_filename_component(TOOLCHAIN_LIB_DIR "${TOOLCHAIN_BIN_DIR}/../lib/swift/macosx" ABSOLUTE) + target_link_directories(codeql-swift-artifacts PUBLIC ${TOOLCHAIN_LIB_DIR}) + target_link_directories(codeql-swift-artifacts PUBLIC ${CODEQL_MACOS_SDK_PATH}/usr/lib/swift) +endif() + diff --git a/empty.cpp b/empty.cpp new file mode 100644 index 0000000..e69de29 diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 460a31d..03313a0 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -11,98 +11,33 @@ import zlib from collections import namedtuple -DEPS = {"llvm": ["LLVMSupport"], - "swift": ["swiftFrontendTool"]} - def getoptions(): parser = argparse.ArgumentParser(description="package swift for codeql compilation") - for p in DEPS: - parser.add_argument(f"--{p}", required=True, type=resolve, - metavar="DIR", help=f"path to {p} build root") + parser.add_argument(f"--llvm-build-tree", required=True, type=resolve, + metavar="DIR", help=f"path to LLVM build tree") + parser.add_argument(f"--swift-build-tree", required=True, type=resolve, + metavar="DIR", help=f"path to Swift build tree") + parser.add_argument(f"--swift-source-tree", required=True, type=resolve, + metavar="DIR", help=f"path to Swift source tree") + default_output = f"swift-prebuilt-{get_platform()}" - parser.add_argument("--keep-tmp-dir", "-K", action="store_true", - help="do not clean up the temporary directory") parser.add_argument("--output", "-o", type=pathlib.Path, metavar="DIR_OR_ZIP", help="output zip file or directory " f"(by default the filename is {default_output})") - update_help_fmt = "Only update the {} library in DIR, triggering rebuilds of required files" - parser.add_argument("--update-shared", "-u", metavar="DIR", type=pathlib.Path, - help=update_help_fmt.format("shared")) - parser.add_argument("--update-static", "-U", metavar="DIR", type=pathlib.Path, - help=update_help_fmt.format("static")) + opts = parser.parse_args() - if opts.output and (opts.update_shared or opts.update_static): - parser.error("provide --output or one of --update-*, not both") if opts.output is None: opts.output = pathlib.Path() opts.output = get_tgt(opts.output, default_output) - return opts - -Libs = namedtuple("Libs", ("archive", "static", "shared")) - -DEPLIST = [x for d in DEPS.values() for x in d] - -CMAKELISTS_DUMMY = f""" -cmake_minimum_required(VERSION 3.12.4) - -project(dummy C CXX) + return opts -find_package(LLVM REQUIRED CONFIG PATHS ${{LLVM_ROOT}}/lib/cmake/llvm NO_DEFAULT_PATH) -find_package(Clang REQUIRED CONFIG PATHS ${{LLVM_ROOT}}/lib/cmake/clang NO_DEFAULT_PATH) -find_package(Swift REQUIRED CONFIG PATHS ${{SWIFT_ROOT}}/lib/cmake/swift NO_DEFAULT_PATH) -add_executable(dummy empty.cpp) -target_link_libraries(dummy PRIVATE {" ".join(DEPLIST)}) -""" +Libs = namedtuple("Libs", ("archive", "static", "shared", "linker_flags")) EXPORTED_LIB = "CodeQLSwiftFrontendTool" -CMAKELISTS_EXPORTED_FMT = """ -add_library({exported} INTERFACE) - -if (BUILD_SHARED_LIBS) - if (APPLE) - set(EXT "dylib") - else() - set(EXT "so") - endif() -else() - set(EXT "a") -endif() - -set (SwiftLLVMWrapperLib libCodeQLSwiftFrontendTool.${{EXT}}) -set (input ${{CMAKE_CURRENT_LIST_DIR}}/${{SwiftLLVMWrapperLib}}) -set (output ${{CMAKE_BINARY_DIR}}/${{SwiftLLVMWrapperLib}}) - -add_custom_command(OUTPUT ${{output}} - COMMAND ${{CMAKE_COMMAND}} -E copy_if_different ${{input}} ${{output}} - DEPENDS ${{input}}) -add_custom_target(copy-llvm-swift-wrapper DEPENDS ${{output}}) - -target_include_directories({exported} INTERFACE ${{CMAKE_CURRENT_LIST_DIR}}/include) -target_link_libraries({exported} INTERFACE - ${{output}} - {libs} -) -add_dependencies(swiftAndLlvmSupport copy-llvm-swift-wrapper) -""" - - -class TempDir: - def __init__(self, cleanup=True): - self.path = None - self.cleanup = cleanup - - def __enter__(self): - self.path = pathlib.Path(tempfile.mkdtemp()) - return self.path - - def __exit__(self, *args): - if self.cleanup: - shutil.rmtree(self.path) - def resolve(p): return pathlib.Path(p).resolve() @@ -118,53 +53,29 @@ def run(prog, *, cwd, env=None, input=None): subprocess.run(prog, cwd=cwd, env=runenv, input=input, text=True) -def build(dir, targets): - print(f"building {' '.join(targets)} in {dir}") - cmd = ["cmake", "--build", ".", "--"] - cmd.extend(targets) - run(cmd, cwd=dir) - - def get_platform(): return "linux" if platform.system() == "Linux" else "macos" -def create_empty_cpp(path): - with open(path / "empty.cpp", "w"): - pass - - -def install(tmp, opts): - print("installing dependencies") - tgt = tmp / "install" - for p in DEPS: - builddir = getattr(opts, p) - run(["cmake", "--build", ".", "--", "install"], cwd=builddir, env={"DESTDIR": tgt}) - if sys.platform != 'linux': - return tgt / "Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain" - return tgt - - -def configure_dummy_project(tmp, *, llvm=None, swift=None, installed=None): +def configure_dummy_project(tmp, *, llvm=None, swift=None): print("configuring dummy cmake project") - if installed is not None: - swift = llvm = installed / "usr" - with open(tmp / "CMakeLists.txt", "w") as out: - out.write(CMAKELISTS_DUMMY) - create_empty_cpp(tmp) + script_dir = pathlib.Path(os.path.realpath(__file__)).parent + print(script_dir) + shutil.copy(script_dir / "CMakeLists.txt", tmp / "CMakeLists.txt") + shutil.copy(script_dir / "empty.cpp", tmp / "empty.cpp") tgt = tmp / "build" tgt.mkdir() - run(["cmake", f"-DLLVM_ROOT={llvm}", f"-DSWIFT_ROOT={swift}", "-DBUILD_SHARED_LIBS=OFF", ".."], + run(["cmake", f"-DCMAKE_PREFIX_PATH={llvm};{swift}", "-DBUILD_SHARED_LIBS=OFF", ".."], cwd=tgt) return tgt def get_libs(configured): print("extracting linking information from dummy project") - with open(configured / "CMakeFiles" / "dummy.dir" / "link.txt") as link: + with open(configured / "CMakeFiles" / "codeql-swift-artifacts.dir" / "link.txt") as link: libs = link.read().split() - libs = libs[libs.index('dummy')+1:] # skip up to -o dummy - ret = Libs([], [], []) + libs = libs[libs.index('codeql-swift-artifacts')+1:] # skip up to -o dummy + ret = Libs([], [], [], []) for l in libs: if l.endswith(".a"): ret.static.append(str((configured / l).resolve())) @@ -173,11 +84,10 @@ def get_libs(configured): ret.shared.append(f"-l{l[3:]}") # drop 'lib' prefix and '.so' suffix elif l.startswith("-l"): ret.shared.append(l) + elif l.startswith("-L") or l.startswith("-Wl"): + ret.linker_flags.append(l) else: raise ValueError(f"cannot understand link.txt: " + l) - # move direct dependencies into archive - ret.archive[:] = ret.static[:len(DEPLIST)] - ret.static[:len(DEPLIST)] = [] return ret @@ -195,7 +105,6 @@ def create_static_lib(tgt, libs): mriscript = f"create {tgt}\n{includedlibs}\nsave\nend" run(["ar", "-M"], cwd=tgt.parent, input=mriscript) else: - includedlibs = " ".join(f"{l}" for l in libs.archive + libs.static) libtool_args = ["libtool", "-static"] libtool_args.extend(libs.archive) libtool_args.extend(libs.static) @@ -214,6 +123,7 @@ def create_shared_lib(tgt, libs): print(f"packaging {libname}") compiler = os.environ.get("CC", "clang") cmd = [compiler, "-shared"] + cmd.extend(libs.linker_flags) if sys.platform == 'linux': cmd.append("-Wl,--whole-archive") @@ -239,7 +149,7 @@ def create_shared_lib(tgt, libs): def copy_includes(src, tgt): print("copying includes") for dir, exts in (("include", ("h", "def", "inc")), ("stdlib", ("h",))): - srcdir = src / "usr" / dir + srcdir = src / dir for ext in exts: for srcfile in srcdir.rglob(f"*.{ext}"): tgtfile = tgt / dir / srcfile.relative_to(srcdir) @@ -247,9 +157,9 @@ def copy_includes(src, tgt): shutil.copy(srcfile, tgtfile) -def create_sdk(installed, tgt): +def export_sdk(tgt, swift_source_tree, swift_build_tree): print("assembling sdk") - srcdir = installed / "usr" / "lib" / "swift" + srcdir = swift_build_tree/ "lib" / "swift" tgtdir = tgt / "usr" / "lib" / "swift" if get_platform() == "linux": srcdir /= "linux" @@ -258,24 +168,30 @@ def create_sdk(installed, tgt): srcdir /= "macosx" for mod in srcdir.glob("*.swiftmodule"): shutil.copytree(mod, tgtdir / mod.name) - shutil.copytree(installed / "usr" / "stdlib" / "public" / "SwiftShims", - tgt / "usr" / "include" / "SwiftShims") + shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims", + tgt / "usr" / "include" / "SwiftShims", + ignore=shutil.ignore_patterns('CMakeLists.txt')) -def create_export_dir(tmp, installed, libs): - print("assembling prebuilt directory") - exportedlibs = [create_static_lib(tmp, libs), create_shared_lib(tmp, libs)] - tgt = tmp / "exported" - tgt.mkdir() +def export_libs(exported_dir, libs): + print("exporting libraries") + exportedlibs = [ + create_static_lib(exported_dir, libs), + create_shared_lib(exported_dir, libs) + ] for l in exportedlibs: - l.rename(tgt / l.name) - with open(tgt / "swift_llvm_prebuilt.cmake", "w") as out: - # drop -l prefix here - sharedlibs = " ".join(l[2:] for l in libs.shared) - out.write(CMAKELISTS_EXPORTED_FMT.format(exported=EXPORTED_LIB, libs=sharedlibs)) - copy_includes(installed, tgt) - create_sdk(installed, tgt / "sdk") - return tgt + l.rename(exported_dir / l.name) + + +def export_headers(exported_dir, swift_source_tree, llvm_build_tree, swift_build_tree): + print("exporting headers") + # Assuming default checkout where LLVM sources are placed next to Swift sources + llvm_source_tree = swift_source_tree.parent / 'llvm-project/llvm' + clang_source_tree = swift_source_tree.parent / 'llvm-project/clang' + clang_tools_build_tree = llvm_build_tree / 'tools/clang' + header_dirs = [ llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, clang_tools_build_tree ] + for h in header_dirs: + copy_includes(h, exported_dir) def zip_dir(src, tgt): @@ -296,27 +212,17 @@ def main(opts): if os.path.exists(tmp): shutil.rmtree(tmp) os.mkdir(tmp) - if opts.update_shared or opts.update_static: - for project, deps in DEPS.items(): - build(getattr(opts, project), deps) - configured = configure_dummy_project(tmp, llvm=opts.llvm, swift=opts.swift) - libs = get_libs(configured) - if opts.update_shared: - create_shared_lib(opts.update_shared, libs) - if opts.update_static: - create_static_lib(opts.update_static, libs) - else: - installed = install(tmp, opts) - swift_syntax_build = opts.swift / "include/swift/Syntax/" - swift_syntax_install = installed / "usr/include/swift/Syntax/" - for header in os.listdir(swift_syntax_build): - if header.endswith('.h') or header.endswith('.def'): - shutil.copy(swift_syntax_build / header, swift_syntax_install / header) - configured = configure_dummy_project(tmp, installed=installed) - libs = get_libs(configured) - exported = create_export_dir(tmp, installed, libs) - zip_dir(exported, opts.output) - tar_dir(exported, opts.output) + configured = configure_dummy_project(tmp, llvm=opts.llvm_build_tree, swift=opts.swift_build_tree) + libs = get_libs(configured) + + exported = tmp / "exported" + exported.mkdir() + export_libs(exported, libs) + export_headers(exported, opts.swift_source_tree, opts.llvm_build_tree, opts.swift_build_tree) + export_sdk(exported / "sdk", opts.swift_source_tree, opts.swift_build_tree) + + zip_dir(exported, opts.output) + tar_dir(exported, opts.output) if __name__ == "__main__": diff --git a/swift-build-presets b/swift-build-presets index bca8930..d83f54f 100644 --- a/swift-build-presets +++ b/swift-build-presets @@ -1,5 +1,5 @@ [preset: codeql-baseline] -bootstrapping=off +bootstrapping=hosttools llvm-cmake-options=-DLLVM_ENABLE_TERMINFO=OFF -DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64 diff --git a/swift-build-system.patch b/swift-build-system.patch index 5d573fa..1bddbf8 100644 --- a/swift-build-system.patch +++ b/swift-build-system.patch @@ -1,75 +1,3 @@ -diff --git a/cmake/modules/CMakeLists.txt b/cmake/modules/CMakeLists.txt -index 8046ea79966..cd4fa6e673b 100644 ---- a/cmake/modules/CMakeLists.txt -+++ b/cmake/modules/CMakeLists.txt -@@ -17,3 +17,54 @@ configure_file( - SwiftConfig.cmake.in - ${swift_cmake_builddir}/SwiftConfig.cmake - @ONLY) -+ -+# Generate install-tree CMake files -+set(SWIFT_CONFIG_CODE " -+# Compute the installation prefix from this SwiftConfig.cmake file location. -+get_filename_component(SWIFT_INSTALL_PREFIX \"\${CMAKE_CURRENT_LIST_FILE}\" PATH)") -+# Construct the proper number of get_filename_component(... PATH) -+# calls to compute the installation prefix. -+string(REGEX REPLACE "/" ";" _count "${SWIFT_INSTALL_PACKAGE_DIR}") -+foreach(p ${_count}) -+ set(SWIFT_CONFIG_CODE "${SWIFT_CONFIG_CODE} -+get_filename_component(SWIFT_INSTALL_PREFIX \"\${SWIFT_INSTALL_PREFIX}\" PATH)") -+endforeach(p) -+set(SWIFT_INCLUDE_DIRS "\${SWIFT_INSTALL_PREFIX}/include") -+set(SWIFT_LIBRARY_DIRS "\${SWIFT_INSTALL_PREFIX}/lib") -+set(SWIFT_CMAKE_DIR "\${SWIFT_INSTALL_PREFIX}/${SWIFT_INSTALL_PACKAGE_DIR}") -+set(SWIFT_BINARY_DIR "\${SWIFT_INSTALL_PREFIX}") -+ -+set(SWIFT_EXPORTS_FILE "\${SWIFT_CMAKE_DIR}/SwiftExports.cmake") -+set(SWIFT_CONFIG_EXPORTS ${SWIFT_EXPORTS}) -+ -+install(TARGETS ${SWIFT_CONFIG_EXPORTS} -+ DESTINATION . -+ EXPORT SwiftExports -+ COMPONENT dev) -+ -+include(CMakePackageConfigHelpers) -+configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/SwiftConfig.cmake.in -+ "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/SwiftConfig.cmake" -+ INSTALL_DESTINATION ${SWIFT_INSTALL_PACKAGE_DIR}/cmake/swift) -+ -+install(FILES -+ ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/SwiftConfig.cmake -+ DESTINATION ${SWIFT_INSTALL_PACKAGE_DIR}) -+ -+install(EXPORT SwiftExports DESTINATION ${SWIFT_INSTALL_PACKAGE_DIR}) -+ -+# installing each header individually to preserve the directory structure -+macro(install_headers directory ext) -+ file(GLOB_RECURSE headers RELATIVE ${directory} ${directory}/*.${ext}) -+ foreach(header ${headers}) -+ get_filename_component(dir ${header} DIRECTORY) -+ install(FILES ${directory}/${header} DESTINATION ${SWIFT_INCLUDE_DIRS}/${dir}) -+ endforeach() -+endmacro() -+install_headers(${SWIFT_INCLUDE_DIR} h) -+install_headers(${SWIFT_MAIN_INCLUDE_DIR} h) -+install_headers(${SWIFT_INCLUDE_DIR} def) -+install_headers(${SWIFT_MAIN_INCLUDE_DIR} def) -+ -+install(DIRECTORY ${CMAKE_SOURCE_DIR}/stdlib/public/SwiftShims DESTINATION stdlib/public) -+ -diff --git a/cmake/modules/SwiftComponents.cmake b/cmake/modules/SwiftComponents.cmake -index 4d77e374deb..88b6645a811 100644 ---- a/cmake/modules/SwiftComponents.cmake -+++ b/cmake/modules/SwiftComponents.cmake -@@ -76,7 +76,7 @@ set(_SWIFT_DEFINED_COMPONENTS - # for the following exceptions. - set(_SWIFT_DEFAULT_COMPONENTS "${_SWIFT_DEFINED_COMPONENTS}") - # 'dev' takes up a lot of disk space and isn't part of a normal toolchain. --list(REMOVE_ITEM _SWIFT_DEFAULT_COMPONENTS "dev") -+#list(REMOVE_ITEM _SWIFT_DEFAULT_COMPONENTS "dev") - # These clang header options conflict with 'clang-builtin-headers'. - list(REMOVE_ITEM _SWIFT_DEFAULT_COMPONENTS "clang-resource-dir-symlink") - list(REMOVE_ITEM _SWIFT_DEFAULT_COMPONENTS "clang-builtin-headers-in-clang-resource-dir") diff --git a/include/swift/AST/Stmt.h b/include/swift/AST/Stmt.h index 532e038c386..f013eee2f98 100644 --- a/include/swift/AST/Stmt.h