Prepare update to 5.8

CMakeLists.txt

@@ -4,6 +4,7 @@ project(codeql-swift-artifacts C CXX)
 find_package(LLVM REQUIRED CONFIG)
 find_package(Clang REQUIRED CONFIG)
 find_package(Swift REQUIRED CONFIG)
+find_package(SwiftSyntax REQUIRED CONFIG)
 
 message("Using LLVM_CONFIG: ${Swift_CONFIG}")
 message("Using Clang_CONFIG: ${LLVM_CONFIG}")
@@ -31,4 +32,3 @@ if(APPLE)
   target_link_directories(codeql-swift-artifacts PUBLIC ${TOOLCHAIN_LIB_DIR})
   target_link_directories(codeql-swift-artifacts PUBLIC ${CODEQL_MACOS_SDK_PATH}/usr/lib/swift)
 endif()
-

patches/swift/fix-linux-compilation-with-swift-parser.patch

@@ -0,0 +1,52 @@
+diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake
+index 58c7eb1bd4f..4f4d5b7edea 100644
+--- a/cmake/modules/AddSwift.cmake
++++ b/cmake/modules/AddSwift.cmake
+@@ -912,17 +912,22 @@ function(add_swift_host_tool executable)
+   endif()
+
+   if(SWIFT_SWIFT_PARSER)
+-    set(extra_relative_rpath "")
+-    if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "")
+-      if (${executable} MATCHES "-bootstrapping")
+-        set(extra_relative_rpath "../")
++    if (SWIFT_HOST_VARIANT_SDK STREQUAL LINUX)
++      if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "" AND ${executable} MATCHES "-bootstrapping")
++        set_property(TARGET ${executable} APPEND PROPERTY INSTALL_RPATH "$ORIGIN/../../lib/swift/host")
++      else()
++        set_property(TARGET ${executable} APPEND PROPERTY INSTALL_RPATH "$ORIGIN/../lib/swift/host")
++      endif()
++    else()
++      set(extra_relative_rpath "")
++      if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "" AND ${executable} MATCHES "-bootstrapping")
++          set(extra_relative_rpath "../")
+       endif()
++      set_property(
++        TARGET ${executable}
++        APPEND PROPERTY INSTALL_RPATH
++	  "@executable_path/../${extra_relative_rpath}lib/swift/host")
+     endif()
+-
+-    set_property(
+-      TARGET ${executable}
+-      APPEND PROPERTY INSTALL_RPATH
+-        "@executable_path/../${extra_relative_rpath}lib/swift/host")
+   endif()
+
+   if(ASHT_THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY)
+diff --git a/lib/ASTGen/CMakeLists.txt b/lib/ASTGen/CMakeLists.txt
+index c7334a6da0b..2a0cb80b743 100644
+--- a/lib/ASTGen/CMakeLists.txt
++++ b/lib/ASTGen/CMakeLists.txt
+@@ -1,4 +1,11 @@
+ if (SWIFT_SWIFT_PARSER)
++  if (SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
++    # including libdispatch leads to multiple module definition problems, as it seems
++    # it is already available to the swift toolchain
++    get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
++    list(REMOVE_ITEM dirs "${SWIFT_PATH_TO_LIBDISPATCH_SOURCE}")
++    set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "${dirs}")
++  endif()
+   # Ensure that we do not link the _StringProcessing module. But we can
+   # only pass this flag for new-enough compilers that support it.
+   file(WRITE "${CMAKE_BINARY_DIR}/tmp/empty-check-string-processing.swift" "")

pkg_swift_llvm.py

@@ -14,10 +14,8 @@
 
 def getoptions():
     parser = argparse.ArgumentParser(description="package swift for codeql compilation")
-    parser.add_argument(f"--llvm-build-tree", required=True, type=resolve,
-                        metavar="DIR", help=f"path to LLVM build tree")
-    parser.add_argument(f"--swift-build-tree", required=True, type=resolve,
-                        metavar="DIR", help=f"path to Swift build tree")
+    parser.add_argument(f"--build-tree", required=True, type=resolve,
+                        metavar="DIR", help=f"path to the build tree")
     parser.add_argument(f"--swift-source-tree", required=True, type=resolve,
                         metavar="DIR", help=f"path to Swift source tree")
 
@@ -34,57 +32,54 @@ def getoptions():
     return opts
 
 
-Libs = namedtuple("Libs", ("archive", "static", "shared", "linker_flags"))
-
 EXPORTED_LIB = "CodeQLSwiftFrontendTool"
 
+Libs = namedtuple("Libs", ("static", "shared", "linker_flags"))
+
 
 def resolve(p):
     return pathlib.Path(p).resolve()
 
 
 def run(prog, *, cwd, env=None, input=None):
-    print("running", " ".join(prog), f"(cwd={cwd})")
+    print("running", *prog, f"(cwd={cwd})")
     if env is not None:
         runenv = dict(os.environ)
         runenv.update(env)
     else:
         runenv = None
-    subprocess.run(prog, cwd=cwd, env=runenv, input=input, text=True)
+    subprocess.run(prog, cwd=cwd, env=runenv, input=input, text=True, check=True)
 
 
 def get_platform():
     return "linux" if platform.system() == "Linux" else "macos"
 
 
-def configure_dummy_project(tmp, *, llvm=None, swift=None):
+def configure_dummy_project(tmp, prefixes):
     print("configuring dummy cmake project")
     script_dir = pathlib.Path(os.path.realpath(__file__)).parent
     print(script_dir)
     shutil.copy(script_dir / "CMakeLists.txt", tmp / "CMakeLists.txt")
     shutil.copy(script_dir / "empty.cpp", tmp / "empty.cpp")
     tgt = tmp / "build"
     tgt.mkdir()
-    run(["cmake", f"-DCMAKE_PREFIX_PATH={llvm};{swift}", "-DBUILD_SHARED_LIBS=OFF", ".."],
-        cwd=tgt)
+    prefixes = ';'.join(str(p) for p in prefixes)
+    run(["cmake", f"-DCMAKE_PREFIX_PATH={prefixes}", "-DBUILD_SHARED_LIBS=OFF", ".."], cwd=tgt)
     return tgt
 
 
 def get_libs(configured):
     print("extracting linking information from dummy project")
     with open(configured / "CMakeFiles" / "codeql-swift-artifacts.dir" / "link.txt") as link:
         libs = link.read().split()
-        libs = libs[libs.index('codeql-swift-artifacts')+1:] # skip up to -o dummy
-    ret = Libs([], [], [], [])
+        libs = libs[libs.index('codeql-swift-artifacts') + 1:]  # skip up to -o dummy
+    ret = Libs([], [], [])
     for l in libs:
         if l.endswith(".a"):
-            ret.static.append(str((configured / l).resolve()))
+            ret.static.append((configured / l).absolute())
         elif l.endswith(".so") or l.endswith(".tbd") or l.endswith(".dylib"):
-            l = pathlib.Path(l).stem
-            ret.shared.append(f"-l{l[3:]}")  # drop 'lib' prefix and '.so' suffix
-        elif l.startswith("-l"):
-            ret.shared.append(l)
-        elif l.startswith("-L") or l.startswith("-Wl"):
+            ret.shared.append((configured / l).absolute())
+        elif l.startswith(("-L", "-Wl", "-l")):
             ret.linker_flags.append(l)
         else:
             raise ValueError(f"cannot understand link.txt: " + l)
@@ -94,60 +89,27 @@ def get_libs(configured):
 def get_tgt(tgt, filename):
     if tgt.is_dir():
         tgt /= filename
-    return tgt.resolve()
+    return tgt.absolute()
 
 
 def create_static_lib(tgt, libs):
     tgt = get_tgt(tgt, f"lib{EXPORTED_LIB}.a")
     print(f"packaging {tgt.name}")
     if sys.platform == 'linux':
-        includedlibs = "\n".join(f"addlib {l}" for l in libs.archive + libs.static)
+        includedlibs = "\n".join(f"addlib {l}" for l in libs.static)
         mriscript = f"create {tgt}\n{includedlibs}\nsave\nend"
         run(["ar", "-M"], cwd=tgt.parent, input=mriscript)
     else:
         libtool_args = ["libtool", "-static"]
-        libtool_args.extend(libs.archive)
         libtool_args.extend(libs.static)
         libtool_args.append("-o")
         libtool_args.append(str(tgt))
         run(libtool_args, cwd=tgt.parent)
     return tgt
 
 
-def create_shared_lib(tgt, libs):
-    ext = "so"
-    if sys.platform != 'linux':
-        ext = "dylib"
-    libname = f"lib{EXPORTED_LIB}.{ext}"
-    tgt = get_tgt(tgt, libname)
-    print(f"packaging {libname}")
-    compiler = os.environ.get("CC", "clang")
-    cmd = [compiler, "-shared"]
-    cmd.extend(libs.linker_flags)
-
-    if sys.platform == 'linux':
-        cmd.append("-Wl,--whole-archive")
-    else:
-        cmd.append("-Wl,-all_load")
-
-    cmd.append(f"-o{tgt}")
-    cmd.extend(libs.archive)
-
-    if sys.platform == 'linux':
-        cmd.append("-Wl,--no-whole-archive")
-    else:
-        cmd.append("-lc++")
-
-    cmd.extend(libs.static)
-    cmd.extend(libs.shared)
-    run(cmd, cwd=tgt.parent)
-    if sys.platform != "linux":
-        run(["install_name_tool", "-id", f"@executable_path/{libname}", libname], cwd=tgt.parent)
-    return tgt
-
-
 def copy_includes(src, tgt):
-    print("copying includes")
+    print(f"copying includes from {src}")
     for dir, exts in (("include", ("h", "def", "inc")), ("stdlib", ("h",))):
         srcdir = src / dir
         for ext in exts:
@@ -159,7 +121,7 @@ def copy_includes(src, tgt):
 
 def export_sdk(tgt, swift_source_tree, swift_build_tree):
     print("assembling sdk")
-    srcdir = swift_build_tree/ "lib" / "swift"
+    srcdir = swift_build_tree / "lib" / "swift"
     tgtdir = tgt / "usr" / "lib" / "swift"
     if get_platform() == "linux":
         srcdir /= "linux"
@@ -168,8 +130,8 @@ def export_sdk(tgt, swift_source_tree, swift_build_tree):
         srcdir /= "macosx"
     for mod in srcdir.glob("*.swiftmodule"):
         shutil.copytree(mod, tgtdir / mod.name)
-    shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims",
-                    tgt / "usr" / "include" / "SwiftShims",
+    shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims" / "swift" / "shims",
+                    tgt / "usr" / "lib" / "swift" / "shims",
                     ignore=shutil.ignore_patterns('CMakeLists.txt'))
 
 
@@ -181,11 +143,11 @@ def export_stdlibs(exported_dir, swift_build_tree):
         ext = 'so'
     lib_dir = swift_build_tree / 'lib/swift' / platform
     stdlibs = [
-            f'libswiftCore.{ext}',
-            'libswiftCompatibility50.a',
-            'libswiftCompatibility51.a',
-            'libswiftCompatibilityConcurrency.a',
-            'libswiftCompatibilityDynamicReplacements.a']
+        f'libswiftCore.{ext}',
+        'libswiftCompatibility50.a',
+        'libswiftCompatibility51.a',
+        'libswiftCompatibilityConcurrency.a',
+        'libswiftCompatibilityDynamicReplacements.a']
     for stdlib in stdlibs:
         lib_path = lib_dir / stdlib
         if lib_path.exists():
@@ -197,13 +159,11 @@ def export_stdlibs(exported_dir, swift_build_tree):
 
 def export_libs(exported_dir, libs, swift_build_tree):
     print("exporting libraries")
-    exportedlibs = [
-            create_static_lib(exported_dir, libs),
-            create_shared_lib(exported_dir, libs)
-            ]
-
-    for l in exportedlibs:
-        l.rename(exported_dir / l.name)
+    create_static_lib(exported_dir, libs)
+    for lib in libs.shared:
+        # export libraries under the build tree (e.g. libSwiftSyntax.so)
+        if lib.is_relative_to(swift_build_tree.parent):
+            shutil.copy(lib, exported_dir)
     export_stdlibs(exported_dir, swift_build_tree)
 
 
@@ -213,7 +173,8 @@ def export_headers(exported_dir, swift_source_tree, llvm_build_tree, swift_build
     llvm_source_tree = swift_source_tree.parent / 'llvm-project/llvm'
     clang_source_tree = swift_source_tree.parent / 'llvm-project/clang'
     clang_tools_build_tree = llvm_build_tree / 'tools/clang'
-    header_dirs = [ llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, clang_tools_build_tree ]
+    header_dirs = [llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree,
+                   clang_tools_build_tree]
     for h in header_dirs:
         copy_includes(h, exported_dir)
 
@@ -230,18 +191,21 @@ def main(opts):
     if os.path.exists(tmp):
         shutil.rmtree(tmp)
     os.mkdir(tmp)
-    configured = configure_dummy_project(tmp, llvm=opts.llvm_build_tree, swift=opts.swift_build_tree)
+    llvm_build_tree = next(opts.build_tree.glob("llvm-*"))
+    swift_build_tree = next(opts.build_tree.glob("swift-*"))
+    earlyswiftsyntax_build_tree = next(opts.build_tree.glob("earlyswiftsyntax-*"))
+    configured = configure_dummy_project(tmp, prefixes=[llvm_build_tree, swift_build_tree,
+                                                        earlyswiftsyntax_build_tree / "cmake" / "modules"])
     libs = get_libs(configured)
 
     exported = tmp / "exported"
     exported.mkdir()
-    export_libs(exported, libs, opts.swift_build_tree)
-    export_headers(exported, opts.swift_source_tree, opts.llvm_build_tree, opts.swift_build_tree)
-    export_sdk(exported / "sdk", opts.swift_source_tree, opts.swift_build_tree)
+    export_libs(exported, libs, swift_build_tree)
+    export_headers(exported, opts.swift_source_tree, llvm_build_tree, swift_build_tree)
+    export_sdk(exported / "sdk", opts.swift_source_tree, swift_build_tree)
 
     zip_dir(exported, opts.output)
 
 
 if __name__ == "__main__":
     main(getoptions())
-

swift-build-presets

@@ -1,6 +1,4 @@
 [preset: codeql-baseline]
-bootstrapping=hosttools
-
 llvm-cmake-options=-DLLVM_ENABLE_TERMINFO=OFF -DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64
 
 skip-ios
@@ -28,13 +26,25 @@ skip-build-android
 enable-experimental-string-processing
 swift-enable-experimental-string-processing=1
 
-[preset: codeql]
+no-assertions
+
+reconfigure
+
+[preset: codeql-release]
 mixin-preset=codeql-baseline
 release
 build-subdir=codeql
 
+[preset: codeql-Linux]
+mixin-preset=codeql-release
+bootstrapping=hosttools
+
+[preset: codeql-macOS]
+mixin-preset=codeql-release
+bootstrapping=bootstrapping
+
 [preset: codeql-debug]
 mixin-preset=codeql-baseline
 debug
 build-subdir=codeql-debug
-
+bootstrapping=bootstrapping