diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db1b94ef5..4340fd1d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,11 +88,8 @@ jobs: - name: Update buck2-prelude submodule run: git submodule update --init --remote --no-fetch --depth 1 --single-branch tools/buck/prelude - run: buck2 run demo - continue-on-error: ${{matrix.os == 'windows'}} # FIXME: cl.exe not found - run: buck2 build ... - continue-on-error: ${{matrix.os == 'windows'}} - run: buck2 test ... - continue-on-error: ${{matrix.os == 'windows'}} reindeer: name: Reindeer diff --git a/tools/buck/toolchains/cxx.bzl b/tools/buck/toolchains/cxx.bzl index 737b3b220..e0eb348cf 100644 --- a/tools/buck/toolchains/cxx.bzl +++ b/tools/buck/toolchains/cxx.bzl @@ -13,6 +13,7 @@ load("@prelude//cxx:linker.bzl", "is_pdb_generated") load("@prelude//linking:link_info.bzl", "LinkStyle") load("@prelude//linking:lto.bzl", "LtoMode") load("@prelude//utils:cmd_script.bzl", "ScriptOs", "cmd_script") +load("@toolchains//msvc:tools.bzl", "VisualStudio") def _system_cxx_toolchain_impl(ctx): archiver_args = ["ar", "rcs"] @@ -34,9 +35,10 @@ def _system_cxx_toolchain_impl(ctx): linker_type = "darwin" pic_behavior = PicBehavior("always_enabled") elif host_info().os.is_windows: - archiver_args = ["lib.exe"] + msvc_tools = ctx.attrs.msvc_tools[VisualStudio] + archiver_args = [msvc_tools.lib_exe] archiver_type = "windows" - asm_compiler = "ml64.exe" + asm_compiler = msvc_tools.ml64_exe asm_compiler_type = "windows_ml64" compiler = _windows_compiler_wrapper(ctx) cxx_compiler = compiler @@ -47,9 +49,7 @@ def _system_cxx_toolchain_impl(ctx): static_library_extension = "lib" shared_library_name_format = "{}.dll" shared_library_versioned_name_format = "{}.dll" - additional_linker_flags = [ - "msvcrt.lib", - ] + additional_linker_flags = ["msvcrt.lib"] pic_behavior = PicBehavior("not_supported") elif ctx.attrs.linker == "g++" or ctx.attrs.cxx_compiler == "g++": pass @@ -148,17 +148,19 @@ def _windows_linker_wrapper(ctx: AnalysisContext) -> cmd_args: def _windows_compiler_wrapper(ctx: AnalysisContext) -> cmd_args: # The wrapper is needed to dynamically find compiler location and # Windows SDK to add necessary includes. - return cmd_script( - ctx = ctx, - name = "windows_compiler", - cmd = cmd_args( - ctx.attrs.windows_compiler_wrapper[RunInfo], - ctx.attrs.compiler, - ), - os = ScriptOs("windows"), - ) + if ctx.attrs.compiler == "cl.exe": + return cmd_script( + ctx = ctx, + name = "windows_compiler", + cmd = cmd_args( + ctx.attrs.windows_compiler_wrapper[RunInfo], + ctx.attrs.msvc_tools[VisualStudio].cl_exe, + ), + os = ScriptOs("windows"), + ) + else: + return cmd_args(ctx.attrs.compiler) -# Use clang, since thats available everywhere and what we have tested with. system_cxx_toolchain = rule( impl = _system_cxx_toolchain_impl, attrs = { @@ -173,6 +175,7 @@ system_cxx_toolchain = rule( "linker": attrs.string(default = "link.exe" if host_info().os.is_windows else "clang++"), "linker_wrapper": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//cxx/tools:linker_wrapper")), "make_comp_db": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//cxx/tools:make_comp_db")), + "msvc_tools": attrs.default_only(attrs.dep(providers = [VisualStudio], default = "toolchains//msvc:msvc_tools")), "windows_compiler_wrapper": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//cxx/tools:windows_compiler_wrapper")), }, is_toolchain_rule = True, diff --git a/tools/buck/toolchains/msvc/BUCK b/tools/buck/toolchains/msvc/BUCK new file mode 100644 index 000000000..ace68c3d9 --- /dev/null +++ b/tools/buck/toolchains/msvc/BUCK @@ -0,0 +1,18 @@ +load(":tools.bzl", "find_msvc_tools") + +python_bootstrap_binary( + name = "vswhere", + main = "vswhere.py", + visibility = [], +) + +python_bootstrap_binary( + name = "run_msvc_tool", + main = "run_msvc_tool.py", + visibility = [], +) + +find_msvc_tools( + name = "msvc_tools", + visibility = ["toolchains//..."], +) diff --git a/tools/buck/toolchains/msvc/run_msvc_tool.py b/tools/buck/toolchains/msvc/run_msvc_tool.py new file mode 100644 index 000000000..687a9d236 --- /dev/null +++ b/tools/buck/toolchains/msvc/run_msvc_tool.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess +import sys +from typing import List, NamedTuple + + +class Tool(NamedTuple): + exe: str + libs: List[str] + paths: List[str] + includes: List[str] + + +def add_env(env, key, entries): + entries = ";".join(entries) + if key in env: + env[key] = entries + ";" + env[key] + else: + env[key] = entries + + +def main(): + tool_json, arguments = sys.argv[1], sys.argv[2:] + with open(tool_json, encoding="utf-8") as f: + tool = Tool(**json.load(f)) + + env = os.environ.copy() + add_env(env, "LIB", tool.libs) + add_env(env, "PATH", tool.paths) + add_env(env, "INCLUDE", tool.includes) + + completed_process = subprocess.run([tool.exe, *arguments], env=env) + sys.exit(completed_process.returncode) + + +if __name__ == "__main__": + main() diff --git a/tools/buck/toolchains/msvc/tools.bzl b/tools/buck/toolchains/msvc/tools.bzl new file mode 100644 index 000000000..db7465bd0 --- /dev/null +++ b/tools/buck/toolchains/msvc/tools.bzl @@ -0,0 +1,65 @@ +load("@prelude//utils:cmd_script.bzl", "ScriptOs", "cmd_script") + +VisualStudio = provider(fields = [ + # Path to cl.exe + "cl_exe", + # Path to lib.exe + "lib_exe", + # Path to ml64.exe + "ml64_exe", +]) + +def _find_msvc_tools_impl(ctx: AnalysisContext) -> ["provider"]: + cl_exe_json = ctx.actions.declare_output("cl.exe.json") + lib_exe_json = ctx.actions.declare_output("lib.exe.json") + ml64_exe_json = ctx.actions.declare_output("ml64.exe.json") + + cmd = [ + ctx.attrs.vswhere[RunInfo], + cmd_args("--cl=", cl_exe_json.as_output(), delimiter = ""), + cmd_args("--lib=", lib_exe_json.as_output(), delimiter = ""), + cmd_args("--ml64=", ml64_exe_json.as_output(), delimiter = ""), + ] + + ctx.actions.run( + cmd, + category = "vswhere", + local_only = True, + ) + + run_msvc_tool = ctx.attrs.run_msvc_tool[RunInfo] + cl_exe_script = cmd_script( + ctx = ctx, + name = "cl", + cmd = cmd_args(run_msvc_tool, cl_exe_json), + os = ScriptOs("windows"), + ) + lib_exe_script = cmd_script( + ctx = ctx, + name = "lib", + cmd = cmd_args(run_msvc_tool, lib_exe_json), + os = ScriptOs("windows"), + ) + ml64_exe_script = cmd_script( + ctx = ctx, + name = "ml64", + cmd = cmd_args(run_msvc_tool, ml64_exe_json), + os = ScriptOs("windows"), + ) + + return [ + DefaultInfo(), + VisualStudio( + cl_exe = cl_exe_script, + lib_exe = lib_exe_script, + ml64_exe = ml64_exe_script, + ), + ] + +find_msvc_tools = rule( + impl = _find_msvc_tools_impl, + attrs = { + "run_msvc_tool": attrs.default_only(attrs.dep(providers = [RunInfo], default = "toolchains//msvc:run_msvc_tool")), + "vswhere": attrs.default_only(attrs.dep(providers = [RunInfo], default = "toolchains//msvc:vswhere")), + }, +) diff --git a/tools/buck/toolchains/msvc/vswhere.py b/tools/buck/toolchains/msvc/vswhere.py new file mode 100644 index 000000000..1ad10d0a7 --- /dev/null +++ b/tools/buck/toolchains/msvc/vswhere.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 + +# Translated from the Rust `cc` crate's windows_registry.rs. +# https://github.com/rust-lang/cc-rs/blob/1.0.79/src/windows_registry.rs + +import argparse +import json +import os +import shutil +import subprocess +import sys +import winreg +from pathlib import Path +from typing import IO, List, NamedTuple + + +class OutputJsonFiles(NamedTuple): + cl: IO[str] + lib: IO[str] + ml64: IO[str] + + +class Tool(NamedTuple): + exe: Path + libs: List[Path] = [] + paths: List[Path] = [] + includes: List[Path] = [] + + +def find_in_path(executable): + which = shutil.which(executable) + if which is None: + print(f"{executable} not found in $PATH", file=sys.stderr) + sys.exit(1) + return Tool(which) + + +def find_with_vswhere_exe(): + program_files = os.environ.get("ProgramFiles(x86)") + if program_files is None: + program_files = os.environ.get("ProgramFiles") + if program_files is None: + print("expected a %ProgramFiles(x86)% or %ProgramFiles% environment variable", file=sys.stderr) + sys.exit(1) + + vswhere_exe = Path(program_files) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + vswhere_json = subprocess.check_output( + [ + vswhere_exe, + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-format", + "json", + "-nologo", + ], + encoding="utf-8", + ) + + vswhere_json = json.loads(vswhere_json) + + # Sort by MSVC version, newest to oldest. + # Version is a sequence of 16-bit integers (at most 4 of them?). + # Example: "17.6.33829.357" + vswhere_json.sort( + key=lambda vs: [int(n) for n in vs["installationVersion"].split(".")], + reverse=True, + ) + + for vs_instance in list(vswhere_json): + installation_path = Path(vs_instance["installationPath"]) + + # Tools version is different from the one above: "14.36.32532" + version_file = installation_path / "VC" / "Auxiliary" / "Build" / "Microsoft.VCToolsVersion.default.txt" + vc_tools_version = version_file.read_text(encoding="utf-8").strip() + + tools_path = installation_path / "VC" / "Tools" / "MSVC" / vc_tools_version + bin_path = tools_path / "bin" / "HostX64" / "x64" + lib_path = tools_path / "lib" / "x64" + include_path = tools_path / "include" + + exe_names = "cl.exe", "lib.exe", "ml64.exe" + tools = [Tool(bin_path / exe) for exe in exe_names] + if not all(tool.exe.exists() for tool in tools): + continue + + add_to_paths = [bin_path] + add_to_libs = [lib_path] + add_to_includes = [include_path] + + ucrt, ucrt_version = get_ucrt_dir() + if ucrt and ucrt_version: + add_to_paths.append(ucrt / "bin" / ucrt_version / "x64") + add_to_libs.append(ucrt / "lib" / ucrt_version / "ucrt" / "x64") + add_to_includes.append(ucrt / "include" / ucrt_version / "ucrt") + + sdk, sdk_version = get_sdk10_dir() + if sdk and sdk_version: + add_to_paths.append(sdk / "bin" / "x64") + add_to_libs.append(sdk / "lib" / sdk_version / "um" / "x64") + add_to_includes.append(sdk / "include" / sdk_version / "um") + add_to_includes.append(sdk / "include" / sdk_version / "cppwinrt") + add_to_includes.append(sdk / "include" / sdk_version / "winrt") + add_to_includes.append(sdk / "include" / sdk_version / "shared") + + for tool in tools: + tool.paths.extend(add_to_paths) + tool.libs.extend(add_to_libs) + tool.includes.extend(add_to_includes) + + return tools + + print("vswhere.exe did not find a suitable MSVC toolchain containing cl.exe, lib.exe, ml64.exe", file=sys.stderr) + sys.exit(1) + + +# To find the Universal CRT we look in a specific registry key for where all the +# Universal CRTs are located and then sort them asciibetically to find the +# newest version. While this sort of sorting isn't ideal, it is what vcvars does +# so that's good enough for us. +# +# Returns a pair of (root, version) for the ucrt dir if found. +def get_ucrt_dir(): + registry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + key_name = "SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots" + registry_key = winreg.OpenKey(registry, key_name) + kits_root = Path(winreg.QueryValueEx(registry_key, "KitsRoot10")[0]) + + available_versions = [ + entry.name + for entry in kits_root.joinpath("lib").iterdir() + if entry.name.startswith("10.") and entry.joinpath("ucrt").is_dir() + ] + + max_version = max(available_versions) if available_versions else None + return kits_root, max_version + + +# Vcvars finds the correct version of the Windows 10 SDK by looking for the +# include `um\Windows.h` because sometimes a given version will only have UCRT +# bits without the rest of the SDK. Since we only care about libraries and not +# includes, we instead look for `um\x64\kernel32.lib`. Since the 32-bit and +# 64-bit libraries are always installed together we only need to bother checking +# x64, making this code a tiny bit simpler. Like we do for the Universal CRT, we +# sort the possibilities asciibetically to find the newest one as that is what +# vcvars does. Before doing that, we check the "WindowsSdkDir" and +# "WindowsSDKVersion" environment variables set by vcvars to use the environment +# sdk version if one is already configured. +# +# Returns a pair of (root, version). +def get_sdk10_dir(): + windows_sdk_dir = os.environ.get("WindowsSdkDir") + windows_sdk_version = os.environ.get("WindowsSDKVersion") + if windows_sdk_dir is not None and windows_sdk_version is not None: + return windows_sdk_dir, windows_sdk_version.removesuffix("\\") + + registry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + key_name = "SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v10.0" + registry_key = winreg.OpenKey(registry, key_name, access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY) + installation_folder = Path(winreg.QueryValueEx(registry_key, "InstallationFolder")[0]) + + available_versions = [ + entry.name + for entry in installation_folder.joinpath("lib").iterdir() + if entry.joinpath("um", "x64", "kernel32.lib").is_file() + ] + + max_version = max(available_versions) if available_versions else None + return installation_folder, max_version + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cl", type=argparse.FileType("w"), required=True) + parser.add_argument("--lib", type=argparse.FileType("w"), required=True) + parser.add_argument("--ml64", type=argparse.FileType("w"), required=True) + output = OutputJsonFiles(**vars(parser.parse_args())) + + # If vcvars has been run, it puts these tools onto $PATH. + if "VCINSTALLDIR" in os.environ: + cl_exe = find_in_path("cl.exe") + lib_exe = find_in_path("lib.exe") + ml64_exe = find_in_path("ml64.exe") + else: + cl_exe, lib_exe, ml64_exe = find_with_vswhere_exe() + + to_json = lambda tool: json.dumps( + tool._asdict(), + indent=4, + default=lambda path: str(path), + ) + + output.cl.write(to_json(cl_exe)) + output.lib.write(to_json(lib_exe)) + output.ml64.write(to_json(ml64_exe)) + + +if __name__ == "__main__": + main()