diff --git a/tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp b/tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp new file mode 100644 index 00000000000..0a8de629111 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp @@ -0,0 +1,12 @@ +module; +#include + +export module hello; + +import "../header.hpp"; + +export namespace hello { + void say(const char *arg) { + printf("%s: %s\n", FOO, arg); + } +} diff --git a/tests/projects/c++/modules/aliased_headerunit/src/header.hpp b/tests/projects/c++/modules/aliased_headerunit/src/header.hpp new file mode 100644 index 00000000000..ab02a679268 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/src/header.hpp @@ -0,0 +1,5 @@ +#pragma once + +namespace hello { + inline constexpr auto FOO = "Hello"; +} diff --git a/tests/projects/c++/modules/aliased_headerunit/src/main.cpp b/tests/projects/c++/modules/aliased_headerunit/src/main.cpp new file mode 100644 index 00000000000..ad786367eee --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/src/main.cpp @@ -0,0 +1,7 @@ +import hello; +import "header.hpp"; + +int main() { + hello::say(hello::FOO); + return 0; +} diff --git a/tests/projects/c++/modules/aliased_headerunit/test.lua b/tests/projects/c++/modules/aliased_headerunit/test.lua new file mode 100644 index 00000000000..c18e5a1d086 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/test.lua @@ -0,0 +1 @@ +inherit(".test_headerunits") diff --git a/tests/projects/c++/modules/aliased_headerunit/xmake.lua b/tests/projects/c++/modules/aliased_headerunit/xmake.lua new file mode 100644 index 00000000000..5b4827d09c8 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/xmake.lua @@ -0,0 +1,8 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +-- header.hpp should be built only one time +target("aliased_headerunit") + set_kind("binary") + add_headerfiles("src/*.hpp") + add_files("src/*.cpp", "src/foo/*.mpp") diff --git a/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua b/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua index 6a9b3538f4a..206e3746fba 100644 --- a/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua +++ b/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua @@ -3,7 +3,8 @@ set_languages("c++20") target("mod") set_kind("static") - add_files("src/mod.mpp", "src/mod.cpp") + add_files("src/mod.mpp", {public = true}) + add_files("src/mod.cpp") target("cpp_with_moduledeps") set_kind("binary") diff --git a/tests/projects/c++/modules/inline_and_template/src/main.cpp b/tests/projects/c++/modules/inline_and_template/src/main.cpp index a34667fbe2f..bf68f307d41 100644 --- a/tests/projects/c++/modules/inline_and_template/src/main.cpp +++ b/tests/projects/c++/modules/inline_and_template/src/main.cpp @@ -1,12 +1,12 @@ +#include + import hello; import say; import foo; -#include - int main() { hello::say_hello(); say{}.hello(); std::cout << foo{}.hello() << std::endl; return 0; -} \ No newline at end of file +} diff --git a/tests/projects/c++/modules/link_order/xmake.lua b/tests/projects/c++/modules/link_order/xmake.lua index 6a11418ac07..b062dbe66f4 100644 --- a/tests/projects/c++/modules/link_order/xmake.lua +++ b/tests/projects/c++/modules/link_order/xmake.lua @@ -4,12 +4,12 @@ set_languages("c++20") target("foo") add_rules("c++") set_kind("static") - add_files("src/foo.mpp") + add_files("src/foo.mpp", {public = true}) target("bar") add_rules("c++") set_kind("static") - add_files("src/bar.mpp") + add_files("src/bar.mpp", {public = true}) target("link_order_1") set_kind("binary") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp index 5bbb5e5c12a..279541ed020 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp @@ -1,5 +1,5 @@ -module bar; +#include -const char *bar() { +const char *hello() { return "Hello world"; -} \ No newline at end of file +} diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp index a13487feac8..48ed6b73b6a 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp @@ -1,3 +1,12 @@ +module; + +#include + export module bar; -export const char *bar(); \ No newline at end of file +export namespace bar { +const char *hello() { + return ::hello(); +} +} + diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp new file mode 100644 index 00000000000..7cd8caadb42 --- /dev/null +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp @@ -0,0 +1,6 @@ +#ifndef A_HPP +#define A_HPP + +[[gnu::visibility("default")]] const char *hello(); + +#endif diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua index 345dd2db4d5..d11bc4f6031 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua @@ -3,5 +3,7 @@ set_languages("c++20") target("bar") set_kind("static") + add_headerfiles("include/(**.hpp)") + add_includedirs("include") add_files("*.cpp") - add_files("*.mpp", { install = true }) + add_files("*.mpp", { public = true }) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua index 30116680705..1592fc623f4 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua @@ -3,4 +3,4 @@ package("bar") on_install(function(package) import("package.tools.xmake").install(package, {}) - end) \ No newline at end of file + end) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp index 4cab3d977c3..a97f3d62be9 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp @@ -1,5 +1,7 @@ export module foo; export namespace foo { + #ifdef FOO_EXPORT void say(const char *); -} \ No newline at end of file + #endif +} diff --git a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua index d79b2ac54a9..9fa37e1d798 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua @@ -4,4 +4,4 @@ set_languages("c++20") target("foo") set_kind("static") add_files("*.cpp") - add_files("*.mpp", { install = true }) + add_files("*.mpp", {defines = "FOO_EXPORT", public = true}) diff --git a/tests/projects/c++/modules/packages/src/main.cpp b/tests/projects/c++/modules/packages/src/main.cpp index 10c501c4ad5..d12361c453a 100644 --- a/tests/projects/c++/modules/packages/src/main.cpp +++ b/tests/projects/c++/modules/packages/src/main.cpp @@ -2,6 +2,6 @@ import foo; import bar; int main() { - foo::say(bar()); + foo::say(bar::hello()); return 0; } diff --git a/tests/projects/c++/modules/packages/xmake.lua b/tests/projects/c++/modules/packages/xmake.lua index fe3cc7dc596..f772d666850 100644 --- a/tests/projects/c++/modules/packages/xmake.lua +++ b/tests/projects/c++/modules/packages/xmake.lua @@ -1,5 +1,5 @@ add_rules("mode.release", "mode.debug") -set_languages("c++20") +set_languages("c++2b") add_repositories("my-repo my-repo") add_requires("foo", "bar") diff --git a/tests/projects/c++/modules/staticlib/xmake.lua b/tests/projects/c++/modules/staticlib/xmake.lua index ca0e9e34339..d353774e455 100644 --- a/tests/projects/c++/modules/staticlib/xmake.lua +++ b/tests/projects/c++/modules/staticlib/xmake.lua @@ -3,7 +3,7 @@ set_languages("c++20") target("mod") set_kind("static") - add_files("src/mod.mpp", "src/mod.cpp") + add_files("src/mod.mpp", "src/mod.cpp", {public = true}) target("hello") set_kind("binary") diff --git a/tests/projects/c++/modules/stdmodules/xmake.lua b/tests/projects/c++/modules/stdmodules/xmake.lua index 0603a4c519f..02447729c3f 100644 --- a/tests/projects/c++/modules/stdmodules/xmake.lua +++ b/tests/projects/c++/modules/stdmodules/xmake.lua @@ -3,11 +3,10 @@ set_languages("c++latest") target("mod") set_kind("static") - add_files("src/*.cpp", "src/*.mpp") - set_policy("build.c++.clang.stdmodules", true) + add_files("src/*.cpp") + add_files("src/*.mpp", {public = true}) target("stdmodules") set_kind("binary") add_files("test/*.cpp") add_deps("mod") - set_policy("build.c++.clang.stdmodules", true) diff --git a/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua b/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua index d7acc0d85e6..6f86e1efc07 100644 --- a/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua +++ b/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua @@ -6,5 +6,4 @@ target("stdmodules_cpp_only") set_kind("binary") add_files("src/*.cpp") set_policy("build.c++.modules", true) - set_policy("build.c++.clang.stdmodules", true) diff --git a/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua b/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua index 32bed613bd5..61d45190850 100644 --- a/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua +++ b/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua @@ -5,10 +5,6 @@ target("mod") set_kind("static") add_files("src/*.cpp", "src/*.mpp") - set_policy("build.c++.clang.stdmodules", true) - target("mod2") set_kind("static") add_files("src/*.cpp", "src/*.mpp") - - set_policy("build.c++.clang.stdmodules", true) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index efef155203c..c4252d41cd5 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -12,21 +12,34 @@ end function main(t) if is_subhost("windows") then - os.exec("xmake f -c") + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + os.exec("xmake f --toolchain=clang -c --yes") + _build() + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end + + os.exec("xmake clean -a") + os.exec("xmake f -c --yes") + _build() + elseif is_subhost("msys") then + os.exec("xmake f -c -p mingw --yes") _build() elseif is_host("linux") then local gcc = find_tool("gcc", {version = true}) - if is_host("linux") and gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - os.exec("xmake f -c") + if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then + os.exec("xmake f -c --yes") _build() end local clang = find_tool("clang", {version = true}) if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang -c") + os.exec("xmake f --toolchain=clang -c --yes") _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") _build() end end diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua index 3b664c96c1d..ba7fde01535 100644 --- a/tests/projects/c++/modules/test_headerunits.lua +++ b/tests/projects/c++/modules/test_headerunits.lua @@ -13,15 +13,33 @@ end function main(t) if is_subhost("windows") then + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "15.0") >= 0 then + -- clang headerunit are bugged + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c --yes") + -- _build() + -- if semver.compare(clang.version, "17.0") >= 0 then + -- os.exec("xmake clean -a") + -- -- clang-scan-deps dependency detection doesn't support header units atm + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --runtimes=c++_shared -c --yes") + -- _build() + -- end + end + local vs = find_vstudio() if vs and vs["2022"] then + os.exec("xmake clean -a") os.exec("xmake f -c --yes") _build() end + elseif is_subhost("msys") then + -- on windows, mingw modulemapper doesn't handle headeunit path correctly, but it's working with mingw on macOS / Linux + -- os.exec("xmake f -c -p mingw --yes") + -- _build() elseif is_host("linux") then local gcc = find_tool("gcc", {version = true}) if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - -- gcc trtbd dependency detection doesn't support header units atm + -- gcc dependency detection doesn't support header units atm os.exec("xmake f --policies=build.c++.gcc.fallbackscanner -c --yes") _build() end @@ -30,15 +48,16 @@ function main(t) if semver.compare(clang.version, "15.0") >= 0 then os.exec("xmake clean -a") -- clang-scan-deps dependency detection doesn't support header units atm - os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c") + os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c --yes") _build() - -- elseif semver.compare(clang.version, "15.0") >= 0 then - -- there is currently a bug on llvm git that prevent to build STL header units https://github.com/llvm/llvm-project/issues/58540 - -- os.exec("xmake clean -a") - -- clang-scan-deps dependency detection doesn't support header units atm - -- os.exec("xmake f --toolchain=clang --policies=build.c++.modules.fallbackscanner.clang --cxxflags=\"-stdlib=libc++\" -c") - -- _build() end + -- libc++ headerunit are bugged + -- if semver.compare(clang.version, "17.0") >= 0 then + -- os.exec("xmake clean -a") + -- -- clang-scan-deps dependency detection doesn't support header units atm + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --runtimes=c++_shared -c --yes") + -- _build() + -- end end end end diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index 360a56cbc83..222ec94589b 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -13,33 +13,47 @@ end function main(t) if is_subhost("windows") then + -- local clang = find_tool("clang", {version = true}) + -- if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then + -- os.exec("xmake f --toolchain=clang -c --yes") + -- _build() + -- clang don't support libc++ std modules atm + -- os.exec("xmake clean -a") + -- os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + -- _build() + -- end local msvc = toolchain.load("msvc") if msvc and msvc:check() then local vcvars = msvc:config("vcvars") if vcvars and vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.35") then local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") if os.isdir(stdmodulesdir) then - os.exec("xmake f -c") + os.exec("xmake clean -a") + os.exec("xmake f -c --yes") _build() end end end + elseif is_subhost("msys") then + -- os.exec("xmake f -c -p mingw --yes") + -- _build() elseif is_host("linux") then -- or is_host("macosx") then -- gcc don't support std modules atm -- local gcc = find_tool("gcc", {version = true}) -- if is_host("linux") and gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - -- os.exec("xmake f -c") + -- os.exec("xmake f -c --yes") -- _build() -- end - local clang = find_tool("clang", {version = true}) - if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + -- local clang = find_tool("clang", {version = true}) + -- if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then -- clang don't support libstdc++ std modules atm -- os.exec("xmake clean -a") - -- os.exec("xmake f --toolchain=clang -c") + -- os.exec("xmake f --toolchain=clang -c --yes") -- _build() - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c") - _build() - end + -- clang don't support libc++ std modules atm + -- os.exec("xmake clean -a") + -- os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + -- _build() + -- end end end diff --git a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua index cffe161b2c5..3225b2814cd 100644 --- a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua @@ -1,4 +1,4 @@ target("a") set_languages("cxxlatest") set_kind("object") - add_files("a.mpp") + add_files("a.mpp", {public = true}) diff --git a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua index 98cff412452..24f9c9b1ffd 100644 --- a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua @@ -2,4 +2,4 @@ add_deps("a") set_languages("cxxlatest") set_kind("object") - add_files("b.mpp") + add_files("b.mpp", {public = true}) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index e7212189e74..ebdc3d4dd79 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -64,8 +64,8 @@ function policy.policies() ["build.sanitizer.undefined"] = {description = "Enable undefined sanitizer for c/c++ building.", type = "boolean"}, -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation ["build.c++.modules"] = {description = "Enable C++ modules for C++ building.", type = "boolean"}, - -- Enable clang std modulemap - ["build.c++.clang.stdmodules"] = {description = "Enable clang std modulemap.", default = false, type = "boolean"}, + -- Enable std module + ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, -- Force C++ modules fallback dependency scanner for clang ["build.c++.clang.fallbackscanner"] = {description = "Force clang fallback module dependency scanner.", default = false, type = "boolean"}, -- Force C++ modules fallback dependency scanner for msvc diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua new file mode 100644 index 00000000000..f92f2ed7ef9 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -0,0 +1,370 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file common.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("private.async.buildjobs") +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("utils.progress") +import("compiler_support") +import("dependency_scanner") + +-- build target modules +function _build_modules(target, sourcebatch, modules, opt) + local objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + + _builder(target).populate_module_map(target, modules) + + -- build modules + for _, objectfile in ipairs(objectfiles) do + local module = modules[objectfile] + if not module then + goto CONTINUE + end + + local name, provide, cppfile = compiler_support.get_provided_module(module) + cppfile = cppfile or module.cppfile + + local fileconfig = target:fileconfig(cppfile) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local build = _should_build(target, cppfile, bmifile, {objectfile = objectfile, requires = module.requires}) + + -- add objectfile if module is not from external dep + if not (fileconfig and fileconfig.external) then + target:add("objectfiles", objectfile) + end + + -- needed to detect rebuild of dependencies + if provide and build then + _mark_build(target, name) + end + + local deps = {} + for _, dep in ipairs(table.keys(module.requires or {})) do + table.insert(deps, opt.batchjobs and target:name() .. dep or dep) + end + + opt.build_module(deps, build, module, name, provide, objectfile, cppfile, fileconfig) + + ::CONTINUE:: + end +end + +-- build target headerunits +function _build_headerunits(target, headerunits, opt) + + local outputdir = compiler_support.headerunits_cachedir(target, {mkdir = true}) + if opt.stl_headerunit then + outputdir = path.join(outputdir, "stl") + end + + for _, headerunit in ipairs(headerunits) do + local outputdir = outputdir + if opt.stl_headerunit and headerunit.name:startswith("experimental/") then + outputdir = path.join(outputdir, "experimental") + end + local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) + local key = path.normalize(headerunit.path) + local build = _should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) + + if build then + _mark_build(target, key) + end + + opt.build_headerunit(headerunit, key, bmifile, outputdir, build) + end +end + +-- should we build this module or headerunit ? +function _should_build(target, sourcefile, bmifile, opt) + + -- force rebuild a module if any of its module dependency is rebuilt + local requires = opt.requires + if requires then + for required, _ in table.orderpairs(requires) do + local m = get_from_target_mapper(target, required) + if m then + local rebuild = compiler_support.memcache():get2("should_build_in" .. target:name(), m.key) + if rebuild then + return true + end + end + end + end + + -- or rebuild it if the file changed for headerunit and namedmodules + local objectfile = opt.objectfile + if compiler_support.has_module_extension(sourcefile) or (opt and opt.headerunit) then + local dryrun = option.get("dry-run") + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + + local dependfile = target:dependfile(bmifile or objectfile) + local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) + + -- need build this object? + local depvalues = {compinst:program(), compflags} + local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 + + if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then + return true + end + end + + return false +end + +-- generate meta module informations for package / other buildsystems import +-- +-- e.g +-- { +-- "defines": ["FOO=BAR"] +-- "imports": ["std", "bar"] +-- "name": "foo" +-- "file": "foo.cppm" +-- } +function _generate_meta_module_info(target, name, sourcefile, requires) + + local modulehash = compiler_support.get_modulehash(target, sourcefile) + local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} + + -- add definitions + module_metadata.defines = _builder(target).get_module_required_defines(target, sourcefile) + + -- add imports + if requires then + for _name, _ in pairs(requires) do + module_metadata.imports = module_metadata.imports or {} + table.append(module_metadata.imports, _name) + end + end + + return module_metadata +end + +function _target_module_map_cachekey(target) + local mode = config.mode() + return target:name() .. "module_mapper" .. (mode or "") +end + +function _is_duplicated_headerunit(target, key) + local mapper, mapper_keys = get_target_module_mapper(target) + for _, mapped_key in ipairs(mapper_keys) do + if mapped_key == key then + return mapper[mapped_key] + end + end +end + +function _builder(target) + + local cachekey = tostring(target) + local builder = compiler_support.memcache():get2("builder", cachekey) + if builder == nil then + if target:has_tool("cxx", "clang", "clangxx") then + builder = import("clang.builder", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + builder = import("gcc.builder", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + builder = import("msvc.builder", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) + end + compiler_support.memcache():set2("builder", cachekey, builder) + end + return builder +end + +function _mark_build(target, name) + compiler_support.memcache():set2("should_build_in" .. target:name(), name, true) +end + +-- build batchjobs for modules +function build_batchjobs_for_modules(modules, batchjobs, rootjob) + return buildjobs(modules, batchjobs, rootjob) +end + +-- build modules for batchjobs +function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + opt.rootjob = batchjobs:group_leave() or opt.rootjob + batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) + + local modulesjobs = {} + + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(deps, build, module, name, provide, objectfile, cppfile, fileconfig) + local job_name = name and target:name() .. name or cppfile + + modulesjobs[job_name] = _builder(target).make_module_build_job(target, batchjobs, job_name, deps, {build = build, module = module, objectfile = objectfile, cppfile = cppfile}) + + if provide and fileconfig and fileconfig.public then + batchjobs:addjob(name .. "_metafile", function(index, total) + local metafilepath = compiler_support.get_metafile(target, cppfile) + depend.on_changed(function() + progress.show((index * 100) / total, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) + json.savefile(metafilepath, metadata) + end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) + end, {rootjob = opt.rootjob}) + end + end + })) + + -- build batchjobs for modules + build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) +end + +-- build modules for batchcmds +function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + local depmtime = 0 + opt.progress = opt.progress or 0 + + -- build modules + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(_, build, module, name, provide, objectfile, cppfile, fileconfig) + depmtime = math.max(depmtime, _builder(target).make_module_build_cmds(target, batchcmds, {build = build, module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + + if provide and fileconfig and fileconfig.public then + local metafilepath = compiler_support.get_metafile(target, cppfile) + depend.on_changed(function() + progress.show(opt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) + json.savefile(metafilepath, metadata) + end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) + end + end + })) + + batchcmds:set_depmtime(depmtime) +end + +-- generate headerunits for batchjobs +function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + -- we need new group(headerunits) + -- e.g. group(build_modules) -> group(headerunits) + opt.rootjob = batchjobs:group_leave() or opt.rootjob + batchjobs:group_enter(target:name() .. "/build_headerunits", {rootjob = opt.rootjob}) + + local build_headerunits = function(headerunits) + local modulesjobs = {} + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, key, bmifile, outputdir, build) + local job_name = target:name() .. key + local job = _builder(target).make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + if job then + modulesjobs[job_name] = job + end + end + })) + build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end +end + +-- generate headerunits for batchcmds +function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + + local build_headerunits = function(headerunits) + local depmtime = 0 + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, _, bmifile, outputdir, build) + depmtime = math.max(depmtime, _builder(target).make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, table.join({build = build}, opt))) + end + })) + batchcmds:set_depmtime(depmtime) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end +end + +-- get or create a target module mapper +function get_target_module_mapper(target) + + opt = opt or {} + local memcache = compiler_support.memcache() + local mapper = memcache:get2(target:name(), "module_mapper") + if not mapper then + mapper = {} + memcache:set2(target:name(), "module_mapper", mapper) + end + + return mapper, table.keys(mapper) +end + +-- get a module or headerunit from target mapper +function get_from_target_mapper(target, name) + local mapper = get_target_module_mapper(target) + if mapper[name] then + return mapper[name] + end +end + +-- add a module to target mapper +function add_module_to_target_mapper(target, name, sourcefile, bmifile, opt) + local mapper = get_target_module_mapper(target) + mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt} +end + +-- add a headerunit to target mapper +function add_headerunit_to_target_mapper(target, headerunit, bmifile) + local mapper = get_target_module_mapper(target) + local key = hash.uuid(path.normalize(headerunit.path)) + local deduplicated = _is_duplicated_headerunit(target, key) + if deduplicated then + mapper[headerunit.name] = {name = headerunit.name, key = key, aliasof = deduplicated.name, headerunit = headerunit} + else + mapper[headerunit.name] = {name = headerunit.name, key = key, headerunit = headerunit, bmi = bmifile} + end + return deduplicated and true or false +end + diff --git a/xmake/rules/c++/modules/modules_support/clang.lua b/xmake/rules/c++/modules/modules_support/clang.lua deleted file mode 100644 index 3bf19238def..00000000000 --- a/xmake/rules/c++/modules/modules_support/clang.lua +++ /dev/null @@ -1,986 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file clang.lua --- - --- imports -import("core.base.option") -import("core.base.json") -import("core.base.semver") -import("core.tool.compiler") -import("core.project.project") -import("core.project.depend") -import("core.project.config") -import("lib.detect.find_tool") -import("utils.progress") -import("private.action.build.object", {alias = "objectbuilder"}) -import("common") -import("stl_headers") - --- get bmi path --- @see https://github.com/xmake-io/xmake/issues/4063 -function _get_bmi_path(bmifile) - if is_host("windows") then - bmifile = bmifile:gsub(":", "_") - end - return bmifile -end - --- get clang path, we need get it's absolute path --- @see https://github.com/xmake-io/xmake/issues/4575#issuecomment-1879596450 -function _get_clang_path(target) - local clang_path = _g.clang_path - if not clang_path then - local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local clang = find_tool("clang", {program = program, - envs = os.getenvs(), cachekey = "modules_support_clang"}) - if clang then - clang_path = clang.program - end - end - clang_path = clang_path or false - _g.clang_path = clang_path - end - return clang_path or nil -end - --- get clang version -function _get_clang_version(target) - local clang_version = _g.clang_version - if not clang_version then - local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local clang = find_tool("clang", {program = program, version = true, - envs = os.getenvs(), cachekey = "modules_support_clang"}) - if clang then - clang_version = clang.version - end - end - clang_version = clang_version or false - _g.clang_version = clang_version - end - return clang_version or nil -end - --- get clang-scan-deps -function _get_clang_scan_deps(target) - local clang_scan_deps = _g.clang_scan_deps - if not clang_scan_deps then - local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local dir = path.directory(program) - local basename = path.basename(program) - local extension = path.extension(program) - program = (basename:gsub("clang", "clang-scan-deps")) .. extension - if dir and dir ~= "." and os.isdir(dir) then - program = path.join(dir, program) - end - local result = find_tool("clang-scan-deps", {program = program, version = true}) - if result then - clang_scan_deps = result.program - end - end - clang_scan_deps = clang_scan_deps or false - _g.clang_scan_deps = clang_scan_deps - end - return clang_scan_deps or nil -end - --- add a module or an header unit into the mapper --- --- e.g --- -fmodule-file=build/.gens/Foo/rules/modules/cache/foo.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm --- on LLVM >= 16 --- -fmodule-file=foo=build/.gens/Foo/rules/modules/cache/foo.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm --- -function _add_module_to_mapper(target, name, bmifile, opt) - opt = opt or {} - local modulemap = _get_modulemap_from_mapper(target, name) - if modulemap then - return - end - - local clang_version = _get_clang_version(target) - local namedmodule = opt.namedmodule and semver.compare(clang_version, "16.0") >= 0 - local modulefileflag = get_modulefileflag(target) - local mapflag = namedmodule and format("%s%s=%s", modulefileflag, name, bmifile) or modulefileflag .. bmifile - modulemap = {flag = mapflag, deps = opt.deps} - common.localcache():set2(_mapper_cachekey(target), "modulemap" .. name, modulemap) -end - -function _mapper_cachekey(target) - local mode = config.mode() - return target:name() .. "_modulemap_" .. (mode or "") -end - --- flush modulemap to mapper file cache -function _flush_mapper(target) - -- not using set2/get2 to flush only current target mapper - common.localcache():save(_mapper_cachekey(target)) -end - --- get modulemap from mapper -function _get_modulemap_from_mapper(target, name) - return common.localcache():get2(_mapper_cachekey(target), "modulemap" .. name) or nil -end - --- use the given stdlib? e.g. libc++ or libstdc++ -function _use_stdlib(target, name) - local stdlib = target:data("cxx.modules.stdlib") or "libstdc++" - return stdlib == name -end - --- set stdlib flags, it will use libstdc++ if we do not set `-stdlib=` -function _set_stdlib_flags(target) - if _use_stdlib(target, "libc++") then - target:add("cxxflags", "-stdlib=libc++") - target:add("ldflags", "-stdlib=libc++") - target:add("shflags", "-stdlib=libc++") - end -end - --- load module support for the current target -function load(target) - local clangmodulesflag, modulestsflag, withoutflag = get_modulesflag(target) - - -- add module flags - if not withoutflag then - target:add("cxxflags", modulestsflag) - end - - -- enable clang modules to emulate std modules - if target:policy("build.c++.clang.stdmodules") then - target:add("cxxflags", clangmodulesflag) - end - - -- fix default visibility for functions and variables [-fvisibility] differs in PCH file vs. current file - -- module.pcm cannot be loaded due to a configuration mismatch with the current compilation. - -- - -- it will happen in binary target depend on library target with modules, and enable release mode at same time. - -- - -- @see https://github.com/xmake-io/xmake/issues/3358#issuecomment-1432586767 - local dep_symbols - local has_library_deps = false - for _, dep in ipairs(target:orderdeps()) do - if dep:is_shared() or dep:is_static() or dep:is_object() then - dep_symbols = dep:get("symbols") - has_library_deps = true - break - end - end - if has_library_deps then - target:set("symbols", dep_symbols and dep_symbols or "none") - end - - -- if use libc++, we need to install libc++ and libc++abi - -- - -- on ubuntu: - -- sudo apt install libc++-dev libc++abi-15-dev - -- - local flags = table.join(target:get("cxxflags") or {}, get_config("cxxflags") or {}) - if table.contains(flags, "-stdlib=libc++", "clang::-stdlib=libc++") then - target:data_set("cxx.modules.stdlib", "libc++") - elseif table.contains(flags, "-stdlib=libstdc++", "clang::-stdlib=libstdc++") then - target:data_set("cxx.modules.stdlib", "libstdc++") - end - _set_stdlib_flags(target) -end - --- get includedirs for stl headers --- --- $ echo '#include ' | clang -x c++ -E - | grep '/vector"' --- # 1 "/usr/include/c++/11/vector" 1 3 --- # 58 "/usr/include/c++/11/vector" 3 --- # 59 "/usr/include/c++/11/vector" 3 --- -function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local tmpfile = os.tmpfile() .. ".cc" - io.writefile(tmpfile, "#include ") - local argv = {"-E", "-x", "c++", tmpfile} - if _use_stdlib(target, "libc++") then - table.insert(argv, 1, "-stdlib=libc++") - end - local result = try {function () return os.iorunv(clang, argv) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if line:startswith("#") and line:find("/vector\"", 1, true) then - local includedir = line:match("\"(.+)/vector\"") - if includedir and os.isdir(includedir) then - table.insert(includedirs, path.normalize(includedir)) - break - end - end - end - end - os.tryrm(tmpfile) -end - --- do compile for batchcmds --- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) -end - --- build module file -function _build_modulefile(target, sourcefile, opt) - local objectfile = opt.objectfile - local dependfile = opt.dependfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - - -- need build this object? - local dryrun = option.get("dry-run") - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(objectfile) and os.mtime(dependfile) or 0 - if not dryrun and not depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return - end - - -- init flags - local common_args = opt.common_args - local requiresflags = opt.requiresflags - local moduleoutputflag = get_moduleoutputflag(target) - - -- trace - progress.show(opt.progress, "${color.build.object}compiling.module.$(mode) %s", opt.provide and opt.provide.name or sourcefile) - - local bmifile - local compileflags = {} - local bmiflags - if opt.provide then - bmifile = _get_bmi_path(opt.provide.bmifile) - if bmifile then - common.memcache():set2(bmifile, "compiling", true) - end - if moduleoutputflag then - compileflags = table.join("-x", "c++-module", moduleoutputflag .. bmifile, requiresflags) - else - bmiflags = table.join("-x", "c++-module", "--precompile", compflags, common_args, requiresflags) - end - else - compileflags = {"-x", "c++"} - end - - if bmiflags then - vprint(compinst:compcmd(sourcefile, bmifile, {compflags = bmiflags, rawargs = true})) - end - - compileflags = table.join2(compileflags, compflags, common_args, requiresflags or {}) - vprint(compinst:compcmd(bmiflags and bmifile or sourcefile, objectfile, {compflags = compileflags, rawargs = true})) - - if not dryrun then - - -- do compile - dependinfo.files = {} - if bmiflags then - assert(compinst:compile(sourcefile, bmifile, {dependinfo = dependinfo, compflags = bmiflags})) - end - assert(compinst:compile(bmiflags and bmifile or sourcefile, objectfile, {dependinfo = dependinfo, compflags = compileflags})) - - -- update files and values to the dependent file - dependinfo.values = depvalues - table.join2(dependinfo.files, sourcefile) - table.join2(dependinfo.files, opt.requires or {}) - depend.save(dependinfo, dependfile) - end -end - --- provide toolchain include directories for stl headerunit when p1689 is not supported -function toolchain_includedirs(target) - local includedirs = _g.includedirs - if includedirs == nil then - includedirs = {} - local clang, toolname = target:tool("cxx") - assert(toolname:startswith("clang")) - _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local _, result = try {function () return os.iorunv(clang, {"-E", "-stdlib=libc++", "-Wp,-v", "-xc", os.nuldev()}) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if os.isdir(line) then - table.insert(includedirs, path.normalize(line)) - elseif line:startswith("End") then - break - end - end - end - _g.includedirs = includedirs - end - return includedirs -end - --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) - local compinst = target:compiler("cxx") - local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.object}generating.module.deps %s", sourcefile) - end - - local outputdir = common.get_outputdir(target, sourcefile) - local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) - if has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then - -- We need absolute path of clang to use clang-scan-deps - -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers - local clang_path = compinst:program() - if not path.is_absolute(clang_path) then - clang_path = _get_clang_path(target) or compinst:program() - end - local clangscandeps = _get_clang_scan_deps(target) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join("--format=p1689", "--", - clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile), - compflags) - vprint(table.concat(table.join(clangscandeps, flags), " ")) - local outdata, errdata = os.iorunv(clangscandeps, flags) - assert(errdata, errdata) - - io.writefile(jsonfile, outdata) - else - common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compflags = compinst:compflags({sourcefile = file, target = target}) - -- exclude -fmodule* and -std=c++/gnu++* flags because, - -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation - table.remove_if(compflags, function(_, flag) - return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") - end) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(compflags, {"-E", "-x", "c++", file, "-o", ifile})) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true - - local rawdependinfo = io.readfile(jsonfile) - if rawdependinfo then - local dependinfo = json.decode(rawdependinfo) - if target:data("cxx.modules.stdlib") == nil then - local has_std_modules = false - for _, r in ipairs(dependinfo.rules) do - for _, required in ipairs(r.requires) do - -- it may be `std:utility`, .. - -- @see https://github.com/xmake-io/xmake/issues/3373 - local logical_name = required["logical-name"] - if logical_name and (logical_name == "std" or logical_name:startswith("std.") or logical_name:startswith("std:")) then - has_std_modules = true - break - end - end - - if has_std_modules then - break - end - end - if has_std_modules then - - -- we need clang >= 17.0 or use clang stdmodules if the current target contains std module - local clang_version = _get_clang_version(target) - assert((clang_version and semver.compare(clang_version, "17.0") >= 0) or target:policy("build.c++.clang.stdmodules"), - [[On llvm <= 16 standard C++ modules are not supported ; - they can be emulated through clang modules and supported only on libc++ ; - please add -stdlib=libc++ cxx flag or disable strict mode]]) - - -- we use libc++ by default if we do not explicitly specify -stdlib:libstdc++ - target:data_set("cxx.modules.stdlib", "libc++") - _set_stdlib_flags(target) - end - end - end - - return {moduleinfo = rawdependinfo} - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end - return changed -end - --- generate target stl header units for batchjobs -function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_stl_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - if not os.isfile(bmifile) then - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - -- don't build same header unit at the same time - if not common.memcache():get2(headerunit.name, "building") then - common.memcache():set2(headerunit.name, "building", true) - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local args = {modulecachepathflag .. stlcachedir, "-c", "-Wno-everything", "-o", bmifile, "-x", "c++-system-header", headerunit.name} - os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args)) - end - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - - -- libc++ have a builtin module mapper - if not _use_stdlib(target, "libc++") then - _add_module_to_mapper(target, headerunit.name, bmifile) - end - end, {rootjob = flushjob}) - end - end -end - --- generate target stl header units for batchcmds -function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - -- don't build same header unit at the same time - if not common.memcache():get2(headerunit.name, "building") then - common.memcache():set2(headerunit.name, "building", true) - local flags = { - path(stlcachedir, function (p) return modulecachepathflag .. p end), - "-c", "-Wno-everything", "-o", path(bmifile), "-x", "c++-system-header", headerunit.name} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - end - -- libc++ have a builtin module mapper - if not _use_stdlib(target, "libc++") then - _add_module_to_mapper(target, headerunit.name, bmifile) - end - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- generate target user header units for batchjobs -function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- get cachedirs - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_user_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - - local outputdir = common.get_outputdir(target, headerunit.path) - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local objectdir = path.directory(objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - -- generate headerunit - local args = { modulecachepathflag .. cachedir, "-c", "-o", bmifile} - if headerunit.type == ":quote" then - table.join2(args, {"-I", path.directory(headerunit.path), "-x", "c++-user-header", headerunit.path}) - elseif headerunit.type == ":angle" then - table.join2(args, {"-x", "c++-system-header", headerunit.name}) - end - os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args)) - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - _add_module_to_mapper(target, headerunit.name, bmifile) - end, {rootjob = flushjob}) - end -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- get cachedirs - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - - local outputdir = common.get_outputdir(target, headerunit.path) - batchcmds:mkdir(outputdir) - - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = (outputdir and path.join(outputdir, bmifilename) or bmifilename) - batchcmds:mkdir(path.directory(objectfile)) - - local flags = {path(cachedir, function (p) return modulecachepathflag .. p end), "-c", "-o", path(bmifile)} - if headerunit.type == ":quote" then - table.join2(flags, {"-I", path(headerunit.path):directory(), "-x", "c++-user-header", path(headerunit.path)}) - elseif headerunit.type == ":angle" then - table.join2(flags, {"-x", "c++-system-header", headerunit.name}) - end - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - batchcmds:add_depfiles(headerunit.path) - - _add_module_to_mapper(target, headerunit.name, bmifile) - - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- build module files for batchjobs -function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) - - -- get flags - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_modules", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build modules - local common_args = {modulecachepathflag .. cachedir} - local modulesjobs = {} - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - -- assume there that provides is only one, until we encounter the case - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local moduleinfo = table.copy(provide) or {} - - if provide then - local fileconfig = target:fileconfig(cppfile) - if fileconfig and fileconfig.install then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = common.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show(opt.progress, "${color.build.object}generating.module.metadata %s", name) - local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = flushjob}) - end - end - - table.join2(moduleinfo, { - name = name or cppfile, - deps = table.keys(module.requires or {}), - sourcefile = cppfile, - job = batchjobs:newjob(name or cppfile, function(index, total) - -- @note we add it at the end to ensure that the full modulemap are already stored in the mapper - local requiresflags - local requires - if module.requires then - requiresflags = get_requiresflags(target, module.requires) - requires = get_requires(target, module.requires) - end - - if provide or common.has_module_extension(cppfile) then - local bmifile = _get_bmi_path(provide and provide.bmi) - if not common.memcache():get2(name or cppfile, "compiling") then - if name and module.external then - common.memcache():set2(name or cppfile, "compiling", true) - end - _build_modulefile(target, provide and provide.sourcefile or cppfile, { - objectfile = objectfile, - dependfile = target:dependfile(bmifile or objectfile), - provide = provide and {bmifile = bmifile, name = name}, - common_args = common_args, - requiresflags = requiresflags, - requires = requires, - progress = (index * 100) / total}) - end - target:add("objectfiles", objectfile) - - if provide then - _add_module_to_mapper(target, name, bmifile, {deps = requiresflags, namedmodule = true}) - end - elseif requiresflags then - local cxxflags = {} - for _, flag in ipairs(requiresflags) do - -- we need to wrap flag to support flag with space - if type(flag) == "string" and flag:find(" ", 1, true) then - table.insert(cxxflags, {flag}) - else - table.insert(cxxflags, flag) - end - end - target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) - - -- force rebuild .cpp file if any of its module dependency is rebuilt - local rebuild = false - for _, requiredfile in ipairs(requires) do - if common.memcache():get2(requiredfile, "compiling") == true then - rebuild = true - break - end - end - if rebuild then - os.tryrm(target:objectfile(cppfile)) - end - end - end)}) - modulesjobs[name or cppfile] = moduleinfo - end - end - - -- build batchjobs for modules - common.build_batchjobs_for_modules(modulesjobs, batchjobs, flushjob) -end - --- build module files for batchcmds -function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- build modules - local depmtime = 0 - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local requiresflags - if module.requires then - requiresflags = get_requiresflags(target, module.requires) - end - - local flags = table.join({path(cachedir, function (p) return modulecachepathflag .. p end)}, requiresflags or {}) - if provide or common.has_module_extension(cppfile) then - local file = provide and path(provide.bmi) or path(cppfile) - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.module.$(mode) %s", name or cppfile) - batchcmds:mkdir(path.directory(objectfile)) - if provide then - _batchcmds_compile(batchcmds, target, table.join(flags, - {"-x", "c++-module", "--precompile", "-c", path(cppfile), "-o", path(provide.bmi)}), cppfile) - _add_module_to_mapper(target, name, provide.bmi, {namedmodule = true}) - -- add requiresflags to module. it will be used for project generation - target:fileconfig_add(cppfile, {force = {cxxflags = requiresflags}}) - end - _batchcmds_compile(batchcmds, target, table.join(flags, - not provide and {"-x", "c++"} or {}, {"-c", file, "-o", path(objectfile)}), file) - target:add("objectfiles", objectfile) - elseif requiresflags then - local cxxflags = {} - for _, flag in ipairs(requiresflags) do - -- we need to wrap flag to support flag with space - if type(flag) == "string" and flag:find(" ", 1, true) then - table.insert(cxxflags, {flag}) - else - table.insert(cxxflags, flag) - end - end - target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) - end - - batchcmds:add_depfiles(cppfile) - depmtime = math.max(depmtime, os.mtime(objectfile)) - end - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- not supported atm -function get_stdmodules(target) - local modules = {} - return modules -end - -function get_bmi_extension() - return ".pcm" -end - -function get_modulesflag(target) - local clangmodulesflag = _g.clangmodulesflag - local modulestsflag = _g.modulestsflag - local withoutflag = _g.withoutflag - if clangmodulesflag == nil and modulestsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules", "cxxflags", {flagskey = "clang_modules"}) then - clangmodulesflag = "-fmodules" - end - if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "clang_modules_ts"}) then - modulestsflag = "-fmodules-ts" - end - local clang_version = _get_clang_version(target) - withoutflag = semver.compare(clang_version, "16.0") >= 0 - assert(withoutflag or modulestsflag, "compiler(clang): does not support c++ module!") - _g.clangmodulesflag = clangmodulesflag or false - _g.modulestsflag = modulestsflag or false - _g.withoutflag = withoutflag or false - end - return clangmodulesflag or nil, modulestsflag or nil, withoutflag or nil -end - -function get_builtinmodulemapflag(target) - local builtinmodulemapflag = _g.builtinmodulemapflag - if builtinmodulemapflag == nil then - -- this flag seems clang on mingw doesn't distribute it - -- @see https://github.com/xmake-io/xmake/pull/2833 - if not target:is_plat("mingw") then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fbuiltin-module-map", "cxxflags", {flagskey = "clang_builtin_module_map"}) then - builtinmodulemapflag = "-fbuiltin-module-map" - end - assert(builtinmodulemapflag, "compiler(clang): does not support c++ module!") - end - _g.builtinmodulemapflag = builtinmodulemapflag or false - end - return builtinmodulemapflag or nil -end - -function get_implicitmodulesflag(target) - local implicitmodulesflag = _g.implicitmodulesflag - if implicitmodulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fimplicit-modules", "cxxflags", {flagskey = "clang_implicit_modules"}) then - implicitmodulesflag = "-fimplicit-modules" - end - assert(implicitmodulesflag, "compiler(clang): does not support c++ module!") - _g.implicitmodulesflag = implicitmodulesflag or false - end - return implicitmodulesflag or nil -end - -function get_implicitmodulemapsflag(target) - local implicitmodulemapsflag = _g.implicitmodulemapsflag - if implicitmodulemapsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fimplicit-module-maps", "cxxflags", {flagskey = "clang_implicit_module_map"}) then - implicitmodulemapsflag = "-fimplicit-module-maps" - end - assert(implicitmodulemapsflag, "compiler(clang): does not support c++ module!") - _g.implicitmodulemapsflag = implicitmodulemapsflag or false - end - return implicitmodulemapsflag or nil -end - -function get_noimplicitmodulemapsflag(target) - local noimplicitmodulemapsflag = _g.noimplicitmodulemapsflag - if noimplicitmodulemapsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fno-implicit-module-maps", "cxxflags", {flagskey = "clang_no_implicit_module_maps"}) then - noimplicitmodulemapsflag = "-fno-implicit-module-maps" - end - assert(noimplicitmodulemapsflag, "compiler(clang): does not support c++ module!") - _g.noimplicitmodulemapsflag = noimplicitmodulemapsflag or false - end - return noimplicitmodulemapsflag or nil -end - -function get_prebuiltmodulepathflag(target) - local prebuiltmodulepathflag = _g.prebuiltmodulepathflag - if prebuiltmodulepathflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fprebuilt-module-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_prebuild_module_path"}) then - prebuiltmodulepathflag = "-fprebuilt-module-path=" - end - assert(prebuiltmodulepathflag, "compiler(clang): does not support c++ module!") - _g.prebuiltmodulepathflag = prebuiltmodulepathflag or false - end - return prebuiltmodulepathflag or nil -end - -function get_modulecachepathflag(target) - local modulecachepathflag = _g.modulecachepathflag - if modulecachepathflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules-cache-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_modules_cache_path"}) then - modulecachepathflag = "-fmodules-cache-path=" - end - assert(modulecachepathflag, "compiler(clang): does not support c++ module!") - _g.modulecachepathflag = modulecachepathflag or false - end - return modulecachepathflag or nil -end - -function get_modulefileflag(target) - local modulefileflag = _g.modulefileflag - if modulefileflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodule-file=" .. os.tmpfile() .. get_bmi_extension(), "cxxflags", {flagskey = "clang_module_file"}) then - modulefileflag = "-fmodule-file=" - end - assert(modulefileflag, "compiler(clang): does not support c++ module!") - _g.modulefileflag = modulefileflag or false - end - return modulefileflag or nil -end - -function has_headerunitsupport(target) - local support_headerunits = _g.support_headerunits - if support_headerunits == nil then - local compinst = target:compiler("cxx") - local _, modulestsflag, withoutflag = get_modulesflag(target) - modulestsflag = withoutflag and "" or modulestsflag - if compinst:has_flags(modulestsflag .. " -std=c++20 -x c++-user-header", "cxxflags", { - snippet = "inline int foo() { return 0; }", - flagskey = "clang_user_header_unit_support", - tryrun = true}) and - compinst:has_flags(modulestsflag .. " -std=c++20 -x c++-system-header", "cxxflags", { - snippet = "inline int foo() { return 0; }", - flagskey = "clang_system_header_unit_support", - tryrun = true}) then - support_headerunits = true - end - _g.support_headerunits = support_headerunits or false - end - return support_headerunits or nil -end - -function has_clangscandepssupport(target) - local support_clangscandeps = _g.support_clangscandeps - if support_clangscandeps == nil then - local clangscandeps = _get_clang_scan_deps(target) - local clang_version = _get_clang_version(target) - if clangscandeps and clang_version and semver.compare(clang_version, "16.0") >= 0 then - support_clangscandeps = true - end - _g.support_clangscandeps = support_clangscandeps or false - end - return support_clangscandeps or nil -end - -function get_moduleoutputflag(target) - local moduleoutputflag = _g.moduleoutputflag - if moduleoutputflag == nil then - local compinst = target:compiler("cxx") - local clang_version = _get_clang_version(target) - if compinst:has_flags("-fmodule-output=", "cxxflags", {flagskey = "clang_module_output", tryrun = true}) and - semver.compare(clang_version, "16.0") >= 0 then - moduleoutputflag = "-fmodule-output=" - end - _g.moduleoutputflag = moduleoutputflag or false - end - return moduleoutputflag or nil -end - -function get_requires(target, requires) - local requires - local flags = get_requiresflags(target, requires) - for _, flag in ipairs(flags) do - requires = requires or {} - table.insert(requires, flag:split("=")[3]) - end - return requires -end - -function get_requiresflags(target, requires) - local flags = {} - -- add deps required module flags - local already_mapped_modules = {} - for name, _ in table.orderpairs(requires) do - -- if already in flags, continue - if already_mapped_modules[name] then - goto continue - end - - for _, dep in ipairs(target:orderdeps()) do - local modulemap_ = _get_modulemap_from_mapper(dep, name) - if modulemap_ then - already_mapped_modules[name] = true - table.insert(flags, modulemap_.flag) - if modulemap_.deps then - table.join2(flags, modulemap_.deps) - end - goto continue - end - end - - -- append target required module mapper flags - local modulemap = _get_modulemap_from_mapper(target, name) - if modulemap then - already_mapped_modules[name] = true - table.insert(flags, modulemap.flag) - if modulemap.deps then - table.join2(flags, modulemap.deps) - end - goto continue - end - - ::continue:: - end - if #flags > 0 then - return table.unique(flags) - end -end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua new file mode 100644 index 00000000000..64243fd3f60 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -0,0 +1,349 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file clang/builder.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("core.base.semver") +import("utils.progress") +import("private.action.build.object", {alias = "objectbuilder"}) +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("compiler_support") +import(".builder", {inherit = true}) + +-- get flags for building a module +function _make_modulebuildflags(target, provide, bmifile, opt) + + -- get flags + local module_outputflag = compiler_support.get_moduleoutputflag(target) + + local flags + local precompile = false + if module_outputflag and provide and not opt.external then -- one step compilation of named module, clang >= 16 + flags = {{"-x", "c++-module", module_outputflag .. bmifile}} + elseif provide then -- two step compilation of named module + precompile = true + flags = {{"-x", "c++-module", "--precompile"}} + + if not opt.external then + table.insert(flags, {}) + end + else -- internal module, no bmi needed + flags = {{"-x", "c++"}} + end + + if opt.name == "std" or opt.name == "std.compat" then + table.join2(flags[1], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"}) + if flags[2] then + table.join2(flags[2], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"}) + end + end + + return precompile, table.unpack(flags) +end + +-- get flags for building a headerunit +function _make_headerunitflags(target, headerunit, bmifile) + + local module_headerflag = compiler_support.get_moduleheaderflag(target) + assert(module_headerflag, "compiler(clang): does not support c++ header units!") + + local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} + + local headertype = (headerunit.type == ":angle") and "system" or "user" + + local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype}) + + return flags +end + +-- do compile +function _compile(target, flags, sourcefile, outputfile, opt) + + opt = opt or {} + local dryrun = option.get("dry-run") + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, flags) + + -- trace + if option.get("verbose") then + print(compinst:compcmd(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) + end + + if not dryrun then + -- do compile + assert(compinst:compile(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags})) + end +end + +-- do compile for batchcmds +-- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx +function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile, opt) + opt = opt or {} + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, opt.bmifile or sourcefile}) + batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) +end + +-- get module requires flags +-- e.g +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/foo.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm +-- on LLVM >= 16 +-- -fmodule-file=foo=build/.gens/Foo/rules/modules/cache/foo.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm +-- +function _get_requiresflags(target, module, opt) + + local modulefileflag = compiler_support.get_modulefileflag(target) + + local name = module.name + local cachekey = target:name() .. name + + local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") + or compiler_support.localcache():get2(cachekey, "requiresflags") + + if not requiresflags or (opt and opt.regenerate) then + requiresflags = {} + for required, _ in table.orderpairs(module.requires) do + local dep_module = get_from_target_mapper(target, required) + + assert(dep_module, "module dependency %s required for %s not found", required, name) + + local bmifile = dep_module.bmi + -- aliased headerunit + if dep_module.aliasof then + local aliased = get_from_target_mapper(target, dep_module.aliasof) + bmifile = aliased.bmi + end + local mapflag = (dep_module.opt and dep_module.opt.namedmodule) and format("%s%s=%s", modulefileflag, required, bmifile) or modulefileflag .. bmifile + table.insert(requiresflags, mapflag) + + -- append deps + if dep_module.opt and dep_module.opt.deps then + local deps = _get_requiresflags(target, {name = dep_module.name or dep_module.sourcefile, bmi = bmifile, requires = dep_module.opt.deps}) + table.join2(requiresflags, deps) + end + end + compiler_support.memcache():set2(cachekey, "requiresflags", table.unique(requiresflags)) + compiler_support.localcache():set2(cachekey, "requiresflags", table.unique(requiresflags)) + end + + return requiresflags +end + +function _append_requires_flags(target, module, name, cppfile, bmifile, opt) + + local cxxflags = {} + local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build}) + + for _, flag in ipairs(requiresflags) do + -- we need to wrap flag to support flag with space + if type(flag) == "string" and flag:find(" ", 1, true) then + table.insert(cxxflags, {flag}) + else + table.insert(cxxflags, flag) + end + end + target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) +end + +-- populate module map +function populate_module_map(target, modules) + local clang_version = compiler_support.get_clang_version(target) + local support_namedmodule = semver.compare(clang_version, "16.0") >= 0 + + for _, module in pairs(modules) do + local name, provide, cppfile = compiler_support.get_provided_module(module) + if provide then + local bmifile = compiler_support.get_bmi_path(provide.bmi) + add_module_to_target_mapper(target, name, cppfile, bmifile, {deps = module.requires, namedmodule = support_namedmodule}) + end + end +end + +-- get defines for a module +function get_module_required_defines(target, sourcefile) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local defines + + for _, flag in ipairs(compflags) do + if flag:startswith("-D") then + defines = defines or {} + table.insert(defines, flag:sub(3)) + end + end + + return defines +end + +-- build module file for batchjobs +function make_module_build_job(target, batchjobs, job_name, deps, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + return { + name = job_name, + deps = deps, + sourcefile = cppfile, + job = batchjobs:newjob(name or opt.cppfile, function(index, total) + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + -- compile if it's a named module + if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) + + _compile(target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) + + if second_step then + _compile(target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + end + end + + table.insert(dependinfo.files, opt.cppfile) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build module file for batchcmds +function make_module_build_cmds(target, batchcmds, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- compile if it's a named module + if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:mkdir(path.directory(opt.objectfile)) + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) + _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) + + if second_step then + _batchcmds_compile(batchcmds, target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + end + end + batchcmds:add_depfiles(opt.cppfile) + + return os.mtime(opt.objectfile) +end + +-- build headerunit file for batchjobs +function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) + + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} + end +end + +-- build headerunit file for batchcmds +function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) + + batchcmds:mkdir(outputdir) + + add_headerunit_to_target_mapper(target, headerunit, bmifile) + + if opt.build then + local name = headerunit.unique and headerunit.name or headerunit.path + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), bmifile) + end + batchcmds:add_depfiles(headerunit.path) + return os.mtime(bmifile) +end + +function get_requires(target, module) + + local _requires + local flags = _get_requiresflags(target, module) + for _, flag in ipairs(flags) do + _requires = _requires or {} + table.insert(_requires, flag:split("=")[3]) + end + return _requires +end + diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua new file mode 100644 index 00000000000..7557bbfde65 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -0,0 +1,401 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file clang/compiler_support.lua +-- + +-- imports +import("core.base.semver") +import("lib.detect.find_tool") +import(".compiler_support", {inherit = true}) + +-- get includedirs for stl headers +-- +-- $ echo '#include ' | clang -x c++ -E - | grep '/vector"' +-- # 1 "/usr/include/c++/11/vector" 1 3 +-- # 58 "/usr/include/c++/11/vector" 3 +-- # 59 "/usr/include/c++/11/vector" 3 +-- +function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) + local tmpfile = os.tmpfile() .. ".cc" + io.writefile(tmpfile, "#include ") + local argv = {"-E", "-x", "c++", tmpfile} + local cpplib = get_cpplibrary_name(target) + if cpplib then + if cpplib == "c++" then + table.insert(argv, 1, "-stdlib=libc++") + elseif cpplib == "stdc++" then + table.insert(argv, 1, "-stdlib=libstdc++") + end + end + local result = try {function () return os.iorunv(clang, argv) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if line:startswith("#") and line:find("/vector\"", 1, true) then + local includedir = line:match("\"(.+)/vector\"") + if includedir and os.isdir(includedir) then + table.insert(includedirs, path.normalize(includedir)) + break + end + end + end + end + os.tryrm(tmpfile) +end + +function get_cpplibrary_name(target) + -- libc++ come first because on windows, if we use libc++ clang will still use msvc crt so MD / MT / MDd / MTd can be set + if target:has_runtime("c++_shared", "c++_static") then + return "c++" + elseif target:has_runtime("stdc++_shared", "stdc++_static") then + return "stdc++" + elseif target:is_plat("windows") and target:has_runtime("MD", "MT", "MDd", "MTd") then + return "msstl" + end +end + +-- load module support for the current target +function load(target) + local clangmodulesflag, modulestsflag, withoutflag = get_modulesflag(target) + + -- add module flags + if not withoutflag then + target:add("cxxflags", modulestsflag) + end + + -- fix default visibility for functions and variables [-fvisibility] differs in PCH file vs. current file + -- module.pcm cannot be loaded due to a configuration mismatch with the current compilation. + -- + -- it will happen in binary target depend on library target with modules, and enable release mode at same time. + -- + -- @see https://github.com/xmake-io/xmake/issues/3358#issuecomment-1432586767 + local dep_symbols + local has_library_deps = false + for _, dep in ipairs(target:orderdeps()) do + if dep:is_shared() or dep:is_static() or dep:is_object() then + dep_symbols = dep:get("symbols") + has_library_deps = true + break + end + end + if has_library_deps then + target:set("symbols", dep_symbols and dep_symbols or "none") + end + + -- on Windows before llvm18 we need to disable delayed-template-parsing because it's incompatible with modules, from llvm >= 18, it's disabled by default + local clang_version = get_clang_version(target) + if semver.compare(clang_version, "18") < 0 then + target:add("cxxflags", "-fno-delayed-template-parsing") + end +end + +-- provide toolchain include directories for stl headerunit when p1689 is not supported +function toolchain_includedirs(target) + local includedirs = _g.includedirs + if includedirs == nil then + includedirs = {} + local clang, toolname = target:tool("cxx") + assert(toolname:startswith("clang")) + _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) + local cpplib = get_cpplibrary_name(target) + local runtime_flag + if cpplib then + if cpplib == "c++" then + runtime_flag = "-stdlib=libc++" + elseif cpplib == "stdc++" then + runtime_flag = "-stdlib=libstdc++" + end + end + local _, result = try {function () return os.iorunv(clang, table.join({"-E", "-Wp,-v", "-xc", os.nuldev()}, runtime_flag or {})) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if os.isdir(line) then + table.insert(includedirs, path.normalize(line)) + elseif line:startswith("End") then + break + end + end + end + _g.includedirs = includedirs + end + return includedirs +end + +-- get clang path +function get_clang_path(target) + local clang_path = _g.clang_path + if not clang_path then + local program, toolname = target:tool("cxx") + if program and (toolname == "clang" or toolname == "clangxx") then + local clang = find_tool("clang", {program = program}) + if clang then + clang_path = clang.program + end + end + clang_path = clang_path or false + _g.clang_path = clang_path + end + return clang_path or nil +end + +-- get clang version +function get_clang_version(target) + local clang_version = _g.clang_version + if not clang_version then + local program, toolname = target:tool("cxx") + if program and (toolname == "clang" or toolname == "clangxx") then + local clang = find_tool("clang", {program = program, version = true}) + if clang then + clang_version = clang.version + end + end + clang_version = clang_version or false + _g.clang_version = clang_version + end + return clang_version or nil +end + +-- get clang-scan-deps +function get_clang_scan_deps(target) + local clang_scan_deps = _g.clang_scan_deps + if not clang_scan_deps then + local program, toolname = target:tool("cxx") + if program and (toolname == "clang" or toolname == "clangxx") then + local dir = path.directory(program) + local basename = path.basename(program) + local extension = path.extension(program) + program = (basename:gsub("clang", "clang-scan-deps")) .. extension + if dir and dir ~= "." and os.isdir(dir) then + program = path.join(dir, program) + end + local result = find_tool("clang-scan-deps", {program = program, version = true}) + if result then + clang_scan_deps = result.program + end + end + clang_scan_deps = clang_scan_deps or false + _g.clang_scan_deps = clang_scan_deps + end + return clang_scan_deps or nil +end + +-- not supported atm +function get_stdmodules(target) + if target:policy("build.c++.modules.std") then + local cpplib = get_cpplibrary_name(target) + if cpplib then + if cpplib == "c++" then + -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 + elseif cpplib == "stdc++" then + -- libstdc++ doesn't have a std module file atm + elseif cpplib == "msstl" then + -- msstl std module file is not compatible with llvm <= 19 + -- local toolchain = target:toolchain("clang") + -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) + -- if msvc then + -- local vcvars = msvc:config("vcvars") + -- if vcvars.VCInstallDir and vcvars.VCToolsVersion then + -- modules = {} + -- + -- local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + -- assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + -- + -- return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + -- end + -- end + end + end + end + return {} +end + +function get_bmi_extension() + return ".pcm" +end + +function get_modulesflag(target) + local clangmodulesflag = _g.clangmodulesflag + local modulestsflag = _g.modulestsflag + local withoutflag = _g.withoutflag + if clangmodulesflag == nil and modulestsflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules", "cxxflags", {flagskey = "clang_modules"}) then + clangmodulesflag = "-fmodules" + end + if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "clang_modules_ts"}) then + modulestsflag = "-fmodules-ts" + end + local clang_version = get_clang_version(target) + withoutflag = semver.compare(clang_version, "16.0") >= 0 + assert(withoutflag or modulestsflag, "compiler(clang): does not support c++ module!") + _g.clangmodulesflag = clangmodulesflag or false + _g.modulestsflag = modulestsflag or false + _g.withoutflag = withoutflag or false + end + return clangmodulesflag or nil, modulestsflag or nil, withoutflag or nil +end + +function get_builtinmodulemapflag(target) + local builtinmodulemapflag = _g.builtinmodulemapflag + if builtinmodulemapflag == nil then + -- this flag seems clang on mingw doesn't distribute it + -- @see https://github.com/xmake-io/xmake/pull/2833 + if not target:is_plat("mingw") then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fbuiltin-module-map", "cxxflags", {flagskey = "clang_builtin_module_map"}) then + builtinmodulemapflag = "-fbuiltin-module-map" + end + assert(builtinmodulemapflag, "compiler(clang): does not support c++ module!") + end + _g.builtinmodulemapflag = builtinmodulemapflag or false + end + return builtinmodulemapflag or nil +end + +function get_implicitmodulesflag(target) + local implicitmodulesflag = _g.implicitmodulesflag + if implicitmodulesflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fimplicit-modules", "cxxflags", {flagskey = "clang_implicit_modules"}) then + implicitmodulesflag = "-fimplicit-modules" + end + assert(implicitmodulesflag, "compiler(clang): does not support c++ module!") + _g.implicitmodulesflag = implicitmodulesflag or false + end + return implicitmodulesflag or nil +end + +function get_implicitmodulemapsflag(target) + local implicitmodulemapsflag = _g.implicitmodulemapsflag + if implicitmodulemapsflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fimplicit-module-maps", "cxxflags", {flagskey = "clang_implicit_module_map"}) then + implicitmodulemapsflag = "-fimplicit-module-maps" + end + assert(implicitmodulemapsflag, "compiler(clang): does not support c++ module!") + _g.implicitmodulemapsflag = implicitmodulemapsflag or false + end + return implicitmodulemapsflag or nil +end + +function get_noimplicitmodulemapsflag(target) + local noimplicitmodulemapsflag = _g.noimplicitmodulemapsflag + if noimplicitmodulemapsflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fno-implicit-module-maps", "cxxflags", {flagskey = "clang_no_implicit_module_maps"}) then + noimplicitmodulemapsflag = "-fno-implicit-module-maps" + end + assert(noimplicitmodulemapsflag, "compiler(clang): does not support c++ module!") + _g.noimplicitmodulemapsflag = noimplicitmodulemapsflag or false + end + return noimplicitmodulemapsflag or nil +end + +function get_prebuiltmodulepathflag(target) + local prebuiltmodulepathflag = _g.prebuiltmodulepathflag + if prebuiltmodulepathflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fprebuilt-module-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_prebuild_module_path"}) then + prebuiltmodulepathflag = "-fprebuilt-module-path=" + end + assert(prebuiltmodulepathflag, "compiler(clang): does not support c++ module!") + _g.prebuiltmodulepathflag = prebuiltmodulepathflag or false + end + return prebuiltmodulepathflag or nil +end + +function get_modulecachepathflag(target) + local modulecachepathflag = _g.modulecachepathflag + if modulecachepathflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules-cache-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_modules_cache_path"}) then + modulecachepathflag = "-fmodules-cache-path=" + end + assert(modulecachepathflag, "compiler(clang): does not support c++ module!") + _g.modulecachepathflag = modulecachepathflag or false + end + return modulecachepathflag or nil +end + +function get_modulefileflag(target) + local modulefileflag = _g.modulefileflag + if modulefileflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-file=" .. os.tmpfile() .. get_bmi_extension(), "cxxflags", {flagskey = "clang_module_file"}) then + modulefileflag = "-fmodule-file=" + end + assert(modulefileflag, "compiler(clang): does not support c++ module!") + _g.modulefileflag = modulefileflag or false + end + return modulefileflag or nil +end + +function get_moduleheaderflag(target) + local moduleheaderflag = _g.moduleheaderflag + if moduleheaderflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-header=system", "cxxflags", {flagskey = "clang_module_header"}) then + moduleheaderflag = "-fmodule-header=" + end + _g.moduleheaderflag = moduleheaderflag or false + end + return moduleheaderflag or nil +end + +function has_clangscandepssupport(target) + local support_clangscandeps = _g.support_clangscandeps + if support_clangscandeps == nil then + local clangscandeps = get_clang_scan_deps(target) + local clang_version = get_clang_version(target) + if clangscandeps and clang_version and semver.compare(clang_version, "16.0") >= 0 then + support_clangscandeps = true + end + _g.support_clangscandeps = support_clangscandeps or false + end + return support_clangscandeps or nil +end + +function get_keepsystemincludesflag(target) + local keepsystemincludesflag = _g.keepsystemincludesflag + if keepsystemincludesflag == nil then + local compinst = target:compiler("cxx") + local clang_version = get_clang_version(target) + if compinst:has_flags("-E -fkeep-system-includes", "cxxflags", {flagskey = "clang_keep_system_includes", tryrun = true}) and + semver.compare(clang_version, "18.0") >= 0 then + keepsystemincludesflag = "-fkeep-system-includes" + end + _g.keepsystemincludesflag = keepsystemincludesflag or false + end + return keepsystemincludesflag or nil +end + +function get_moduleoutputflag(target) + local moduleoutputflag = _g.moduleoutputflag + if moduleoutputflag == nil then + local compinst = target:compiler("cxx") + local clang_version = get_clang_version(target) + if compinst:has_flags("-fmodule-output=", "cxxflags", {flagskey = "clang_module_output", tryrun = true}) and + semver.compare(clang_version, "16.0") >= 0 then + moduleoutputflag = "-fmodule-output=" + end + _g.moduleoutputflag = moduleoutputflag or false + end + return moduleoutputflag or nil +end + diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua new file mode 100644 index 00000000000..16586d4e0d8 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -0,0 +1,82 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file clang/dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.semver") +import("core.project.depend") +import("utils.progress") +import("compiler_support") +import(".dependency_scanner", {inherit = true}) + +-- generate dependency files +function generate_dependencies(target, sourcebatch, opt) + local compinst = target:compiler("cxx") + local changed = false + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function() + if opt.progress then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + end + + local outputdir = compiler_support.get_outputdir(target, sourcefile) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) + if compiler_support.has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then + -- We need absolute path of clang to use clang-scan-deps + -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers + local clang_path = compinst:program() + if not path.is_absolute(clang_path) then + clang_path = compiler_support.get_clang_path(target) or compinst:program() + end + local clangscandeps = compiler_support.get_clang_scan_deps(target) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join({"--format=p1689", "--", + clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) + vprint(table.concat(table.join(clangscandeps, flags), " ")) + local outdata, err = os.iorunv(clangscandeps, flags) + assert(err, err) + + io.writefile(jsonfile, outdata) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target) + local compflags = compinst:compflags({sourcefile = file, target = target}) + -- exclude -fmodule* and -std=c++/gnu++* flags because, + -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation + table.remove_if(compflags, function(_, flag) + return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") + end) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) + os.vrunv(compinst:program(), flags) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local rawdependinfo = io.readfile(jsonfile) + return {moduleinfo = rawdependinfo} + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) + end + return changed +end diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua new file mode 100644 index 00000000000..3bf89b885e7 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -0,0 +1,281 @@ +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file compiler_support.lua +-- + +-- imports +import("core.base.json") +import("core.base.hashset") +import("core.cache.memcache", {alias = "_memcache"}) +import("core.cache.localcache", {alias = "_localcache"}) +import("lib.detect.find_file") +import("core.project.project") +import("core.project.config") + +function _compiler_support(target) + local cachekey = tostring(target) + local compiler_support = memcache():get2("compiler_support", cachekey) + if compiler_support == nil then + if target:has_tool("cxx", "clang", "clangxx") then + compiler_support = import("clang.compiler_support", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + compiler_support = import("gcc.compiler_support", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + compiler_support = import("msvc.compiler_support", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) + end + memcache():set2("compiler_support", cachekey, compiler_support) + end + return compiler_support +end + +-- load module support for the current target +function load(target) + _compiler_support(target).load(target) +end + +-- patch sourcebatch +function patch_sourcebatch(target, sourcebatch) + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + + local dependfile = target:dependfile(sourcefile or objectfile) + table.insert(sourcebatch.dependfiles, dependfile) + end +end + +-- cull sourcebatch objectfiles +function cull_objectfiles(target, sourcebatch) + + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = target:fileconfig(sourcefile) + if not (fileconfig and fileconfig.external) then + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + end + end +end + +-- get bmi extension +function get_bmi_extension(target) + return _compiler_support(target).get_bmi_extension() +end + +-- get bmi path +-- @see https://github.com/xmake-io/xmake/issues/4063 +function get_bmi_path(bmifile) + bmifile = bmifile:gsub(":", "_PARTITION_") + return path.normalize(bmifile) +end + +-- has module extension? e.g. *.mpp, ... +function has_module_extension(sourcefile) + local modulexts = _g.modulexts + if modulexts == nil then + modulexts = hashset.of(".mpp", ".mxx", ".cppm", ".ixx") + _g.modulexts = modulexts + end + local extension = path.extension(sourcefile) + return modulexts:has(extension:lower()) +end + +-- this target contains module files? +function contains_modules(target) + -- we can not use `"c++.build.builder"`, because it contains sourcekind/cxx. + local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false + if not target_with_modules then + target_with_modules = target:policy("build.c++.modules") + end + if not target_with_modules then + for _, dep in ipairs(target:orderdeps()) do + local sourcebatches = dep:sourcebatches() + if sourcebatches["c++.build.modules"] then + target_with_modules = true + break + end + end + end + return target_with_modules +end + +-- load module infos +function load_moduleinfos(target, sourcebatch) + local moduleinfos + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + if os.isfile(dependfile) then + local data = io.load(dependfile) + if data then + moduleinfos = moduleinfos or {} + local moduleinfo = json.decode(data.moduleinfo) + moduleinfo.sourcefile = sourcefile + if moduleinfo then + table.insert(moduleinfos, moduleinfo) + end + end + end + end + return moduleinfos +end + +function find_quote_header_file(target, sourcefile, file) + local p = path.join(path.directory(path.absolute(sourcefile, project.directory())), file) + assert(os.isfile(p)) + return p +end + +function find_angle_header_file(target, file) + local headerpaths = _compiler_support(target).toolchain_includedirs(target) + for _, dep in ipairs(target:orderdeps()) do + local includedirs = table.join(dep:get("sysincludedirs") or {}, dep:get("includedirs") or {}) + table.join2(headerpaths, includedirs) + end + for _, pkg in ipairs(target:orderpkgs()) do + local includedirs = table.join(pkg:get("sysincludedirs") or {}, pkg:get("includedirs") or {}) + table.join2(headerpaths, includedirs) + end + table.join2(headerpaths, target:get("includedirs")) + local p = find_file(file, headerpaths) + assert(p, "find <%s> not found!", file) + return p +end + +-- get stdmodules +function get_stdmodules(target) + return _compiler_support(target).get_stdmodules(target) +end + +-- get memcache +function memcache() + return _memcache.cache("cxxmodules") +end + +-- get localcache +function localcache() + return _localcache.cache("cxxmodules") +end + + +-- get stl headerunits cache directory +function stlheaderunits_cachedir(target, opt) + opt = opt or {} + local stlcachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "stl-headerunits") + if opt.mkdir and not os.isdir(stlcachedir) then + os.mkdir(stlcachedir) + os.mkdir(path.join(stlcachedir, "experimental")) + end + return stlcachedir +end +-- get stl modules cache directory +function stlmodules_cachedir(target, opt) + opt = opt or {} + local stlcachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "stl-modules") + if opt.mkdir and not os.isdir(stlcachedir) then + os.mkdir(stlcachedir) + end + return stlcachedir +end + +-- get headerunits cache directory +function headerunits_cachedir(target, opt) + opt = opt or {} + local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "headerunits") + if opt.mkdir and not os.isdir(cachedir) then + os.mkdir(cachedir) + end + return cachedir +end + +-- get modules cache directory +function modules_cachedir(target, opt) + opt = opt or {} + local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "modules") + if opt.mkdir and not os.isdir(cachedir) then + os.mkdir(cachedir) + end + return cachedir +end + +function get_modulehash(target, modulepath) + local key = path.directory(modulepath) .. target:name() + return hash.uuid(key):split("-", {plain = true})[1]:lower() +end + +function get_metafile(target, modulefile) + local outputdir = get_outputdir(target, modulefile) + return path.join(outputdir, path.filename(modulefile) .. ".meta-info") +end + +function get_outputdir(target, module) + local cachedir = module and modules_cachedir(target) or headerunits_cachedir(target) + local modulehash = get_modulehash(target, module.path or module) + local outputdir = path.join(cachedir, modulehash) + if not os.exists(outputdir) then + os.mkdir(outputdir) + end + return outputdir +end + +-- get name provide info and cpp sourcefile of a module +function get_provided_module(module) + + local name, provide, cppfile + if module.provides then + -- assume there that provides is only one, until we encounter the cases + -- "Some compiler may choose to implement the :private module partition as a separate module for lookup purposes, and if so, it should be indicated as a separate provides entry." + local length = 0 + for k, v in pairs(module.provides) do + length = length + 1 + name = k + provide = v + cppfile = provide.sourcefile + if length > 1 then + raise("multiple provides are not supported now!") + end + break + end + end + + return name, provide, cppfile +end + +function install_module_target(target) + local sourcebatch = target:sourcebatches()["c++.build.modules.install"] + if sourcebatch and sourcebatch.sourcefiles then + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = target:fileconfig(sourcefile) + local install = fileconfig and fileconfig.public or false + if install then + local modulehash = get_modulehash(target, sourcefile) + local prefixdir = path.join("modules", modulehash) + target:add("installfiles", sourcefile, {prefixdir = prefixdir}) + local metafile = get_metafile(target, sourcefile) + if os.exists(metafile) then + target:add("installfiles", metafile, {prefixdir = prefixdir}) + end + end + end + end +end + diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua similarity index 56% rename from xmake/rules/c++/modules/modules_support/common.lua rename to xmake/rules/c++/modules/modules_support/dependency_scanner.lua index d42d0f95814..830263d0766 100644 --- a/xmake/rules/c++/modules/modules_support/common.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -14,99 +14,39 @@ -- -- Copyright (C) 2015-present, TBOOX Open Source Group. -- --- @author Arthapz, ruki --- @file common.lua +-- @author ruki, Arthapz +-- @file dependency_scanner.lua -- -- imports import("core.base.json") -import("core.base.bytes") import("core.base.hashset") -import("core.project.config") -import("core.tool.compiler") -import("core.cache.memcache", {alias = "_memcache"}) -import("core.cache.localcache", {alias = "_localcache"}) -import("core.project.project") -import("lib.detect.find_file") -import("private.async.buildjobs") +import("compiler_support") import("stl_headers") --- get memcache -function memcache() - return _memcache.cache("cxxmodules") -end - --- get localcache -function localcache() - return _localcache.cache("cxxmodules") -end - --- get stl modules cache directory -function stlmodules_cachedir(target, opt) - opt = opt or {} - local stlcachedir = path.join(config.buildir(), "stlmodules", "cache", target:arch(), config.mode() or "release") - if opt.mkdir and not os.isdir(stlcachedir) then - os.mkdir(stlcachedir) - os.mkdir(path.join(stlcachedir, "experimental")) - end - return stlcachedir -end - --- get modules cache directory -function modules_cachedir(target, opt) - opt = opt or {} - local cachedir = path.join(target:autogendir(), "rules", "modules", "cache") - if opt.mkdir and not os.isdir(cachedir) then - os.mkdir(cachedir) - end - return cachedir -end - --- get headerunits info -function get_headerunits(target, sourcebatch, modules) - local headerunits - local stl_headerunits - for _, objectfile in ipairs(sourcebatch.objectfiles) do - local m = modules[objectfile] - if m then - for name, r in pairs(m.requires) do - if r.method ~= "by-name" then - local unittype = r.method == "include-angle" and ":angle" or ":quote" - if stl_headers.is_stl_header(name) then - stl_headerunits = stl_headerunits or {} - if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then - table.insert(stl_headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) - end - else - headerunits = headerunits or {} - if not table.find_if(headerunits, function(i, v) return v.name == name end) then - table.insert(headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) - end - end - end - end +function _dependency_scanner(target) + local cachekey = tostring(target) + local dependency_scanner = compiler_support.memcache():get2("dependency_scanner", cachekey) + if dependency_scanner == nil then + if target:has_tool("cxx", "clang", "clangxx") then + dependency_scanner = import("clang.dependency_scanner", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + dependency_scanner = import("gcc.dependency_scanner", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + dependency_scanner = import("msvc.dependency_scanner", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) end + compiler_support.memcache():set2("dependency_scanner", cachekey, dependency_scanner) end - return headerunits, stl_headerunits + return dependency_scanner end --- patch sourcebatch -function patch_sourcebatch(target, sourcebatch) - sourcebatch.sourcekind = "cxx" - sourcebatch.objectfiles = {} - sourcebatch.dependfiles = {} - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local objectfile = target:objectfile(sourcefile) - local dependfile = target:dependfile(objectfile) - table.insert(sourcebatch.objectfiles, objectfile) - table.insert(sourcebatch.dependfiles, dependfile) - end -end - -function parse_meta_info(target, metafile) +function _parse_meta_info(target, metafile) local metadata = json.loadfile(metafile) - if metadata._VENDOR_extension.xmake then - return metadata._VENDOR_extension.xmake.file, metadata._VENDOR_extension.xmake.name, metadata + if metadata.file and metadata.name then + return metadata.file, metadata.name, metadata end local filename = path.basename(metafile) @@ -133,130 +73,6 @@ function parse_meta_info(target, metafile) return filename, name, metadata end --- extract packages modules dependencies -function get_all_package_modules(target, modules, opt) - local package_modules - - -- parse all meta-info and append their informations to the package store - for _, package in pairs(target:pkgs()) do - package_modules = package_modules or {} - local modulesdir = path.join(package:installdir(), "modules") - local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) - for _, metafile in ipairs(metafiles) do - local modulefile, name, metadata = parse_meta_info(target, metafile) - package_modules[name] = { - file = path.join(modulesdir, modulefile), - metadata = metadata - } - end - end - return package_modules -end - --- cull unused modules -function cull_unused_modules(target, modules, package_modules_data) - local needed_modules = {} - -- append all target dependencies - for _, module in pairs(modules) do - if module.requires then - for required, _ in pairs(module.requires) do - table.insert(needed_modules, required) - end - end - end - - -- append all package dependencies - local culled - local module_names = table.keys(package_modules_data) - for _, name in ipairs(module_names) do - culled = culled or {} - if table.find(needed_modules, name) and package_modules_data[name] and not culled[name] then - culled[name] = package_modules_data[name] - - if culled[name].metadata.imports then - table.join2(needed_modules, culled[name].metadata.imports) - table.join2(module_names, culled[name].metadata.imports) - end - end - end - return culled -end - --- get modules support -function modules_support(target) - local cachekey = tostring(target) - local module_builder = memcache():get2("modules_support", cachekey) - if module_builder == nil then - if target:has_tool("cxx", "clang", "clangxx") then - module_builder = import("clang", {anonymous = true}) - elseif target:has_tool("cxx", "gcc", "gxx") then - module_builder = import("gcc", {anonymous = true}) - elseif target:has_tool("cxx", "cl") then - module_builder = import("msvc", {anonymous = true}) - else - local _, toolname = target:tool("cxx") - raise("compiler(%s): does not support c++ module!", toolname) - end - memcache():set2("modules_support", cachekey, module_builder) - end - return module_builder -end - --- get bmi extension -function bmi_extension(target) - return modules_support(target).get_bmi_extension() -end - --- has module extension? e.g. *.mpp, ... -function has_module_extension(sourcefile) - local modulexts = _g.modulexts - if modulexts == nil then - modulexts = hashset.of(".mpp", ".mxx", ".cppm", ".ixx") - _g.modulexts = modulexts - end - local extension = path.extension(sourcefile) - return modulexts:has(extension:lower()) -end - --- this target contains module files? -function contains_modules(target) - -- we can not use `"c++.build.modules.builder"`, because it contains sourcekind/cxx. - local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false - if not target_with_modules then - target_with_modules = target:policy("build.c++.modules") - end - if not target_with_modules then - for _, dep in ipairs(target:orderdeps()) do - local sourcebatches = dep:sourcebatches() - if sourcebatches["c++.build.modules"] then - target_with_modules = true - break - end - end - end - return target_with_modules -end - --- load module infos -function load_moduleinfos(target, sourcebatch) - local moduleinfos - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - if os.isfile(dependfile) then - local data = io.load(dependfile) - if data then - moduleinfos = moduleinfos or {} - local moduleinfo = json.decode(data.moduleinfo) - moduleinfo.sourcefile = sourcefile - if moduleinfo then - table.insert(moduleinfos, moduleinfo) - end - end - end - end - return moduleinfos -end - -- parse module dependency data --[[ { @@ -300,7 +116,7 @@ function _parse_dependencies_data(target, moduleinfos) -- try to find the compiled module path in outputs filed (MSVC doesn't generate compiled-module-path) if not bmifile then for _, output in ipairs(rule.outputs) do - if output:endswith(".ifc") or output:endswith(".pcm") or output:endswith(".bmi") then + if output:endswith(compiler_support.get_bmi_extension(target)) then bmifile = output break end @@ -308,11 +124,11 @@ function _parse_dependencies_data(target, moduleinfos) -- we didn't found the compiled module path, so we assume it if not bmifile then - local name = provide["logical-name"] .. bmi_extension(target) + local name = provide["logical-name"] .. compiler_support.get_bmi_extension(target) -- partition ":" character is invalid path character on windows -- @see https://github.com/xmake-io/xmake/issues/2954 name = name:replace(":", "-") - bmifile = path.join(get_outputdir(target, name), name) + bmifile = path.join(compiler_support.get_outputdir(target, name), name) end end m.provides[provide["logical-name"]] = { @@ -354,6 +170,7 @@ function _parse_dependencies_data(target, moduleinfos) return modules end + -- check circular dependencies for the given module function _check_circular_dependencies_of_module(name, moduledeps, modulesources, depspath) for _, dep in ipairs(moduledeps[name]) do @@ -418,7 +235,7 @@ function _topological_sort_visit(node, nodes, modules, output) if not n.tempmarked then local m2 = modules[n.objectfile] if m2 then - for name, provide in pairs(m1.provides) do + for name, _ in pairs(m1.provides) do if m2.requires and m2.requires[name] then _topological_sort_visit(n, nodes, modules, output) end @@ -448,43 +265,80 @@ function _topological_sort_get_first_unmarked_node(nodes) end end --- topological sort -function sort_modules_by_dependencies(objectfiles, modules) - local output = {} - local nodes = {} - for _, objectfile in ipairs(objectfiles) do - local m = modules[objectfile] - if m then - table.insert(nodes, {marked = false, tempmarked = false, objectfile = objectfile}) - end - end - while _topological_sort_has_node_without_mark(nodes) do - local node = _topological_sort_get_first_unmarked_node(nodes) - _topological_sort_visit(node, nodes, modules, output) +function _fill_needed_module(target, modules, module) + local needed_modules = {} + + for required_name, required_module in pairs(module.requires) do + table.insert(needed_modules, required_name) + table.join2(needed_modules, _fill_needed_module(target, modules, required_module)) end - return output + + return needed_modules end -function find_quote_header_file(target, sourcefile, file) - local p = path.join(path.directory(path.absolute(sourcefile, project.directory())), file) - assert(os.isfile(p)) - return p +function _get_package_modules(target, package, opt) + local package_modules + + local modulesdir = path.join(package:installdir(), "modules") + local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) + for _, metafile in ipairs(metafiles) do + package_modules = package_modules or {} + local modulefile, name, metadata = _parse_meta_info(target, metafile) + package_modules[name] = {file = path.join(modulesdir, modulefile), metadata = metadata} + end + + return package_modules end -function find_angle_header_file(target, file) - local headerpaths = modules_support(target).toolchain_includedirs(target) - for _, dep in ipairs(target:orderdeps()) do - local includedirs = table.join(dep:get("sysincludedirs") or {}, dep:get("includedirs") or {}) - table.join2(headerpaths, includedirs) +-- get module dependencies +function get_module_dependencies(target, sourcebatch, opt) + local cachekey = target:name() .. "/" .. sourcebatch.rulename + local modules = compiler_support.memcache():get2("modules", cachekey) + if modules == nil or opt.regenerate then + modules = compiler_support.localcache():get2("modules", cachekey) + opt.progress = opt.progress or 0 + local changed = _dependency_scanner(target).generate_dependencies(target, sourcebatch, opt) + if changed or modules == nil then + local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) + modules = _parse_dependencies_data(target, moduleinfos) + if modules then + _check_circular_dependencies(modules) + end + modules = cull_unused_modules(target, modules) + compiler_support.localcache():set2("modules", cachekey, modules) + compiler_support.localcache():save() + end + compiler_support.memcache():set2("modules", cachekey, modules) end - for _, pkg in pairs(target:pkgs()) do - local includedirs = table.join(pkg:get("sysincludedirs") or {}, pkg:get("includedirs") or {}) - table.join2(headerpaths, includedirs) + return modules +end + +-- get headerunits info +function get_headerunits(target, sourcebatch, modules) + local headerunits + local stl_headerunits + for _, objectfile in ipairs(sourcebatch.objectfiles) do + local m = modules[objectfile] + if m then + for name, r in pairs(m.requires) do + if r.method ~= "by-name" then + local unittype = r.method == "include-angle" and ":angle" or ":quote" + if stl_headers.is_stl_header(name) then + stl_headerunits = stl_headerunits or {} + if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then + table.insert(stl_headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) + end + else + headerunits = headerunits or {} + if not table.find_if(headerunits, function(i, v) return v.name == name end) then + table.insert(headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) + end + end + end + end + end end - table.join2(headerpaths, target:get("includedirs")) - local p = find_file(file, headerpaths) - assert(p, "find <%s> not found!", file) - return p + return headerunits, stl_headerunits end -- https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html @@ -545,7 +399,7 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess local module_depname = line:match("import%s+(.+)%s*;") -- we need to parse module interface dep in cxx/impl_unit.cpp, e.g. hello.mpp and hello_impl.cpp -- @see https://github.com/xmake-io/xmake/pull/2664#issuecomment-1213167314 - if not module_depname and not has_module_extension(sourcefile) then + if not module_depname and not compiler_support.has_module_extension(sourcefile) then module_depname = module_name_private end if module_depname and not module_deps_set:has(module_depname) then @@ -560,12 +414,12 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess module_depname = module_depname:sub(2, -2) module_dep["lookup-method"] = "include-quote" module_dep["unique-on-source-path"] = true - module_dep["source-path"] = find_quote_header_file(target, sourcefile, module_depname) + module_dep["source-path"] = compiler_support.find_quote_header_file(target, sourcefile, module_depname) elseif module_depname:startswith("<") then module_depname = module_depname:sub(2, -2) module_dep["lookup-method"] = "include-angle" module_dep["unique-on-source-path"] = true - module_dep["source-path"] = find_angle_header_file(target, module_depname) + module_dep["source-path"] = compiler_support.find_angle_header_file(target, module_depname) end module_dep["logical-name"] = module_depname table.insert(module_deps, module_dep) @@ -575,13 +429,13 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess end if module_name_export or internal then - local outputdir = get_outputdir(target, sourcefile) + local outputdir = compiler_support.get_outputdir(target, sourcefile) local provide = {} provide["logical-name"] = module_name_export or module_name_private provide["source-path"] = sourcefile provide["is-interface"] = not internal - provide["compiled-module-path"] = path.join(outputdir, (module_name_export or module_name_private) .. bmi_extension(target)) + provide["compiled-module-path"] = path.join(outputdir, (module_name_export or module_name_private) .. compiler_support.get_bmi_extension(target)) rule.provides = {} table.insert(rule.provides, provide) @@ -593,145 +447,104 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess io.writefile(jsonfile, jsondata) end --- get module dependencies -function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:name() .. "/" .. sourcebatch.rulename - local modules = memcache():get2("modules", cachekey) - if modules == nil or opt.regenerate then - modules = localcache():get2("modules", cachekey) - opt.progress = opt.progress or 0 - local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt) - if changed or modules == nil then - local moduleinfos = load_moduleinfos(target, sourcebatch) - modules = _parse_dependencies_data(target, moduleinfos) - modules = table.join(modules or {}, modules_support(target).get_stdmodules(target)) - if modules then - _check_circular_dependencies(modules) - end - localcache():set2("modules", cachekey, modules) - localcache():save() - end - memcache():set2("modules", cachekey, modules) - end - return modules -end - --- generate headerunits for batchcmds -function generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local user_headerunits, stl_headerunits = get_headerunits(target, sourcebatch, modules) - -- build stl header units as other headerunits may need them - if stl_headerunits then - modules_support(target).generate_stl_headerunits_for_batchcmds(target, batchcmds, stl_headerunits, opt) - end - if user_headerunits then - modules_support(target).generate_user_headerunits_for_batchcmds(target, batchcmds, user_headerunits, opt) - end -end - --- build batchjobs for modules -function build_batchjobs_for_modules(modules, batchjobs, rootjob) - return buildjobs(modules, batchjobs, rootjob) -end - --- build modules for batchjobs -function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules) - modules_support(target).build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) -end +-- extract packages modules dependencies +function get_all_packages_modules(target, opt) + local packages_modules --- build modules for batchcmds -function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules) - modules_support(target).build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) -end + -- parse all meta-info and append their informations to the package store + local packages = target:pkgs() or {} --- append headerunits objectfiles to link -function append_dependency_objectfiles(target) - local cachekey = target:name() .. "dependency_objectfiles" - local cache = localcache():get(cachekey) - if cache then - if target:is_binary() then - target:add("ldflags", cache, {force = true, expand = false}) - elseif target:is_static() then - target:add("arflags", cache, {force = true, expand = false}) - elseif target:is_shared() then - target:add("shflags", cache, {force = true, expand = false}) - end + for _, deps in pairs(target:orderdeps()) do + table.join2(packages, deps:pkgs()) end -end --- generate meta module informations for package / other buildsystems import --- based on https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2473r1.pdf --- --- e.g --- { --- "include_paths": ["foo/", "bar/"] --- "definitions": ["FOO=BAR"] --- "imports": ["std", "bar"] --- "_VENDOR_extension": {} --- } -function generate_meta_module_info(target, name, sourcefile, requires) - local module_metadata = {} - - -- add include paths - module_metadata.include_paths = table.wrap(target:get("includedirs")) or {} - for _, deps in ipairs(target:orderdeps()) do - table.join2(module_metadata.include_paths, deps:get("includedirs") or {}) + for _, package in pairs(packages) do + local package_modules = _get_package_modules(target, package, opt) + if package_modules then + packages_modules = packages_modules or {} + table.join2(packages_modules, package_modules) + end end - -- add definitions - module_metadata.definitions = table.wrap(target:get("defines")) or {} - for _, deps in ipairs(target:orderdeps()) do - table.join2(module_metadata.definitions, deps:get("defines") or {}) - end + return packages_modules +end - -- add imports - if requires then - for name, _ in pairs(requires) do - module_metadata.imports = module_metadata.imports or {} - table.append(module_metadata.imports, name) +-- topological sort +function sort_modules_by_dependencies(objectfiles, modules) + local output = {} + local nodes = {} + for _, objectfile in ipairs(objectfiles) do + local m = modules[objectfile] + if m then + table.insert(nodes, {marked = false, tempmarked = false, objectfile = objectfile}) end end - - local modulehash = get_modulehash(target, sourcefile) - module_metadata._VENDOR_extension = { xmake = { name = name, file = path.join(modulehash, path.filename(sourcefile)) }} - return module_metadata + while _topological_sort_has_node_without_mark(nodes) do + local node = _topological_sort_get_first_unmarked_node(nodes) + _topological_sort_visit(node, nodes, modules, output) + end + return output end -function install_module_target(target) - local sourcebatch = target:sourcebatches()["c++.build.modules.install"] - if sourcebatch and sourcebatch.sourcefiles then - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local install = (fileconfig and not fileconfig.install) and false or true - if install then - local modulehash = get_modulehash(target, sourcefile) - prefixdir = path.join("modules", modulehash) - target:add("installfiles", sourcefile, {prefixdir = prefixdir}) - local metafile = get_metafile(target, sourcefile) - if os.exists(metafile) then - target:add("installfiles", metafile, {prefixdir = prefixdir}) +-- get source modulefile for external target deps +function get_targetdeps_modules(target) + local sourcefiles + for _, dep in ipairs(target:orderdeps()) do + local sourcebatch = dep:sourcebatches()["c++.build.modules.builder"] + if sourcebatch and sourcebatch.sourcefiles then + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = dep:fileconfig(sourcefile) + local public = (fileconfig and fileconfig.public and not fileconfig.external) or false + if public then + sourcefiles = sourcefiles or {} + table.insert(sourcefiles, sourcefile) + target:fileconfig_add(sourcefile, {external = true}) end end end end + return sourcefiles end -function get_modulehash(target, modulepath) - local key = path.directory(modulepath) .. target:name() - return hash.uuid(key):split("-", {plain = true})[1]:lower() -end +-- cull unused packages modules +-- removed named module not used in the translation units +-- when building a library we only cull external modules because we need module objectfiles to be linked inside the library +-- on an executable we cull explicitly referenced module +function cull_unused_modules(target, modules) -function get_metafile(target, modulefile) - local outputdir = get_outputdir(target, modulefile) - return path.join(outputdir, path.filename(modulefile) .. ".meta-info") -end + local cull_all_modules = target:kind() == "executable" + + local needed_modules = {} + for _, module in pairs(modules) do + local fileconfig = target:fileconfig(module.sourcefile) + local external = fileconfig and fileconfig.external + if not (cull_all_modules and external) then + goto CONTINUE + end -function get_outputdir(target, module) - local cachedir = modules_cachedir(target) - local modulehash = get_modulehash(target, module.path or module) - local outputdir = path.join(cachedir, modulehash) - if not os.exists(outputdir) then - os.mkdir(outputdir) + if module.provides and module.requires then + table.join2(needed_modules, _fill_needed_module(target, modules, module)) + end + + ::CONTINUE:: + end + + local culled = {} + for objectfile, module in pairs(modules) do + -- if cull_all_modules and modules.provides then + -- local name,_,_ = compiler_support.get_provided_module(module) + -- if module.requires then + -- for required, _ in pairs(module.requires) do + -- table.insert(needed_modules, required) + -- end + -- end + -- if table.find(needed_modules, name) then + -- culled[objectfile] = module + -- end + -- else + culled[objectfile] = module + -- end end - return outputdir + + return culled end diff --git a/xmake/rules/c++/modules/modules_support/gcc.lua b/xmake/rules/c++/modules/modules_support/gcc.lua deleted file mode 100644 index 6cc5c29d6ad..00000000000 --- a/xmake/rules/c++/modules/modules_support/gcc.lua +++ /dev/null @@ -1,622 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file gcc.lua --- - --- imports -import("core.base.option") -import("core.base.json") -import("core.tool.compiler") -import("core.project.project") -import("core.project.depend") -import("core.project.config") -import("utils.progress") -import("private.action.build.object", {alias = "objectbuilder"}) -import("common") - --- get and create the path of module mapper -function _get_module_mapper(target) - local mapper_file = path.join(config.buildir(), target:name(), "mapper.txt") - if not os.isfile(mapper_file) then - io.writefile(mapper_file, "") - end - return mapper_file -end - --- add a module or header unit into the mapper --- --- e.g --- /usr/include/c++/11/iostream build/.gens/stl_headerunit/linux/x86_64/release/stlmodules/cache/iostream.gcm --- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm --- -function _add_module_to_mapper(file, modulepath, bmi) - for line in io.lines(file) do - if line:startswith(modulepath .. " ") then - return false - end - end - local f = io.open(file, "a") - f:print("%s %s", modulepath, bmi) - f:close() - return true -end - --- get a module from mapper -function _get_module_from_mapper(file, module) - for line in io.lines(file) do - if line:startswith(module .. " ") then - return line:split(" ", {plain = true}) - end - end -end - --- load module support for the current target -function load(target) - local modulesflag = get_modulesflag(target) - local modulemapperflag = get_modulemapperflag(target) - if os.isfile(_get_module_mapper(target)) then - os.rm(_get_module_mapper(target)) - end - target:add("cxxflags", {modulesflag, modulemapperflag .. path.translate(_get_module_mapper(target))}, {force = true, expand = false}) - -- fix cxxabi issue - -- @see https://github.com/xmake-io/xmake/issues/2716#issuecomment-1225057760 - -- https://github.com/xmake-io/xmake/issues/3855 - if target:policy("build.c++.gcc.modules.cxx11abi") then - target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=1") - else - target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=0") - end -end - --- get includedirs for stl headers --- --- $ echo '#include ' | gcc -x c++ -E - | grep '/vector"' --- # 1 "/usr/include/c++/11/vector" 1 3 --- # 58 "/usr/include/c++/11/vector" 3 --- # 59 "/usr/include/c++/11/vector" 3 --- -function _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) - local tmpfile = os.tmpfile() .. ".cc" - io.writefile(tmpfile, "#include ") - local result = try {function () return os.iorunv(gcc, {"-E", "-x", "c++", tmpfile}) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if line:startswith("#") and line:find("/vector\"", 1, true) then - local includedir = line:match("\"(.+)/vector\"") - if includedir and os.isdir(includedir) then - table.insert(includedirs, path.normalize(includedir)) - break - end - end - end - end - os.tryrm(tmpfile) -end - --- do compile for batchcmds --- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) -end - --- build module file -function _build_modulefile(target, sourcefile, opt) - local objectfile = opt.objectfile - local dependfile = opt.dependfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = table.join("-x", "c++", compinst:compflags({sourcefile = sourcefile, target = target})) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile, {target = target}) or {}) - - -- need build this object? - local dryrun = option.get("dry-run") - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(objectfile) and os.mtime(dependfile) or 0 - if not dryrun and not depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return - end - - -- trace - progress.show(opt.progress, "${color.build.object}compiling.module.$(mode) %s", opt.name) - vprint(compinst:compcmd(sourcefile, objectfile, {compflags = compflags, rawargs = true})) - - if not dryrun then - - -- do compile - dependinfo.files = {} - assert(compinst:compile(sourcefile, objectfile, {dependinfo = dependinfo, compflags = compflags})) - - -- update files and values to the dependent file - dependinfo.values = depvalues - table.join2(dependinfo.files, sourcefile) - depend.save(dependinfo, dependfile) - end -end - --- provide toolchain include directories for stl headerunit when p1689 is not supported -function toolchain_includedirs(target) - local includedirs = _g.includedirs - if includedirs == nil then - includedirs = {} - local gcc, toolname = target:tool("cxx") - assert(toolname == "gcc") - _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) - local _, result = try {function () return os.iorunv(gcc, {"-E", "-Wp,-v", "-xc", os.nuldev()}) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if os.isdir(line) then - table.insert(includedirs, path.normalize(line)) - elseif line:startswith("End") then - break - end - end - end - _g.includedirs = includedirs - end - return includedirs -end - --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) - local compinst = target:compiler("cxx") - local common_args = {"-E", "-x", "c++"} - local depformatflag = get_depflag(target, "p1689r5") or get_depflag(target, "trtbd") - local depfileflag = get_depfileflag(target) - local depoutputflag = get_depoutputflag(target) - local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.object}generating.module.deps %s", sourcefile) - end - - local outputdir = common.get_outputdir(target, sourcefile) - local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) - if depformatflag and depfileflag and depoutputflag and not target:policy("build.c++.gcc.fallbackscanner") then - local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) - local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) - local args = {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depformatflag, depfileflag .. jsonfile, depoutputflag .. target:objectfile(sourcefile), "-o", ifile} - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - -- module mapper flag force gcc to check the imports but this is not wanted at this stage - local modulemapperflag = get_modulemapperflag(target) .. path.translate(_get_module_mapper(target)) - table.remove(compflags, table.unpack(table.find(compflags, modulemapperflag))) - os.vrunv(compinst:program(), table.join(compflags, common_args, args)) - os.rm(ifile) - os.rm(dfile) - else - common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = file, target = target}) - -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation - table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(common_args, compflags, {file, "-o", ifile})) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true - - local dependinfo = io.readfile(jsonfile) - return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end - return changed -end - --- generate target stl header units for batchjobs -function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - local mapper_file = _get_module_mapper(target) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local modulemapperflag = get_modulemapperflag(target) - - -- build headerunits - local projectdir = os.projectdir() - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - if not os.isfile(bmifile) then - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local args = {"-c", "-x", "c++-system-header", headerunit.name} - local flags = table.join(compinst:compflags({target = target}), args) - -- we need to support reading and writing mapperfile in parallel, otherwise it will be broken - -- @see tests/c++/modules/stl_headerunit_cpp_only - local mapper_file_tmp = os.tmpfile() - os.cp(mapper_file, mapper_file_tmp) - _add_module_to_mapper(mapper_file_tmp, headerunit.path, path.absolute(bmifile, projectdir)) - for idx, flag in ipairs(flags) do - if flag:startswith(modulemapperflag) then - flags[idx] = modulemapperflag .. mapper_file_tmp - break - end - end - os.vrunv(compinst:program(), flags) - _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir)) - os.tryrm(mapper_file_tmp) - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - else - _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir)) - end - end -end - --- generate target stl header units for batchcmds -function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local mapper_file = _get_module_mapper(target) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - - -- build headerunits - local projectdir = os.projectdir() - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - if not os.isfile(bmifile) then - local flags = {"-c", "-x", "c++-system-header", headerunit.name} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - end - batchcmds:add_depfiles(headerunit.path) - _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir)) - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) -end - --- generate target user header units for batchjobs -function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - local mapper_file = _get_module_mapper(target) - - -- build headerunits - local projectdir = os.projectdir() - for _, headerunit in ipairs(headerunits) do - local headerunit_path - if headerunit.type == ":quote" then - headerunit_path = path.join(".", path.relative(headerunit.path, projectdir)) - elseif headerunit.type == ":angle" then - -- if path is relative then its a subtarget path - headerunit_path = path.is_absolute(headerunit.path) and headerunit.path or path.join(".", headerunit.path) - end - local objectfile = target:objectfile(headerunit_path) - local outputdir = common.get_outputdir(target, headerunit.path) - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local objectdir = path.directory(objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - - -- generate headerunit - local args = { "-c" } - if headerunit.type == ":quote" then - local includedir - local p = headerunit.path - if p:endswith(headerunit.name) then - includedir = p:sub(1, #p - #headerunit.name - 1) - else - includedir = path.directory(p) - end - if path.is_absolute(includedir) then - includedir = path.relative(includedir, projectdir) - end - table.join2(args, { "-I", includedir, "-x", "c++-user-header", headerunit.name }) - elseif headerunit.type == ":angle" then - table.join2(args, { "-x", "c++-system-header", headerunit.name }) - end - os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args)) - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - _add_module_to_mapper(mapper_file, headerunit_path, path.absolute(bmifile, projectdir)) - end -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local mapper_file = _get_module_mapper(target) - - -- build headerunits - local projectdir = os.projectdir() - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local flags = {"-c"} - local headerunit_path - if headerunit.type == ":quote" then - local includedir - local p = headerunit.path - if p:endswith(headerunit.name) then - includedir = p:sub(1, #p - #headerunit.name - 1) - else - includedir = path.directory(p) - end - if path.is_absolute(includedir) then - includedir = path.relative(includedir, projectdir) - end - table.join2(flags, {"-I", path(includedir), "-x", "c++-user-header", headerunit.name}) - headerunit_path = path.join(".", path.relative(headerunit.path, projectdir)) - elseif headerunit.type == ":angle" then - table.join2(flags, {"-x", "c++-system-header", headerunit.name}) - -- if path is relative then its a subtarget path - headerunit_path = path.is_absolute(headerunit.path) and headerunit.path or path.join(".", headerunit.path) - end - local objectfile = target:objectfile(headerunit_path) - local outputdir = common.get_outputdir(target, headerunit.path) - batchcmds:mkdir(outputdir) - - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = (outputdir and path.join(outputdir, bmifilename) or bmifilename) - batchcmds:mkdir(path.directory(objectfile)) - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - batchcmds:add_depfiles(headerunit.path) - - _add_module_to_mapper(mapper_file, headerunit_path, path.absolute(bmifile, projectdir)) - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) -end - --- build module files for batchjobs -function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) - local mapper_file = _get_module_mapper(target) - - -- build modules - local projectdir = os.projectdir() - local modulesjobs = {} - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - -- assume there that provides is only one, until we encounter the case - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local moduleinfo = table.copy(provide) or {} - local dependfile = (provide and provide.bmi) and target:dependfile(provide.bmi) or target:dependfile(objectfile) - - if provide then - local fileconfig = target:fileconfig(cppfile) - if fileconfig and fileconfig.install then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = common.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}generating.module.metadata %s", name) - local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - end - end - - table.join2(moduleinfo, { - name = name or cppfile, - deps = table.keys(module.requires or {}), - sourcefile = cppfile, - job = batchjobs:newjob(name or cppfile, function(index, total) - -- append dependencies module now to ensures deps modulemap is filled - for required, _ in pairs(module.requires) do - local m - for _, dep in ipairs(target:orderdeps()) do - m = _get_module_from_mapper(_get_module_mapper(dep), required) - if m then - break - end - end - if m then - _add_module_to_mapper(mapper_file, m[1], m[2]) - break - end - end - - if provide or common.has_module_extension(cppfile) then - if not common.memcache():get2(name or cppfile, "compiling") then - if name and module.external then - common.memcache():set2(name or cppfile, "compiling", true) - end - _build_modulefile(target, cppfile, { - objectfile = objectfile, - dependfile = dependfile, - name = name or cppfile, - progress = (index * 100) / total}) - end - target:add("objectfiles", objectfile) - end - end)}) - if provide then - _add_module_to_mapper(mapper_file, name, path.absolute(provide.bmi, projectdir)) - end - modulesjobs[name or cppfile] = moduleinfo - end - end - - -- build batchjobs for modules - common.build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) -end - --- build module files for batchcmds -function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) - local modulemapperflag = get_modulemapperflag(target) - local mapper_file = _get_module_mapper(target) - - -- build modules - local projectdir = os.projectdir() - local depmtime = 0 - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - -- append dependencies module now to ensures deps modulemap is filled - for required, _ in pairs(module.requires) do - local m - for _, dep in ipairs(target:orderdeps()) do - m = _get_module_from_mapper(_get_module_mapper(dep), required) - if m then - break - end - end - if m then - _add_module_to_mapper(mapper_file, m[1], m[2]) - break - end - end - if provide or common.has_module_extension(cppfile) then - local flags = {"-x", "c++", "-c", path(cppfile), "-o", path(objectfile)} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.module.$(mode) %s", name or cppfile) - batchcmds:mkdir(path.directory(objectfile)) - _batchcmds_compile(batchcmds, target, flags, cppfile) - batchcmds:add_depfiles(cppfile) - target:add("objectfiles", objectfile) - _add_module_to_mapper(mapper_file, name, path.absolute(provide.bmi, projectdir)) - end - depmtime = math.max(depmtime, os.mtime(provide and provide.bmi or objectfile)) - end - end - batchcmds:set_depmtime(depmtime) -end - --- not supported atm -function get_stdmodules(target) - local modules = {} - return modules -end - -function get_bmi_extension() - return ".gcm" -end - -function get_modulesflag(target) - local modulesflag = _g.modulesflag - if modulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "gcc_modules_ts"}) then - modulesflag = "-fmodules-ts" - end - assert(modulesflag, "compiler(gcc): does not support c++ module!") - _g.modulesflag = modulesflag or false - end - return modulesflag or nil -end - -function get_modulemapperflag(target) - local modulemapperflag = _g.modulemapperflag - if modulemapperflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodule-mapper=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_module_mapper"}) then - modulemapperflag = "-fmodule-mapper=" - end - assert(modulemapperflag, "compiler(gcc): does not support c++ module!") - _g.modulemapperflag = modulemapperflag or false - end - return modulemapperflag or nil -end - -function get_depflag(target, format) - local depflag = _g.depflag - if depflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fdep-format=" .. format, "cxxflags", {flagskey = "gcc_dep_format"}) then - depflag = "-fdep-format=" .. format - end - _g.depflag = depflag or false - end - return depflag or nil -end - -function get_depfileflag(target) - local depfileflag = _g.depfileflag - if depfileflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fdep-file=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_dep_file", - on_check = function (ok, errors) - if errors:find("cc1plus: error: to generate dependencies") then - ok = true - end - return ok, errors - end}) then - depfileflag = "-fdep-file=" - end - _g.depfileflag = depfileflag or false - end - return depfileflag or nil -end - -function get_depoutputflag(target) - local depoutputflag = _g.depoutputflag - if depoutputflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fdep-output=" .. os.tmpfile() .. ".o", "cxxflags", {flagskey = "gcc_dep_output", - on_check = function (ok, errors) - if errors:find("cc1plus: error: to generate dependencies") then - ok = true - end - return ok, errors - end}) then - depoutputflag = "-fdep-output=" - end - _g.depoutputflag = depoutputflag or false - end - return depoutputflag or nil -end - -function get_cppversionflag(target) - local cppversionflag = _g.cppversionflag - if cppversionflag == nil then - local compinst = target:compiler("cxx") - local flags = compinst:compflags({target = target}) - cppversionflag = table.find_if(flags, function(v) string.startswith(v, "-std=c++") end) or "-std=c++20" - _g.cppversionflag = cppversionflag - end - return cppversionflag or nil -end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua new file mode 100644 index 00000000000..f0599c7f2ab --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -0,0 +1,370 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file gcc/builder.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("core.base.semver") +import("utils.progress") +import("private.action.build.object", {alias = "objectbuilder"}) +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("compiler_support") +import(".builder", {inherit = true}) + +-- get flags for building a module +function _make_modulebuildflags(target, opt) + return {"-x", "c++", "-c"} +end + +-- get flags for building a headerunit +function _make_headerunitflags(target, headerunit, headerunit_mapper) + + -- get flags + local module_headerflag = compiler_support.get_moduleheaderflag(target) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + assert(module_headerflag and module_onlyflag, "compiler(gcc): does not support c++ header units!") + + local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(path.normalize(headerunit.path))} or {} + + local headertype = (headerunit.type == ":angle") and "system" or "user" + + local flags = table.join(local_directory, {module_mapperflag .. headerunit_mapper, + module_headerflag .. headertype, + module_onlyflag, + "-xc++-header"}) + return flags +end + +-- do compile +function _compile(target, flags, sourcefile, outputfile) + + local dryrun = option.get("dry-run") + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, flags) + + -- trace + if option.get("verbose") then + print(compinst:compcmd(sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) + end + + if not dryrun then + -- do compile + assert(compinst:compile(sourcefile, outputfile, {target = target, compflags = flags})) + end +end + +-- do compile for batchcmds +-- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx +function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) + + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, sourcefile}) + batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) +end + +function _module_map_cachekey(target) + local mode = config.mode() + return target:name() .. "module_mapper" .. (mode or "") +end + +-- generate a module mapper file for build a headerunit +function _generate_headerunit_modulemapper_file(module) + + local path = os.tmpfile() + local mapper_file = io.open(path, "wb") + + mapper_file:write("root " .. os.projectdir()) + mapper_file:write("\n") + + mapper_file:write(mapper_file, module.name:replace("\\", "/") .. " " .. module.bmifile:replace("\\", "/")) + mapper_file:write("\n") + + mapper_file:close() + + return path + +end + +function _get_maplines(target, module) + local maplines = {} + + local m_name, m = compiler_support.get_provided_module(module) + if m then + table.insert(maplines, m_name .. " " .. compiler_support.get_bmi_path(m.bmi)) + end + + for required, _ in table.orderpairs(module.requires) do + local dep_module + local dep_target + for _, dep in ipairs(target:orderdeps()) do + dep_module = get_from_target_mapper(dep, required) + if dep_module then + dep_target = dep + break + end + end + + -- if not in target dep + if not dep_module then + dep_module = get_from_target_mapper(target, required) + if dep_module then + dep_target = target + end + end + + assert(dep_module, "module dependency %s required for %s not found", required, m_name) + + local bmifile = dep_module.bmi + local mapline + -- aliased headerunit + if dep_module.aliasof then + local aliased = get_from_target_mapper(target, dep_module.aliasof) + bmifile = aliased.bmi + mapline = dep_module.headerunit.path:replace("\\", "/") .. " " .. bmifile:replace("\\", "/") + -- headerunit + elseif dep_module.headerunit then + mapline = dep_module.headerunit.path:replace("\\", "/") .. " " .. bmifile:replace("\\", "/") + -- named module + else + mapline = required .. " " .. bmifile:replace("\\", "/") + end + table.insert(maplines, mapline) + + -- append deps + if dep_module.opt and dep_module.opt.deps then + local deps = _get_maplines(dep_target, { name = dep_module.name, bmi = bmifile, requires = dep_module.opt.deps }) + table.join2(maplines, deps) + end + end + + -- remove duplicates + return table.unique(maplines) +end + +-- generate a module mapper file for build a module +-- e.g +-- /usr/include/c++/11/iostream build/.gens/stl_headerunit/linux/x86_64/release/stlmodules/cache/iostream.gcm +-- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm +-- +function _generate_modulemapper_file(target, module) + + local maplines = _get_maplines(target, module) + + local path = os.tmpfile() + local mapper_file = io.open(path, "wb") + + mapper_file:write("root " .. os.projectdir():replace("\\", "/")) + mapper_file:write("\n") + + for _, mapline in ipairs(maplines) do + mapper_file:write(mapline) + mapper_file:write("\n") + end + + mapper_file:close() + + return path +end + +-- populate module map +function populate_module_map(target, modules) + + -- append all modules + for _, module in pairs(modules) do + local name, provide = compiler_support.get_provided_module(module) + if provide then + add_module_to_target_mapper(target, name, provide.sourcefile, compiler_support.get_bmi_path(provide.bmi)) + end + end + + -- then update their deps + for _, module in pairs(modules) do + local name, provide = compiler_support.get_provided_module(module) + if provide then + local bmifile = compiler_support.get_bmi_path(provide.bmi) + add_module_to_target_mapper(target, name, provide.sourcefile, bmifile, {deps = module.requires}) + end + end +end + +-- get defines for a module +function get_module_required_defines(target, sourcefile) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local defines + + for _, flag in ipairs(compflags) do + if flag:startswith("-D") then + defines = defines or {} + table.insert(defines, flag:sub(3)) + end + end + + return defines +end + +-- build module file for batchjobs +function make_module_build_job(target, batchjobs, job_name, deps, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + return { + name = job_name, + deps = deps, + sourcefile = opt.cppfile, + job = batchjobs:newjob(name or opt.cppfile, function(index, total) + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(module_mapper)) + end + + local flags = _make_modulebuildflags(target, opt) + _compile(target, flags, opt.cppfile, opt.objectfile) + os.tryrm(module_mapper) + end + end + table.insert(dependinfo.files, opt.cppfile) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build module file for batchcmds +function make_module_build_cmds(target, batchcmds, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + if opt.build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + if option.get("diagnosis") then + batchcmds:print("mapper file: %s", io.readfile(module_mapper)) + end + batchcmds:mkdir(path.directory(opt.objectfile)) + + _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile, opt.objectfile) + end + end + + batchcmds:add_depfiles(opt.cppfile) + + return os.mtime(opt.objectfile) +end + +-- build headerunit file for batchjobs +function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) + + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) + if not already_exists then + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + if option.get("diagnosis") then + print("mapper file:\n%s", io.readfile(headerunit_mapper)) + end + _compile(target, _make_headerunitflags(target, headerunit, headerunit_mapper, opt), path.translate(path.filename(headerunit.name)), bmifile) + os.tryrm(headerunit_mapper) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} + end +end + +-- build headerunit file for batchcmds +function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) + + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + batchcmds:mkdir(outputdir) + + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + add_headerunit_to_target_mapper(target, _headerunit, bmifile) + + if opt.build then + local name = headerunit.unique and headerunit.name or headerunit.path + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + if option.get("diagnosis") then + batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) + end + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headerunit_mapper), bmifile) + end + + batchcmds:rm(headerunit_mapper) + batchcmds:add_depfiles(headerunit.path) + return os.mtime(bmifile) +end + diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua new file mode 100644 index 00000000000..7ad8a65d586 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -0,0 +1,225 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file gcc/compiler_support.lua +-- + +-- imports +import("core.base.semver") +import("core.project.config") +import("lib.detect.find_tool") +import(".compiler_support", {inherit = true}) + +-- get includedirs for stl headers +-- +-- $ echo '#include ' | gcc -x c++ -E - | grep '/vector"' +-- # 1 "/usr/include/c++/11/vector" 1 3 +-- # 58 "/usr/include/c++/11/vector" 3 +-- # 59 "/usr/include/c++/11/vector" 3 +-- +function _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) + local tmpfile = os.tmpfile() .. ".cc" + io.writefile(tmpfile, "#include ") + local result = try {function () return os.iorunv(gcc, {"-E", "-x", "c++", tmpfile}) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if line:startswith("#") and line:find("/vector\"", 1, true) then + local includedir = line:match("\"(.+)/vector\"") + if includedir and os.isdir(includedir) then + table.insert(includedirs, path.normalize(includedir)) + break + end + end + end + end + os.tryrm(tmpfile) +end + +-- load module support for the current target +function load(target) + local modulesflag = get_modulesflag(target) + target:add("cxxflags", modulesflag, {force = true, expand = false}) + + -- fix cxxabi issue + -- @see https://github.com/xmake-io/xmake/issues/2716#issuecomment-1225057760 + -- https://github.com/xmake-io/xmake/issues/3855 + if target:policy("build.c++.gcc.modules.cxx11abi") then + target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=1") + else + target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=0") + end +end + +-- provide toolchain include directories for stl headerunit when p1689 is not supported +function toolchain_includedirs(target) + local includedirs = _g.includedirs + if includedirs == nil then + includedirs = {} + local gcc, toolname = target:tool("cxx") + assert(toolname == "gcc" or toolname == "gxx") + _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) + local _, result = try {function () return os.iorunv(gcc, {"-E", "-Wp,-v", "-xc", os.nuldev()}) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if os.isdir(line) then + table.insert(includedirs, path.normalize(line)) + elseif line:startswith("End") then + break + end + end + end + _g.includedirs = includedirs + end + return includedirs +end + +function get_target_module_mapperpath(target) + local path = path.join(modules_cachedir(target, {mkdir = true}), "..", "mapper.txt") + if not os.isfile(path) then + io.writefile(path, "") + end + return path +end + +-- not supported atm +function get_stdmodules(target) + if target:policy("build.c++.modules.std") then + end + return {} +end + +function get_bmi_extension() + return ".gcm" +end + +function get_modulesflag(target) + local modulesflag = _g.modulesflag + if modulesflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "gcc_modules_ts"}) then + modulesflag = "-fmodules-ts" + end + assert(modulesflag, "compiler(gcc): does not support c++ module!") + _g.modulesflag = modulesflag or false + end + return modulesflag or nil +end + +function get_moduleheaderflag(target) + local moduleheaderflag = _g.moduleheaderflag + if moduleheaderflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-header", "cxxflags", {flagskey = "gcc_module_header"}) then + moduleheaderflag = "-fmodule-header=" + end + _g.moduleheaderflag = moduleheaderflag or false + end + return moduleheaderflag or nil +end + +function get_moduleonlyflag(target) + local moduleonlyflag = _g.moduleonlyflag + if moduleonlyflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-only", "cxxflags", {flagskey = "gcc_module_only"}) then + moduleonlyflag = "-fmodule-only" + end + _g.moduleonlyflag = moduleonlyflag or false + end + return moduleonlyflag or nil +end + +function get_modulemapperflag(target) + local modulemapperflag = _g.modulemapperflag + if modulemapperflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-mapper=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_module_mapper"}) then + modulemapperflag = "-fmodule-mapper=" + end + assert(modulemapperflag, "compiler(gcc): does not support c++ module!") + _g.modulemapperflag = modulemapperflag or false + end + return modulemapperflag or nil +end + +function get_depsflag(target) + local depflag = _g.depflag + if depflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fdeps-format=p1689r5", "cxxflags", {flagskey = "gcc_deps_format", + on_check = function (ok, errors) + if errors:find("-M") then + ok = true + end + return ok, errors + end}) then + depflag = "-fdeps-format=p1689r5" + end + _g.depflag = depflag or false + end + return depflag or nil +end + +function get_depsfileflag(target) + local depfileflag = _g.depfileflag + if depfileflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fdeps-file=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_deps_file", + on_check = function (ok, errors) + if errors:find("-M") then + ok = true + end + return ok, errors + end}) then + depfileflag = "-fdeps-file=" + end + _g.depfileflag = depfileflag or false + end + return depfileflag or nil +end + +function get_depstargetflag(target) + local depoutputflag = _g.depoutputflag + if depoutputflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fdeps-target=" .. os.tmpfile() .. ".o", "cxxflags", {flagskey = "gcc_deps_output", + on_check = function (ok, errors) + if errors:find("-M") then + ok = true + end + return ok, errors + end}) then + depoutputflag = "-fdeps-target=" + end + _g.depoutputflag = depoutputflag or false + end + return depoutputflag or nil +end + +function get_cppversionflag(target) + local cppversionflag = _g.cppversionflag + if cppversionflag == nil then + local compinst = target:compiler("cxx") + local flags = compinst:compflags({target = target}) + cppversionflag = table.find_if(flags, function(v) string.startswith(v, "-std=c++") end) or "-std=c++20" + _g.cppversionflag = cppversionflag + end + return cppversionflag or nil +end + diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua new file mode 100644 index 00000000000..982eae96059 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -0,0 +1,76 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file gcc/dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.semver") +import("core.project.depend") +import("utils.progress") +import("compiler_support") +import("builder") +import(".dependency_scanner", {inherit = true}) + +-- generate dependency files +function generate_dependencies(target, sourcebatch, opt) + local compinst = target:compiler("cxx") + local baselineflags = {"-E", "-x", "c++"} + local depsformatflag = compiler_support.get_depsflag(target, "p1689r5") + local depsfileflag = compiler_support.get_depsfileflag(target) + local depstargetflag = compiler_support.get_depstargetflag(target) + local changed = false + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function() + if opt.progress then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + end + + local outputdir = compiler_support.get_outputdir(target, sourcefile) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) + if depsformatflag and depsfileflag and depstargetflag and not target:policy("build.c++.gcc.fallbackscanner") then + local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) + local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) + os.vrunv(compinst:program(), flags) + os.rm(ifile) + os.rm(dfile) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local compflags = compinst:compflags({sourcefile = file, target = target}) + -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation + table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) + os.vrunv(compinst:program(), flags) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local dependinfo = io.readfile(jsonfile) + return { moduleinfo = dependinfo } + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) + end + return changed +end + diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua deleted file mode 100644 index e90ae3df3a4..00000000000 --- a/xmake/rules/c++/modules/modules_support/msvc.lua +++ /dev/null @@ -1,821 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki, Arthapz --- @file msvc.lua --- - --- imports -import("core.base.option") -import("core.base.json") -import("core.tool.compiler") -import("core.project.project") -import("core.project.depend") -import("core.project.config") -import("core.base.semver") -import("utils.progress") -import("private.action.build.object", {alias = "objectbuilder"}) -import("common") - --- add a module or header unit into the mapper --- --- e.g --- /reference Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc --- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc --- -function _add_module_to_mapper(target, argument, namekey, path, objectfile, bmifile, deps) - local modulemap = _get_modulemap_from_mapper(target) - if modulemap[namekey] then - return - end - local mapflag = {argument, path .. "=" .. bmifile} - modulemap[namekey] = {flag = mapflag, objectfile = objectfile, deps = deps} - common.localcache():set2(_mapper_cachekey(target), "modulemap", modulemap) -end - -function _mapper_cachekey(target) - local mode = config.mode() - return target:name() .. "_modulemap_" .. (mode or "") -end - --- flush mapper file cache -function _flush_mapper(target) - -- not using set2/get2 to flush only current target mapper - common.localcache():save(_mapper_cachekey(target)) -end - --- get modulemap from mapper -function _get_modulemap_from_mapper(target) - return common.localcache():get2(_mapper_cachekey(target), "modulemap") or {} -end - --- do compile -function _compile(target, flags, sourcefile) - local compinst = target:compiler("cxx") - local msvc = target:toolchain("msvc") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - os.vrunv(compinst:program(), winos.cmdargv(table.join(compflags or {}, flags)), {envs = msvc:runenvs()}) -end - --- do compile for batchcmds --- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) -end - --- add an objectfile to the linker flags --- --- e.g --- foo.obj --- -function _add_objectfile_to_link_arguments(target, objectfile) - local cachekey = target:name() .. "dependency_objectfiles" - local cache = common.localcache():get(cachekey) or {} - if table.contains(cache, objectfile) then - return - end - table.insert(cache, objectfile) - common.localcache():set(cachekey, cache) - common.localcache():save(cachekey) -end - --- build module file -function _build_modulefile(target, sourcefile, opt) - local objectfile = opt.objectfile - local dependfile = opt.dependfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - - -- need build this object? - local dryrun = option.get("dry-run") - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(objectfile) and os.mtime(dependfile) or 0 - if not dryrun and not depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return - end - - -- init flags - local flags = table.join("-TP", compflags, opt.flags or {}) - - -- trace - progress.show(opt.progress, "${color.build.object}compiling.module.$(mode) %s", opt.name) - vprint(compinst:compcmd(sourcefile, objectfile, {compflags = flags, rawargs = true})) - - if not dryrun then - - -- do compile - dependinfo.files = {} - assert(compinst:compile(sourcefile, objectfile, {dependinfo = dependinfo, compflags = flags})) - - -- update files and values to the dependent file - dependinfo.values = depvalues - table.join2(dependinfo.files, sourcefile) - depend.save(dependinfo, dependfile) - end -end - --- load module support for the current target -function load(target) - - -- add modules flags if visual studio version is < 16.11 (vc 14.29) - local modulesflag = get_modulesflag(target) - local msvc = target:toolchain("msvc") - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.29") < 0 then - target:add("cxxflags", modulesflag) - end - - -- enable std modules if c++23 by defaults - if target:data("c++.msvc.enable_std_import") == nil then - local languages = target:get("languages") - local isatleastcpp23 = false - for _, language in ipairs(languages) do - if language:startswith("c++") or language:startswith("cxx") then - isatleastcpp23 = true - local version = tonumber(language:match("%d+")) - if (not version or version <= 20) and not language:match("latest") then - isatleastcpp23 = false - break - end - end - end - local stdmodulesdir - local msvc = target:toolchain("msvc") - if msvc then - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.35") > 0 then - stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - end - end - target:data_set("c++.msvc.enable_std_import", isatleastcpp23 and os.isdir(stdmodulesdir)) - end -end - --- provide toolchain include dir for stl headerunit when p1689 is not supported -function toolchain_includedirs(target) - for _, toolchain_inst in ipairs(target:toolchains()) do - if toolchain_inst:name() == "msvc" then - local vcvars = toolchain_inst:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion then - return { path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "include") } - end - break - end - end - raise("msvc toolchain includedirs not found!") -end - --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) - local msvc = target:toolchain("msvc") - local scandependenciesflag = get_scandependenciesflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - local common_flags = {"-TP", scandependenciesflag} - local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function () - if opt.progress then - progress.show(opt.progress, "${color.build.object}generating.module.deps %s", sourcefile) - end - local outputdir = common.get_outputdir(target, sourcefile) - - local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") - if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then - local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} - _compile(target, table.join(common_flags, flags), sourcefile) - else - common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = file, target = target}) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(compflags, - {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true - - local dependinfo = io.readfile(jsonfile) - return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end - return changed -end - --- generate header unit module bmi for batchjobs -function generate_headerunit_for_batchjob(target, name, flags, objectfile, index, total) - -- don't generate same header unit bmi at the same time across targets - if not common.memcache():get2(name, "generating") then - local common_flags = {"-TP", "-c"} - common.memcache():set2(name, "generating", true) - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", name) - _compile(target, table.join(common_flags, flags)) - _add_objectfile_to_link_arguments(target, objectfile) - common.memcache():set2(name, "generating", false) - end -end - --- generate header unit module bmi for batchcmds -function generate_headerunit_for_batchcmds(target, name, flags, objectfile, batchcmds, opt) - local common_flags = {"-TP", "-c"} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", name) - _batchcmds_compile(batchcmds, target, table.join(common_flags, flags)) - _add_objectfile_to_link_arguments(target, objectfile) -end - --- generate target stl header unit modules for batchjobs -function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - - -- get flags - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_stl_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - local objectfile = bmifile .. ".obj" - batchjobs:addjob(headerunit.name, function(index, total) - depend.on_changed(function() - local flags = { - exportheaderflag, - headernameflag .. ":angle", - headerunit.name, - ifcoutputflag, - headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir, - "-Fo" .. objectfile - } - generate_headerunit_for_batchjob(target, headerunit.name, flags, objectfile, index, total) - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, headerunit.name, objectfile, bmifile) - end, {rootjob = flushjob}) - end -end - --- generate target stl header units for batchcmds -function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - local objectfile = bmifile .. ".obj" - local flags = { - exportheaderflag, - headernameflag .. ":angle", - headerunit.name, - ifcoutputflag, - path(headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir), - path(objectfile, function (p) return "-Fo" .. p end)} - generate_headerunit_for_batchcmds(target, headerunit.name, flags, objectfile, batchcmds, opt) - batchcmds:add_depfiles(headerunit.path) - _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, headerunit.name, objectfile, bmifile) - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - -- get flags - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_user_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - local outputdir = common.get_outputdir(target, headerunit) - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - local objectdir = path.directory(objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - -- generate headerunit - local flags = { - exportheaderflag, - headernameflag .. headerunit.type, - headerunit.path, - ifcoutputflag, - outputdir, - "/Fo" .. objectfile - } - generate_headerunit_for_batchjob(target, headerunit.name, flags, objectfile, index, total) - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - _add_module_to_mapper(target, headerunitflag, headerunit.name, headerunit.path, objectfile, bmifile) - end, {rootjob = flushjob}) - end -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - local outputdir = common.get_outputdir(target, headerunit) - batchcmds:mkdir(outputdir) - - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchcmds:mkdir(path.directory(objectfile)) - - local flags = { - exportheaderflag, - headernameflag .. headerunit.type, - headerunit.path, - ifcoutputflag, - outputdir, - "/Fo" .. objectfile - } - generate_headerunit_for_batchcmds(target, headerunit.name, flags, objectfile, batchcmds, opt) - batchcmds:add_depfiles(headerunit.path) - - _add_module_to_mapper(target, headerunitflag, headerunit.name, headerunit.path, objectfile, bmifile) - - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- build module files for batchjobs -function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) - - -- get flags - local ifcoutputflag = get_ifcoutputflag(target) - local interfaceflag = get_interfaceflag(target) - local referenceflag = get_referenceflag(target) - local internalpartitionflag = get_internalpartitionflag(target) - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_modules", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - if target:data("c++.msvc.enable_std_import") then - for objectfile, module in pairs(get_stdmodules(target)) do - table.insert(objectfiles, objectfile) - modules[objectfile] = module - modules[objectfile].external = true - end - end - - local modulesjobs = {} - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - -- assume there that provides is only one, until we encounter the case - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local moduleinfo = table.copy(provide) or {} - local flags = {"-TP"} - local dependfile = target:dependfile(objectfile) - - if provide then - table.join2(flags, {ifcoutputflag, path(provide.bmi), provide.interface and interfaceflag or internalpartitionflag}) - dependfile = target:dependfile(provide.bmi) - - local fileconfig = target:fileconfig(cppfile) - if fileconfig and fileconfig.install then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = common.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}generating.module.metadata %s", name) - local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = flushjob}) - end - end - - table.join2(moduleinfo, { - name = name or cppfile, - deps = table.keys(module.requires or {}), - sourcefile = cppfile, - job = batchjobs:newjob(name or cppfile, function(index, total) - -- append module mapper flags first - -- @note we add it at the end to ensure that the full modulemap are already stored in the mapper - local requiresflags - if module.requires then - requiresflags = get_requiresflags(target, module.requires, {expand = true}) - end - local _flags = table.join(flags, requiresflags or {}) - - if provide or common.has_module_extension(cppfile) then - if not common.memcache():get2(name or cppfile, "compiling") then - if name and module.external then - common.memcache():set2(name or cppfile, "compiling", true) - end - _build_modulefile(target, cppfile, { - objectfile = objectfile, - dependfile = dependfile, - name = name or module.cppfile, - flags = _flags, - progress = (index * 100) / total}) - end - _add_objectfile_to_link_arguments(target, objectfile) - elseif requiresflags then - requiresflags = get_requiresflags(target, module.requires) - target:fileconfig_add(cppfile, {force = {cxxflags = table.join(flags, requiresflags)}}) - end - - if provide then - _add_module_to_mapper(target, referenceflag, name, name, objectfile, provide.bmi, requiresflags) - end - end)}) - modulesjobs[name or cppfile] = moduleinfo - end - end - - -- build batchjobs for modules - common.build_batchjobs_for_modules(modulesjobs, batchjobs, flushjob) -end - --- build module files for batchcmds -function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) - - -- get flags - local ifcoutputflag = get_ifcoutputflag(target) - local interfaceflag = get_interfaceflag(target) - local referenceflag = get_referenceflag(target) - local internalpartitionflag = get_internalpartitionflag(target) - - if target:data("c++.msvc.enable_std_import") then - for objectfile, module in pairs(get_stdmodules(target)) do - table.insert(objectfiles, objectfile) - modules[objectfile] = module - end - end - - -- build modules - local depmtime = 0 - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - -- append required modulemap flags to module - local requiresflags - if module.requires then - requiresflags = get_requiresflags(target, module.requires, {expand = true}) - end - - local flags = table.join({"-TP", "-c", path(cppfile), path(objectfile, function (p) return "-Fo" .. p end)}) - if provide or common.has_module_extension(cppfile) then - if provide then - table.join2(flags, {ifcoutputflag, path(provide.bmi), provide.interface and interfaceflag or internalpartitionflag}) - end - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.module.$(mode) %s", name or cppfile) - batchcmds:mkdir(path.directory(objectfile)) - _batchcmds_compile(batchcmds, target, table.join(flags, requiresflags or {}), cppfile) - batchcmds:add_depfiles(cppfile) - _add_objectfile_to_link_arguments(target, path.translate(objectfile)) - if provide then - _add_module_to_mapper(target, referenceflag, name, name, objectfile, provide.bmi, requiresflags) - -- add requiresflags to module. it will be used for project generation - target:fileconfig_add(cppfile, {force = {cxxflags = requiresflags}}) - end - depmtime = math.max(depmtime, os.mtime(provide and provide.bmi or objectfile)) - elseif requiresflags then - requiresflags = get_requiresflags(target, module.requires) - target:fileconfig_add(cppfile, {force = {cxxflags = table.join(flags, requiresflags)}}) - end - end - end - - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - -function get_stdmodules(target) - local modules = {} - - -- build c++23 standard modules if needed - if target:data("c++.msvc.enable_std_import") then - local msvc = target:toolchain("msvc") - if msvc then - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion then - local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") - - local stlcachedir = common.stlmodules_cachedir(target) - local modulefile = path.join(stdmodulesdir, "std.ixx") - local bmifile = path.join(stlcachedir, "std.ixx" .. get_bmi_extension()) - local objfile = bmifile .. ".obj" - modules[objfile] = {provides = {std = {interface = true, sourcefile = modulefile, bmi = bmifile}}} - stlcachedir = common.stlmodules_cachedir(target) - modulefile = path.join(stdmodulesdir, "std.compat.ixx") - bmifile = path.join(stlcachedir, "std.compat.ixx" .. get_bmi_extension()) - objfile = bmifile .. ".obj" - modules[objfile] = {provides = {["std.compat"] = {interface = true, sourcefile = modulefile, bmi = bmifile}}, requires = {std = {unique = false, method = "by-name"}}} - end - end - end - return modules -end - -function get_bmi_extension() - return ".ifc" -end - -function get_modulesflag(target) - local modulesflag = _g.modulesflag - if modulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-experimental:module", "cxxflags", {flagskey = "cl_experimental_module"}) then - modulesflag = "-experimental:module" - end - local msvc = target:toolchain("msvc") - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.29") < 0 then - assert(modulesflag, "compiler(msvc): does not support c++ module!") - end - _g.modulesflag = modulesflag or false - end - return modulesflag or nil -end - -function get_ifcoutputflag(target) - local ifcoutputflag = _g.ifcoutputflag - if ifcoutputflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-ifcOutput", os.tmpfile()}, "cxxflags", {flagskey = "cl_ifc_output"}) then - ifcoutputflag = "-ifcOutput" - end - assert(ifcoutputflag, "compiler(msvc): does not support c++ module flag(/ifcOutput)!") - _g.ifcoutputflag = ifcoutputflag or false - end - return ifcoutputflag or nil -end - -function get_ifcsearchdirflag(target) - local ifcsearchdirflag = _g.ifcsearchdirflag - if ifcsearchdirflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-ifcSearchDir", os.tmpdir()}, "cxxflags", {flagskey = "cl_ifc_search_dir"}) then - ifcsearchdirflag = "-ifcSearchDir" - end - assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module flag(/ifcSearchDir)!") - _g.ifcsearchdirflag = ifcsearchdirflag or false - end - return ifcsearchdirflag or nil -end - -function get_interfaceflag(target) - local interfaceflag = _g.interfaceflag - if interfaceflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-interface", "cxxflags", {flagskey = "cl_interface"}) then - interfaceflag = "-interface" - end - assert(interfaceflag, "compiler(msvc): does not support c++ module flag(/interface)!") - _g.interfaceflag = interfaceflag or false - end - return interfaceflag -end - -function get_referenceflag(target) - local referenceflag = _g.referenceflag - if referenceflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-reference", "Foo=" .. os.tmpfile()}, "cxxflags", {flagskey = "cl_reference"}) then - referenceflag = "-reference" - end - assert(referenceflag, "compiler(msvc): does not support c++ module flag(/reference)!") - _g.referenceflag = referenceflag or false - end - return referenceflag or nil -end - -function get_headernameflag(target) - local headernameflag = _g.headernameflag - if headernameflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:quote"}, "cxxflags", {flagskey = "cl_header_name_quote"}) and - compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:angle"}, "cxxflags", {flagskey = "cl_header_name_angle"}) then - headernameflag = "-headerName" - end - _g.headernameflag = headernameflag or false - end - return headernameflag or nil -end - -function get_headerunitflag(target) - local headerunitflag = _g.headerunitflag - if headerunitflag == nil then - local compinst = target:compiler("cxx") - local ifcfile = os.tmpfile() - if compinst:has_flags({"-std:c++latest", "-headerUnit:quote", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_quote"}) and - compinst:has_flags({"-std:c++latest", "-headerUnit:angle", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_angle"}) then - headerunitflag = "-headerUnit" - end - _g.headerunitflag = headerunitflag or false - end - return headerunitflag or nil -end - -function get_exportheaderflag(target) - local exportheaderflag = _g.exportheaderflag - if exportheaderflag == nil then - if get_headernameflag(target) then - exportheaderflag = "-exportHeader" - end - _g.exportheaderflag = exportheaderflag or false - end - return exportheaderflag or nil -end - -function get_scandependenciesflag(target) - local scandependenciesflag = _g.scandependenciesflag - if scandependenciesflag == nil then - local compinst = target:compiler("cxx") - local scan_dependencies_jsonfile = os.tmpfile() .. ".json" - if compinst:has_flags("-scanDependencies " .. scan_dependencies_jsonfile, "cxflags", {flagskey = "cl_scan_dependencies", - on_check = function (ok, errors) - if os.isfile(scan_dependencies_jsonfile) then - ok = true - end - if ok and not os.isfile(scan_dependencies_jsonfile) then - ok = false - end - return ok, errors - end}) then - scandependenciesflag = "-scanDependencies" - end - _g.scandependenciesflag = scandependenciesflag or false - end - return scandependenciesflag or nil -end - --- get requireflags from module mapper -function get_requiresflags(target, requires, opt) - opt = opt or {} - local flags = {} - local modulemap = _get_modulemap_from_mapper(target) - -- add deps required module flags - local already_mapped_modules = {} - for name, _ in table.orderpairs(requires) do - -- if already in flags, continue - if already_mapped_modules[name] then - goto continue - end - - for _, dep in ipairs(target:orderdeps()) do - local modulemap_ = _get_modulemap_from_mapper(dep) - if modulemap_[name] then - table.join2(flags, modulemap_[name].flag) - -- we need to ignore headerunits from deps - -- @see https://github.com/xmake-io/xmake/issues/3925 - local skip = 0 - for _, flag in ipairs(modulemap_[name].deps) do - if flag:find("headerUnit:quote", 1, true) then - skip = 2 - end - if skip == 0 then - table.insert(flags, flag) - end - if skip > 0 then - skip = skip - 1 - end - end - already_mapped_modules[name] = true - goto continue - end - end - - -- append target required module mapper flags - if modulemap[name] then - table.join2(flags, modulemap[name].flag) - table.join2(flags, modulemap[name].deps or {}) - goto continue - end - - ::continue:: - end - local requireflags = {} - local contains = {} - for i = 1, #flags, 2 do - local value = flags[i + 1] - if not contains[value] then - local key = flags[i] - if opt.expand then - table.insert(requireflags, key) - table.insert(requireflags, value) - else - table.insert(requireflags, {key, value}) - end - contains[value] = true - end - end - if #requireflags > 0 then - return requireflags - end -end - -function get_cppversionflag(target) - local cppversionflag = _g.cppversionflag - if cppversionflag == nil then - local compinst = target:compiler("cxx") - local flags = compinst:compflags({target = target}) - cppversionflag = table.find_if(flags, function(v) string.startswith(v, "/std:c++") end) or "/std:c++latest" - end - return cppversionflag or nil -end - -function get_internalpartitionflag(target) - local internalpartitionflag = _g.internalpartitionflag - if internalpartitionflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-internalPartition", "cxxflags", {flagskey = "cl_internal_partition"}) then - internalpartitionflag = "-internalPartition" - end - _g.internalpartitionflag = internalpartitionflag or false - end - return internalpartitionflag or nil -end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua new file mode 100644 index 00000000000..824989777ca --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -0,0 +1,333 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file msvc/builder.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("core.base.semver") +import("utils.progress") +import("private.action.build.object", {alias = "objectbuilder"}) +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("private.tools.vstool") +import("compiler_support") +import(".builder", {inherit = true}) + +-- get flags for building a module +function _make_modulebuildflags(target, provide, bmifile, sourcefile, objectfile, opt) + + -- get flags + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local ifconlyflag = compiler_support.get_ifconlyflag(target) + local interfaceflag = compiler_support.get_interfaceflag(target) + local internalpartitionflag = compiler_support.get_internalpartitionflag(target) + local objectfile_or_ifconlyflag = (opt.external and ifconlyflag) and ifconlyflag or "-Fo" .. objectfile + + local flags + if provide then -- named module + flags = {"-TP", ifcoutputflag, bmifile, provide.interface and interfaceflag or internalpartitionflag, objectfile_or_ifconlyflag, "-c", sourcefile} + else + flags = {"-TP", objectfile_or_ifconlyflag, "-c", sourcefile} + end + return flags +end + +-- get flags for building a headerunit +function _make_headerunitflags(target, headerunit, bmifile, opt) + + -- get flags + local exportheaderflag = compiler_support.get_exportheaderflag(target) + local headernameflag = compiler_support.get_headernameflag(target) + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + assert(headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") + + local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} + + local flags = table.join(local_directory, {"-TP", + exportheaderflag, + headernameflag .. headerunit.type, + ifcoutputflag, + bmifile, + headerunit.type == ":angle" and headerunit.name or headerunit.path}) + return flags +end + +-- do compile +function _compile(target, flags, sourcefile) + + local dryrun = option.get("dry-run") + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local msvc = target:toolchain("msvc") + local flags = table.join(compflags or {}, flags) + + -- trace + if option.get("verbose") then + print(os.args(table.join(compinst:program(), flags))) + end + + if not dryrun then + -- do compile + os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) + end +end + +-- do compile for batchcmds +-- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx +function _batchcmds_compile(batchcmds, target, flags, sourcefile) + + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) +end + +-- get module requires flags +-- e.g +-- /reference Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc +-- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc +-- +function _get_requiresflags(target, module, opt) + + local referenceflag = compiler_support.get_referenceflag(target) + local headerunitflag = compiler_support.get_headerunitflag(target) + + local name = module.name + local cachekey = target:name() .. name + + local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") + or compiler_support.localcache():get2(cachekey, "requiresflags") + if not requiresflags or (opt and opt.regenerate) then + local deps_flags = {} + for required, _ in table.orderpairs(module.requires) do + local dep_module = get_from_target_mapper(target, required) + + assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) + + local mapflag + local bmifile = dep_module.bmi + -- aliased headerunit + if dep_module.aliasof then + local aliased = get_from_target_mapper(target, dep_module.aliasof) + bmifile = aliased.bmi + mapflag = {headerunitflag .. aliased.headerunit.type, required .. "=" .. bmifile} + -- headerunit + elseif dep_module.headerunit then + mapflag = {headerunitflag .. dep_module.headerunit.type, required .. "=" .. bmifile} + -- named module + else + mapflag = {referenceflag, required .. "=" .. bmifile} + end + table.insert(deps_flags, mapflag) + + -- append deps + if dep_module.opt and dep_module.opt.deps then + local deps = _get_requiresflags(target, { name = dep_module.name or dep_module.sourcefile, bmi = bmifile, requires = dep_module.opt.deps }) + table.join2(deps_flags, deps) + end + end + + requiresflags = {} + -- remove duplicates + local contains = {} + for _, map in ipairs(deps_flags) do + local name, _ = map[2]:split("=")[1], map[2]:split("=")[2] + if not contains[name] then + table.insert(requiresflags, map) + contains[name] = true + end + end + compiler_support.memcache():set2(cachekey, "requiresflags", requiresflags) + compiler_support.localcache():set2(cachekey, "requiresflags", requiresflags) + end + + return requiresflags +end + +function _append_requires_flags(target, module, name, cppfile, bmifile, opt) + local cxxflags = {} + local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build}) + + for _, flag in ipairs(requiresflags) do + -- we need to wrap flag to support flag with space + if type(flag) == "string" and flag:find(" ", 1, true) then + table.insert(cxxflags, {flag}) + else + table.insert(cxxflags, flag) + end + end + target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) +end + +-- populate module map +function populate_module_map(target, modules) + for _, module in pairs(modules) do + local name, provide, cppfile = compiler_support.get_provided_module(module) + if provide then + local bmifile = compiler_support.get_bmi_path(provide.bmi) + add_module_to_target_mapper(target, name, cppfile, bmifile, {deps = module.requires}) + end + end +end + +-- get defines for a module +function get_module_required_defines(target, sourcefile) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local defines + + for _, flag in ipairs(compflags) do + if flag:startswith("-D") or flag:startswith("/D") then + defines = defines or {} + table.insert(defines, flag:sub(3)) + end + end + + return defines +end + +-- build module file for batchjobs +function make_module_build_job(target, batchjobs, job_name, deps, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + return { + name = job_name, + deps = deps, + sourcefile = opt.cppfile, + job = batchjobs:newjob(name or opt.cppfile, function(index, total) + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + -- compile if it's a named module + if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {external = external}) + + _compile(target, flags, opt.cppfile) + end + + table.insert(dependinfo.files, opt.cppfile) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build module file for batchcmds +function make_module_build_cmds(target, batchcmds, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + if opt.build then + if provide or compiler_support.has_module_extension(opt.cppfile) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:mkdir(path.directory(opt.objectfile)) + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {batchcmds = true, external = external}) + _batchcmds_compile(batchcmds, target, flags, opt.cppfile) + end + end + batchcmds:add_depfiles(opt.cppfile) + + return os.mtime(opt.objectfile) +end + +-- build headerunit file for batchjobs +function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) + + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + local name = headerunit.unique and headerunit.name or headerunit.path + + if opt.build then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile, {}), name) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} + end +end + +-- build headerunit file for batchcmds +function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) + + batchcmds:mkdir(outputdir) + + add_headerunit_to_target_mapper(target, headerunit, bmifile) + + if opt.build then + local name = headerunit.unique and headerunit.name or headerunit.path + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile, {batchcmds = true, bmifile = bmifile})) + end + batchcmds:add_depfiles(headerunit.path) + return os.mtime(bmifile) +end + diff --git a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua new file mode 100644 index 00000000000..3f67e0291a5 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua @@ -0,0 +1,243 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file msvc/compiler_support.lua +-- + +-- imports +import("core.base.semver") +import("core.project.config") +import("lib.detect.find_tool") +import(".compiler_support", {inherit = true}) + +-- load module support for the current target +function load(target) + + local msvc = target:toolchain("msvc") + local vcvars = msvc:config("vcvars") + + -- enable std modules if c++23 by defaults + if target:data("c++.msvc.enable_std_import") == nil and target:policy("build.c++.modules.std") then + local languages = target:get("languages") + local isatleastcpp23 = false + for _, language in ipairs(languages) do + if language:startswith("c++") or language:startswith("cxx") then + isatleastcpp23 = true + local version = tonumber(language:match("%d+")) + if (not version or version <= 20) and not language:match("latest") then + isatleastcpp23 = false + break + end + end + end + local stdmodulesdir + local msvc = target:toolchain("msvc") + if msvc then + local vcvars = msvc:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.35") > 0 then + stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + end + end + target:data_set("c++.msvc.enable_std_import", isatleastcpp23 and os.isdir(stdmodulesdir)) + end +end + +-- provide toolchain include dir for stl headerunit when p1689 is not supported +function toolchain_includedirs(target) + for _, toolchain_inst in ipairs(target:toolchains()) do + if toolchain_inst:name() == "msvc" then + local vcvars = toolchain_inst:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion then + return { path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "include") } + end + break + end + end + raise("msvc toolchain includedirs not found!") +end + +-- build c++23 standard modules if needed +function get_stdmodules(target) + if target:policy("build.c++.modules.std") then + if target:data("c++.msvc.enable_std_import") then + local msvc = target:toolchain("msvc") + if msvc then + local vcvars = msvc:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion then + modules = {} + + local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + + return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + end + end + end + end + + return {} +end + +function get_bmi_extension() + return ".ifc" +end + +function get_ifcoutputflag(target) + local ifcoutputflag = _g.ifcoutputflag + if ifcoutputflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-ifcOutput", os.tmpfile()}, "cxxflags", {flagskey = "cl_ifc_output"}) then + ifcoutputflag = "-ifcOutput" + end + assert(ifcoutputflag, "compiler(msvc): does not support c++ module flag(/ifcOutput)!") + _g.ifcoutputflag = ifcoutputflag or false + end + return ifcoutputflag or nil +end + +function get_ifconlyflag(target) + local ifconlyflag = _g.ifconlyflag + if ifconlyflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-ifcOnly"}, "cxxflags", {flagskey = "cl_ifc_only"}) then + ifconlyflag = "-ifcOnly" + end + _g.ifconlyflag = ifconlyflag or false + end + return ifconlyflag or nil +end + +function get_ifcsearchdirflag(target) + local ifcsearchdirflag = _g.ifcsearchdirflag + if ifcsearchdirflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-ifcSearchDir", os.tmpdir()}, "cxxflags", {flagskey = "cl_ifc_search_dir"}) then + ifcsearchdirflag = "-ifcSearchDir" + end + assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module flag(/ifcSearchDir)!") + _g.ifcsearchdirflag = ifcsearchdirflag or false + end + return ifcsearchdirflag or nil +end + +function get_interfaceflag(target) + local interfaceflag = _g.interfaceflag + if interfaceflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-interface", "cxxflags", {flagskey = "cl_interface"}) then + interfaceflag = "-interface" + end + assert(interfaceflag, "compiler(msvc): does not support c++ module flag(/interface)!") + _g.interfaceflag = interfaceflag or false + end + return interfaceflag +end + +function get_referenceflag(target) + local referenceflag = _g.referenceflag + if referenceflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-reference", "Foo=" .. os.tmpfile()}, "cxxflags", {flagskey = "cl_reference"}) then + referenceflag = "-reference" + end + assert(referenceflag, "compiler(msvc): does not support c++ module flag(/reference)!") + _g.referenceflag = referenceflag or false + end + return referenceflag or nil +end + +function get_headernameflag(target) + local headernameflag = _g.headernameflag + if headernameflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:quote"}, "cxxflags", {flagskey = "cl_header_name_quote"}) and + compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:angle"}, "cxxflags", {flagskey = "cl_header_name_angle"}) then + headernameflag = "-headerName" + end + _g.headernameflag = headernameflag or false + end + return headernameflag or nil +end + +function get_headerunitflag(target) + local headerunitflag = _g.headerunitflag + if headerunitflag == nil then + local compinst = target:compiler("cxx") + local ifcfile = os.tmpfile() + if compinst:has_flags({"-std:c++latest", "-headerUnit:quote", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_quote"}) and + compinst:has_flags({"-std:c++latest", "-headerUnit:angle", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_angle"}) then + headerunitflag = "-headerUnit" + end + _g.headerunitflag = headerunitflag or false + end + return headerunitflag or nil +end + +function get_exportheaderflag(target) + local exportheaderflag = _g.exportheaderflag + if exportheaderflag == nil then + if get_headernameflag(target) then + exportheaderflag = "-exportHeader" + end + _g.exportheaderflag = exportheaderflag or false + end + return exportheaderflag or nil +end + +function get_scandependenciesflag(target) + local scandependenciesflag = _g.scandependenciesflag + if scandependenciesflag == nil then + local compinst = target:compiler("cxx") + local scan_dependencies_jsonfile = os.tmpfile() .. ".json" + if compinst:has_flags("-scanDependencies " .. scan_dependencies_jsonfile, "cxflags", {flagskey = "cl_scan_dependencies", + on_check = function (ok, errors) + if os.isfile(scan_dependencies_jsonfile) then + ok = true + end + if ok and not os.isfile(scan_dependencies_jsonfile) then + ok = false + end + return ok, errors + end}) then + scandependenciesflag = "-scanDependencies" + end + _g.scandependenciesflag = scandependenciesflag or false + end + return scandependenciesflag or nil +end + +function get_cppversionflag(target) + local cppversionflag = _g.cppversionflag + if cppversionflag == nil then + local compinst = target:compiler("cxx") + local flags = compinst:compflags({target = target}) + cppversionflag = table.find_if(flags, function(v) string.startswith(v, "/std:c++") end) or "/std:c++latest" + end + return cppversionflag or nil +end + +function get_internalpartitionflag(target) + local internalpartitionflag = _g.internalpartitionflag + if internalpartitionflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-internalPartition", "cxxflags", {flagskey = "cl_internal_partition"}) then + internalpartitionflag = "-internalPartition" + end + _g.internalpartitionflag = internalpartitionflag or false + end + return internalpartitionflag or nil +end diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua new file mode 100644 index 00000000000..ed5027e2298 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -0,0 +1,72 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file msvc/dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.semver") +import("core.project.depend") +import("private.tools.vstool") +import("utils.progress") +import("compiler_support") +import("builder") +import(".dependency_scanner", {inherit = true}) + +-- generate dependency files +function generate_dependencies(target, sourcebatch, opt) + local msvc = target:toolchain("msvc") + local scandependenciesflag = compiler_support.get_scandependenciesflag(target) + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local common_flags = {"-TP", scandependenciesflag} + local changed = false + + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function () + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + local outputdir = compiler_support.get_outputdir(target, sourcefile) + + local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") + if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then + local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} + local compinst = target:compiler("cxx") + local msvc = target:toolchain("msvc") + local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) + os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = file, target = target}) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + os.vrunv(compinst:program(), table.join(compflags, + {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local dependinfo = io.readfile(jsonfile) + return { moduleinfo = dependinfo } + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) + end + return changed +end + diff --git a/xmake/rules/c++/modules/modules_support/stl_headers.lua b/xmake/rules/c++/modules/modules_support/stl_headers.lua index 2bc108765e4..4b2d6ed230d 100644 --- a/xmake/rules/c++/modules/modules_support/stl_headers.lua +++ b/xmake/rules/c++/modules/modules_support/stl_headers.lua @@ -14,7 +14,7 @@ -- -- Copyright (C) 2015-present, TBOOX Open Source Group. -- --- @author Arthapz, ruki +-- @author ruki, Arthapz -- @file stl_headers.lua -- diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index ab90f67a7b3..56ec1534378 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -14,7 +14,7 @@ -- -- Copyright (C) 2015-present, TBOOX Open Source Group. -- --- @author ruki +-- @author ruki, Arthapz -- @file xmake.lua -- @@ -28,11 +28,11 @@ rule("c++.build.modules") add_deps("c++.build.modules.install") on_config(function (target) - import("modules_support.common") + import("modules_support.compiler_support") -- we disable to build across targets in parallel, because the source files may depend on other target modules -- @see https://github.com/xmake-io/xmake/issues/1858 - if common.contains_modules(target) then + if compiler_support.contains_modules(target) then -- @note this will cause cross-parallel builds to be disabled for all sub-dependent targets, -- even if some sub-targets do not contain C++ modules. -- @@ -47,11 +47,8 @@ rule("c++.build.modules") -- @see https://github.com/xmake-io/xmake/issues/3000 target:set("policy", "build.ccache", false) - -- get modules support - local modules_support = common.modules_support(target) - - -- load module support - modules_support.load(target) + -- load compiler support + compiler_support.load(target) -- mark this target with modules target:data_set("cxx.has_modules", true) @@ -66,48 +63,44 @@ rule("c++.build.modules.builder") -- parallel build support to accelerate `xmake build` to build modules before_build_files(function(target, batchjobs, sourcebatch, opt) if target:data("cxx.has_modules") then - import("modules_support.common") - common.patch_sourcebatch(target, sourcebatch, opt) - local modules = common.get_module_dependencies(target, sourcebatch, opt) + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) -- extract packages modules dependencies - local package_modules_data = common.get_all_package_modules(target, modules, opt) + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) if package_modules_data then - -- cull unused modules - package_modules_data = common.cull_unused_modules(target, modules, package_modules_data) - if package_modules_data then - -- append to sourcebatch - for name, package_module_data in pairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - end - - -- we need to repatch and regenerate dependencies at this point - common.patch_sourcebatch(target, sourcebatch, opt) - opt.regenerate = true - modules = common.get_module_dependencies(target, sourcebatch, opt) + -- append to sourcebatch + for _, package_module_data in pairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end end + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + + opt.batchjobs = true + -- build modules - common.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - - -- generate headerunits and we need to do it before building modules - local user_headerunits, stl_headerunits = common.get_headerunits(target, sourcebatch, modules) - if user_headerunits or stl_headerunits then - -- we need new group(headerunits) - -- e.g. group(build_modules) -> group(headerunits) - opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/generate_headerunits", {rootjob = opt.rootjob}) - local modules_support = common.modules_support(target) - if stl_headerunits then - -- build stl header units as other headerunits may need them - -- TODO maybe we need new group(build_modules) -> group(user_headerunits) -> group(stl_headerunits) - modules_support.generate_stl_headerunits_for_batchjobs(target, batchjobs, stl_headerunits, opt) - end - if user_headerunits then - modules_support.generate_user_headerunits_for_batchjobs(target, batchjobs, user_headerunits, opt) - end - end + builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + -- cull external modules objectfile + compiler_support.cull_objectfiles(target, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} @@ -117,43 +110,63 @@ rule("c++.build.modules.builder") -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) if target:data("cxx.has_modules") then - import("modules_support.common") + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end - -- patch sourcebatch - common.patch_sourcebatch(target, sourcebatch, opt) + -- append std module + table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) - -- generate headerunits - local modules = common.get_module_dependencies(target, sourcebatch, opt) - common.generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in pairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + end + end + + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + + opt.batchjobs = false + + -- build headerunits + builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- build modules - common.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + -- cull external modules objectfile + compiler_support.cull_objectfiles(target, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end end) - before_link(function (target) - import("modules_support.common") - if target:data("cxx.has_modules") then - common.append_dependency_objectfiles(target) - end - end) - after_clean(function (target) import("core.base.option") - import("modules_support.common") + import("modules_support.compiler_support") import("private.action.clean.remove_files") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when cleaning targets - if common.contains_modules(target) then - remove_files(common.modules_cachedir(target)) + if compiler_support.contains_modules(target) then + remove_files(compiler_support.modules_cachedir(target)) if option.get("all") then - remove_files(common.stlmodules_cachedir(target)) - common.localcache():clear() - common.localcache():save() + remove_files(compiler_support.stlmodules_cachedir(target)) + compiler_support.localcache():clear() + compiler_support.localcache():save() end end end) @@ -163,11 +176,11 @@ rule("c++.build.modules.install") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") before_install(function (target) - import("modules_support.common") + import("modules_support.compiler_support") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when installing targets - if common.contains_modules(target) then - common.install_module_target(target) + if compiler_support.contains_modules(target) then + compiler_support.install_module_target(target) end end)