Merge pull request #5686 from Arthapz/fix-module-scanning

Fix module scanning

tests/projects/c++/modules/dependency_flag_update/src/bar.mpp

@@ -0,0 +1,9 @@
+module;
+
+#include <iostream>
+
+export module bar;
+
+namespace bar {
+export void hello() { std::cout << "Hello world2" << std::endl; }
+} // namespace bar

tests/projects/c++/modules/dependency_flag_update/src/foo.mpp

@@ -0,0 +1,9 @@
+module;
+
+#include <iostream>
+
+export module foo;
+
+namespace foo {
+export void hello() { std::cout << "Hello world" << std::endl; }
+} // namespace foo

tests/projects/c++/modules/dependency_flag_update/src/main.cpp

@@ -0,0 +1,12 @@
+#if defined(FOO)
+import foo;
+using namespace foo;
+#else
+import bar;
+using namespace bar;
+#endif
+
+int main() {
+  hello();
+  return 0;
+}

tests/projects/c++/modules/dependency_flag_update/test.lua

@@ -0,0 +1 @@
+inherit(".test_dependency_scanner")

tests/projects/c++/modules/dependency_flag_update/xmake.lua

@@ -0,0 +1,11 @@
+add_rules("mode.release", "mode.debug")
+set_languages("c++20")
+
+option("foo")
+    set_default("true")
+    add_defines("FOO")
+
+target("dependency_flag_update")
+    set_kind("binary")
+    add_files("src/*.cpp", "src/*.mpp")
+    add_options("foo")

tests/projects/c++/modules/dependency_flag_update2/src/bar.mpp

@@ -0,0 +1,9 @@
+module;
+
+#include <iostream>
+
+export module bar;
+
+namespace bar {
+export void hello() { std::cout << "Hello world2" << std::endl; }
+} // namespace bar

tests/projects/c++/modules/dependency_flag_update2/src/foo.mpp

@@ -0,0 +1,9 @@
+module;
+
+#include <iostream>
+
+export module foo;
+
+namespace foo {
+export void hello() { std::cout << "Hello world" << std::endl; }
+} // namespace foo

tests/projects/c++/modules/dependency_flag_update2/src/foobar.mpp

@@ -0,0 +1,11 @@
+export module foobar;
+
+#if defined(FOO)
+import foo;
+namespace impl = foo;
+#else
+import bar;
+namespace impl = bar;
+#endif
+
+export void hello() { impl::hello(); }

tests/projects/c++/modules/dependency_flag_update2/src/main.cpp

@@ -0,0 +1,6 @@
+import foobar;
+
+int main() {
+  hello();
+  return 0;
+}

tests/projects/c++/modules/dependency_flag_update2/test.lua

@@ -0,0 +1 @@
+inherit(".test_dependency_scanner")

tests/projects/c++/modules/dependency_flag_update2/xmake.lua

@@ -0,0 +1,12 @@
+add_rules("mode.release", "mode.debug")
+set_languages("c++20")
+
+option("foo")
+    set_default("true")
+    add_defines("FOO")
+
+target("dependency_flag_update3")
+    set_kind("binary")
+    add_files("src/*.cpp", "src/*.mpp")
+    add_options("foo")
+

tests/projects/c++/modules/test_dependency_scanner.lua

@@ -0,0 +1,76 @@
+import("lib.detect.find_tool")
+import("core.base.semver")
+import("utils.ci.is_running", {alias = "ci_is_running"})
+
+function _build()
+    os.run("xmake f --foo=n")
+    local outdata
+    if ci_is_running() then
+        outdata = os.iorun("xmake -rvD")
+    else
+        outdata = os.iorun("xmake -rv")
+    end
+    if outdata then
+        if outdata:find("FOO") then
+            raise("Modules dependency scanner update does not work\n%s", outdata)
+        end
+    end
+    outdata = os.iorun("xmake")
+    if outdata then
+        if outdata:find("compiling") or outdata:find("linking") or outdata:find("generating") then
+            raise("Modules incremental compilation does not work\n%s", outdata)
+        end
+    end
+end
+
+function can_build()
+    if is_subhost("windows") then
+        return true
+    elseif is_subhost("msys") then
+        return true
+    elseif is_host("linux") then
+        local gcc = find_tool("gcc", {version = true})
+        if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then
+            return true
+        end
+        local clang = find_tool("clang", {version = true})
+        if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then
+            return true
+        end
+    end
+end
+
+function main(t)
+    if is_subhost("windows") then
+        local clang = find_tool("clang", {version = true})
+        if clang and clang.version and semver.compare(clang.version, "17.0") >= 0 then
+            os.exec("xmake f --toolchain=clang -c --yes")
+            _build()
+            os.exec("xmake clean -a")
+            os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes")
+            _build()
+        end
+
+        os.exec("xmake clean -a")
+        os.exec("xmake f -c --yes")
+        _build()
+    elseif is_subhost("msys") then
+        os.exec("xmake f -c -p mingw --yes")
+        _build()
+    elseif is_host("linux") then
+        local gcc = find_tool("gcc", {version = true})
+        if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then
+            os.exec("xmake f -c --yes")
+            _build()
+        end
+        local clang = find_tool("clang", {version = true})
+        if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then
+            os.exec("xmake clean -a")
+            os.exec("xmake f --toolchain=clang -c --yes")
+            _build()
+            os.exec("xmake clean -a")
+            os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes")
+            _build()
+        end
+    end
+end

xmake/rules/c++/modules/modules_support/builder.lua

@@ -21,6 +21,7 @@
 -- imports
 import("core.base.json")
 import("core.base.option")
+import("core.base.hashset")
 import("async.runjobs")
 import("private.async.buildjobs")
 import("core.tool.compiler")
@@ -460,3 +461,23 @@ function add_headerunit_to_target_mapper(target, headerunit, bmifile)
     return deduplicated and true or false
 end
 
+-- check if dependencies changed
+function is_dependencies_changed(target, module)
+    local cachekey = target:name() .. module.name
+    local requires = hashset.from(table.keys(module.requires or {}))
+    local oldrequires = compiler_support.memcache():get2(cachekey, "oldrequires")
+    local changed = false
+    if oldrequires then
+        if oldrequires ~= requires then
+           requires_changed = true
+        else
+           for required in requires:items() do
+              if not oldrequires:has(required) then
+                  requires_changed = true
+                  break
+              end
+           end
+        end
+    end
+    return requires, changed
+end

xmake/rules/c++/modules/modules_support/clang/builder.lua

@@ -122,12 +122,11 @@ function _get_requiresflags(target, module, opt)
     local name = module.name
     local cachekey = target:name() .. name
 
+    local requires, requires_changed = is_dependencies_changed(target, module)
     local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags")
-                       or compiler_support.localcache():get2(cachekey, "requiresflags")
-
-    if not requiresflags or (opt and opt.regenerate) then
+    if not requiresflags or requires_changed then
         requiresflags = {}
-        for required, _ in table.orderpairs(module.requires) do
+        for required in requires:orderitems() do
             local dep_module = get_from_target_mapper(target, required)
             assert(dep_module, "module dependency %s required for %s not found", required, name)
 
@@ -146,15 +145,16 @@ function _get_requiresflags(target, module, opt)
                 table.join2(requiresflags, deps)
             end
         end
-        compiler_support.memcache():set2(cachekey, "requiresflags", table.unique(requiresflags))
-        compiler_support.localcache():set2(cachekey, "requiresflags", table.unique(requiresflags))
+        requiresflags = table.unique(requiresflags)
+        compiler_support.memcache():set2(cachekey, "requiresflags", requiresflags)
+        compiler_support.memcache():set2(cachekey, "oldrequires", requires)
     end
     return requiresflags
 end
 
 function _append_requires_flags(target, module, name, cppfile, bmifile, opt)
     local cxxflags = {}
-    local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build})
+    local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires})
     for _, flag in ipairs(requiresflags) do
         -- we need to wrap flag to support flag with space
         if type(flag) == "string" and flag:find(" ", 1, true) then
@@ -358,4 +358,3 @@ function get_requires(target, module)
     end
     return _requires
 end
-

xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua

@@ -31,6 +31,9 @@ function generate_dependency_for(target, sourcefile, opt)
     local compinst = target:compiler("cxx")
     local changed = false
     local dependfile = target:dependfile(sourcefile)
+    local flags = compinst:compflags({sourcefile = sourcefile, target = target}) or {}
+    local fileconfig = target:fileconfig(sourcefile)
+
     depend.on_changed(function()
         if opt.progress then
             progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
@@ -47,13 +50,12 @@ function generate_dependency_for(target, sourcefile, opt)
                 clang_path = compiler_support.get_clang_path(target) or compinst:program()
             end
             local clangscandeps = compiler_support.get_clang_scan_deps(target)
-            local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
-            local flags = table.join({"--format=p1689", "--",
-                                     clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {})
+            local dependency_flags = table.join({"--format=p1689", "--",
+                                                 clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, flags)
             if option.get("verbose") then
-                print(os.args(table.join(clangscandeps, flags)))
+                print(os.args(table.join(clangscandeps, dependency_flags)))
             end
-            local outdata, errdata = os.iorunv(clangscandeps, flags)
+            local outdata, errdata = os.iorunv(clangscandeps, dependency_flags)
             assert(errdata, errdata)
 
             io.writefile(jsonfile, outdata)
@@ -63,15 +65,15 @@ function generate_dependency_for(target, sourcefile, opt)
             end
             fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
                 local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target)
-                local compflags = compinst:compflags({sourcefile = file, target = target})
+                local compflags = table.clone(flags)
                 -- exclude -fmodule* and -std=c++/gnu++* flags because
                 -- when they are set clang try to find bmi of imported modules but they don't exists in this point of compilation
                 table.remove_if(compflags, function(_, flag)
                     return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++")
                 end)
                 local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
-                local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile})
-                os.vrunv(compinst:program(), flags)
+                compflags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile})
+                os.vrunv(compinst:program(), compflags)
                 local content = io.readfile(ifile)
                 os.rm(ifile)
                 return content
@@ -81,8 +83,7 @@ function generate_dependency_for(target, sourcefile, opt)
 
         local rawdependinfo = io.readfile(jsonfile)
         return {moduleinfo = rawdependinfo}
-    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
-
+    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = flags})
     return changed
 end
 

xmake/rules/c++/modules/modules_support/dependency_scanner.lua

@@ -237,7 +237,7 @@ end
 function get_module_dependencies(target, sourcebatch, opt)
     local cachekey = target:name() .. "/" .. sourcebatch.rulename
     local modules = compiler_support.memcache():get2("modules", cachekey)
-    if modules == nil or opt.regenerate then
+    if modules == nil then
         modules = compiler_support.localcache():get2("modules", cachekey)
         opt.progress = opt.progress or 0
         local changed = _generate_dependencies(target, sourcebatch, opt)

xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua

@@ -35,6 +35,7 @@ function generate_dependency_for(target, sourcefile, opt)
     local depsfileflag = compiler_support.get_depsfileflag(target)
     local depstargetflag = compiler_support.get_depstargetflag(target)
     local dependfile = target:dependfile(sourcefile)
+    local flags = compinst:compflags({sourcefile = sourcefile, target = target}) or {}
     local changed = false
 
     depend.on_changed(function()
@@ -48,22 +49,21 @@ function generate_dependency_for(target, sourcefile, opt)
         if has_depsflags and not target:policy("build.c++.gcc.fallbackscanner") then
             local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i"))
             local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d"))
-            local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
-            local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile})
-            os.vrunv(compinst:program(), flags)
+            local compflags = table.join(flags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile})
+            os.vrunv(compinst:program(), compflags)
             os.rm(ifile)
             os.rm(dfile)
         else
             if not has_depsflags then
                 wprint("GCC doesn't support module scanning ! using fallback scanner")
             end
             fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
-                local compflags = compinst:compflags({sourcefile = file, target = target})
+                local compflags = table.clone(flags)
                 -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation
                 table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end)
                 local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
-                local flags = table.join(baselineflags, compflags or {}, {file,  "-o", ifile})
-                os.vrunv(compinst:program(), flags)
+                compflags = table.join(baselineflags, compflags or {}, {file,  "-o", ifile})
+                os.vrunv(compinst:program(), compflags)
                 local content = io.readfile(ifile)
                 os.rm(ifile)
                 return content
@@ -73,7 +73,7 @@ function generate_dependency_for(target, sourcefile, opt)
 
         local dependinfo = io.readfile(jsonfile)
         return { moduleinfo = dependinfo }
-    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
+    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = flags})
     return changed
 end