CI failures in nightly ubuntu: segfault during loading/precompilation

[lgoettgens]

See https://github.com/oscar-system/Oscar.jl/actions/runs/7730599925/job/21076347030#step:8:393

This only happened in this form since 1.11.0-DEV.1428, although there are successful runs with this julia version (e.g. https://github.com/oscar-system/Oscar.jl/actions/runs/7729826314/job/21073900157?pr=3293).

Log of the above CI run:

ERROR: The following 1 direct dependency failed to precompile:

jl_ihgr8PYdeT [3114afc2-c41a-41d1-969f-1994370dbe37]

Failed to precompile jl_ihgr8PYdeT [3114afc2-c41a-41d1-969f-1994370dbe37] to "/home/runner/.julia/compiled/v1.11/jl_ihgr8PYdeT/jl_BZwNw6".
[4264] signal 11 (1): Segmentation fault
in expression starting at /home/runner/work/Oscar.jl/Oscar.jl/src/imports.jl:14
jl_object_id__cold at /cache/build/builder-amdci4-2/julialang/julia-master/src/builtins.c:441
jl_idset_peek_bp at /cache/build/builder-amdci4-2/julialang/julia-master/src/idset.c:34
jl_idset_get at /cache/build/builder-amdci4-2/julialang/julia-master/src/idset.c:40
jl_as_global_root at /cache/build/builder-amdci4-2/julialang/julia-master/src/staticdata.c:2503 [inlined]
jl_as_global_root at /cache/build/builder-amdci4-2/julialang/julia-master/src/staticdata.c:2483
inst_datatype_inner at /cache/build/builder-amdci4-2/julialang/julia-master/src/jltypes.c:2076
jl_inst_arg_tuple_type at /cache/build/builder-amdci4-2/julialang/julia-master/src/jltypes.c:2259
arg_type_tuple at /cache/build/builder-amdci4-2/julialang/julia-master/src/gf.c:2294 [inlined]
jl_lookup_generic_ at /cache/build/builder-amdci4-2/julialang/julia-master/src/gf.c:3082 [inlined]
ijl_apply_generic at /cache/build/builder-amdci4-2/julialang/julia-master/src/gf.c:3129
initialize_julia_module at /home/runner/.julia/packages/CxxWrap/5IZvn/src/CxxWrap.jl:405
__init__ at /home/runner/.julia/packages/Polymake/DbLWv/src/Polymake.jl:106
jfptr___init___22936 at /home/runner/.julia/compiled/v1.11/Polymake/QizyK_w5EM9.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
jl_module_run_initializer at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:76
run_module_init at ./loading.jl:1204
register_restored_modules at ./loading.jl:1192
_include_from_serialized at ./loading.jl:1137
_include_from_serialized at ./loading.jl:1103 [inlined]
#_require_search_from_serialized#1041 at ./loading.jl:1690
_require_search_from_serialized at ./loading.jl:1645
jfptr__require_search_from_serialized_40650.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
_require at ./loading.jl:2111
__require_prelocked at ./loading.jl:1980
jfptr___require_prelocked_6[392](https://github.com/oscar-system/Oscar.jl/actions/runs/7730599925/job/21076347030#step:8:395)5.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
jl_f__call_in_world at /cache/build/builder-amdci4-2/julialang/julia-master/src/builtins.c:894
#invoke_in_world#3 at ./essentials.jl:1054 [inlined]
invoke_in_world at ./essentials.jl:1051 [inlined]
_require_prelocked at ./loading.jl:1971
macro expansion at ./loading.jl:1909 [inlined]
macro expansion at ./lock.jl:272 [inlined]
__require at ./loading.jl:1870
jfptr___require_63863.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
jl_f__call_in_world at /cache/build/builder-amdci4-2/julialang/julia-master/src/builtins.c:894
#invoke_in_world#3 at ./essentials.jl:1054 [inlined]
invoke_in_world at ./essentials.jl:1051 [inlined]
require at ./loading.jl:1863
jfptr_require_63860.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
call_require at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:486 [inlined]
eval_import_path at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:523
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:812
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:886
ijl_toplevel_eval_in at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:994
eval at ./boot.jl:428 [inlined]
include_string at ./loading.jl:2249
_include at ./loading.jl:2309
include at ./Base.jl:526
jfptr_include_42585.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
jl_f__call_latest at /cache/build/builder-amdci4-2/julialang/julia-master/src/builtins.c:875
include at /home/runner/work/Oscar.jl/Oscar.jl/src/Oscar.jl:20
unknown function (ip: 0x7fb3b0102342)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
do_call at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:126
eval_value at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:223
eval_stmt_value at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:174 [inlined]
eval_body at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:675
jl_interpret_toplevel_thunk at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:815
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:943
jl_eval_module_expr at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:215 [inlined]
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:743
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:886
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:886
ijl_toplevel_eval_in at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:994
eval at ./boot.jl:428 [inlined]
include_string at ./loading.jl:2249
_include at ./loading.jl:2309
include at ./Base.jl:526 [inlined]
include_package_for_output at ./loading.jl:2427
jfptr_include_package_for_output_64184.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
do_call at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:126
eval_value at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:223
eval_stmt_value at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:174 [inlined]
eval_body at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:675
jl_interpret_toplevel_thunk at /cache/build/builder-amdci4-2/julialang/julia-master/src/interpreter.c:815
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:943
jl_toplevel_eval_flex at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:886
ijl_toplevel_eval_in at /cache/build/builder-amdci4-2/julialang/julia-master/src/toplevel.c:994
eval at ./boot.jl:428 [inlined]
include_string at ./loading.jl:2249
include_string at ./loading.jl:2259 [inlined]
exec_options at ./client.jl:316
_start at ./client.jl:525
jfptr__start_65541.1 at /opt/hostedtoolcache/julia/nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci4-2/julialang/julia-master/src/julia.h:2161 [inlined]
true_main at /cache/build/builder-amdci4-2/julialang/julia-master/src/jlapi.c:947
jl_repl_entrypoint at /cache/build/builder-amdci4-2/julialang/julia-master/src/jlapi.c:1106
main at /cache/build/builder-amdci4-2/julialang/julia-master/cli/loader_exe.c:58
unknown function (ip: 0x7fb3b7429d8f)
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
unknown function (ip: 0x[401](https://github.com/oscar-system/Oscar.jl/actions/runs/7730599925/job/21076347030#step:8:404)0b8)
Allocations: 2666557 (Pool: 2666[412](https://github.com/oscar-system/Oscar.jl/actions/runs/7730599925/job/21076347030#step:8:415); Big: 145); GC: 4
ERROR: LoadError: Failed to precompile Oscar [f1435218-dba5-11e9-1e4d-f1a5fab5fc13] to "/home/runner/.julia/compiled/v1.11/Oscar/jl_9yplz9".
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:35
  [2] compilecache(pkg::Base.PkgId, path::String, internal_stderr::IO, internal_stdout::IO, keep_loaded_modules::Bool; reasons::Dict{String, Int64})
    @ Base ./loading.jl:2691
  [3] compilecache
    @ ./loading.jl:2558 [inlined]
  [4] (::Base.var"#1045#1046"{Base.PkgId})()
    @ Base ./loading.jl:2147
  [5] mkpidlock(f::Base.var"#1045#1046"{Base.PkgId}, at::String, pid::Int32; kwopts::@Kwargs{stale_age::Int64, wait::Bool})
    @ FileWatching.Pidfile /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/FileWatching/src/pidfile.jl:95
  [6] #mkpidlock#6
    @ /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/FileWatching/src/pidfile.jl:90 [inlined]
  [7] trymkpidlock(::Function, ::Vararg{Any}; kwargs::@Kwargs{stale_age::Int64})
    @ FileWatching.Pidfile /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/FileWatching/src/pidfile.jl:116
  [8] #invokelatest#2
    @ ./essentials.jl:1022 [inlined]
  [9] invokelatest
    @ ./essentials.jl:1017 [inlined]
 [10] maybe_cachefile_lock(f::Base.var"#1045#1046"{Base.PkgId}, pkg::Base.PkgId, srcpath::String; stale_age::Int64)
    @ Base ./loading.jl:3277
 [11] maybe_cachefile_lock
    @ ./loading.jl:3274 [inlined]
 [12] _require(pkg::Base.PkgId, env::String)
    @ Base ./loading.jl:2143
 [13] __require_prelocked(uuidkey::Base.PkgId, env::String)
    @ Base ./loading.jl:1980
 [14] #invoke_in_world#3
    @ ./essentials.jl:1054 [inlined]
 [15] invoke_in_world
    @ ./essentials.jl:1051 [inlined]
 [16] _require_prelocked(uuidkey::Base.PkgId, env::String)
    @ Base ./loading.jl:1971
 [17] macro expansion
    @ ./loading.jl:1909 [inlined]
 [18] macro expansion
    @ ./lock.jl:272 [inlined]
 [19] __require(into::Module, mod::Symbol)
    @ Base ./loading.jl:1870
 [20] #invoke_in_world#3
    @ ./essentials.jl:1054 [inlined]
 [21] invoke_in_world
    @ ./essentials.jl:1051 [inlined]
 [22] require(into::Module, mod::Symbol)
    @ Base ./loading.jl:1863
 [23] include
    @ ./Base.jl:526 [inlined]
 [24] include_package_for_output(pkg::Base.PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, concrete_deps::Vector{Pair{Base.PkgId, UInt128}}, source::Nothing)
    @ Base ./loading.jl:2[427](https://github.com/oscar-system/Oscar.jl/actions/runs/7730599925/job/21076347030#step:8:430)
 [25] top-level scope
    @ stdin:3
in expression starting at /tmp/jl_ihgr8PYdeT/src/jl_ihgr8PYdeT.jl:1
in expression starting at stdin:
Stacktrace:
 [1] pkgerror(msg::String)
   @ Pkg.Types /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Pkg/src/Types.jl:70
 [2] precompile(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; internal_call::Bool, strict::Bool, warn_loaded::Bool, already_instantiated::Bool, timing::Bool, _from_loading::Bool, kwargs::@Kwargs{io::Base.DevNull})
   @ Pkg.API /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Pkg/src/API.jl:1683
 [3] precompile(pkgs::Vector{Pkg.Types.PackageSpec}; io::Base.DevNull, kwargs::@Kwargs{})
   @ Pkg.API /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Pkg/src/API.jl:159
 [4] precompile
   @ /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Pkg/src/API.jl:148 [inlined]
 [5] precompile(; name::Nothing, uuid::Nothing, version::Nothing, url::Nothing, rev::Nothing, path::Nothing, mode::Pkg.Types.PackageMode, subdir::Nothing, kwargs::@Kwargs{io::Base.DevNull})
   @ Pkg.API /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Pkg/src/API.jl:174
 [6] top-level scope
   @ none:1
┌ Error: Unexpected error: /tmp/jl_ihgr8PYdeT/done.log was not created, but precompilation exited
└ @ Aqua ~/.julia/packages/Aqua/9p8ck/src/persistent_tasks.jl:109
Persistent tasks: Test Failed at /home/runner/.julia/packages/Aqua/9p8ck/src/persistent_tasks.jl:29
  Expression: !(has_persistent_tasks(package; kwargs...))

Stacktrace:
 [1] macro expansion
   @ /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Test/src/Test.jl:679 [inlined]
 [2] test_persistent_tasks(package::Base.PkgId; broken::Bool, kwargs::@Kwargs{})
   @ Aqua ~/.julia/packages/Aqua/9p8ck/src/persistent_tasks.jl:29
 [3] test_persistent_tasks
   @ ~/.julia/packages/Aqua/9p8ck/src/persistent_tasks.jl:25 [inlined]
 [4] #test_persistent_tasks#69
   @ ~/.julia/packages/Aqua/9p8ck/src/persistent_tasks.jl:34 [inlined]
 [5] test_persistent_tasks
   @ ~/.julia/packages/Aqua/9p8ck/src/persistent_tasks.jl:33 [inlined]
 [6] macro expansion
   @ ~/.julia/packages/Aqua/9p8ck/src/Aqua.jl:105 [inlined]
 [7] macro expansion
   @ /opt/hostedtoolcache/julia/nightly/x64/share/julia/stdlib/v1.11/Test/src/Test.jl:1598 [inlined]
 [8] test_all(testtarget::Module; ambiguities::Bool, unbound_args::Bool, undefined_exports::Bool, project_extras::Bool, stale_deps::Bool, deps_compat::Bool, piracies::Bool, persistent_tasks::Bool)
   @ Aqua ~/.julia/packages/Aqua/9p8ck/src/Aqua.jl:104
Skipping Oscar.docsproject
Skipping Oscar.dp
Skipping Oscar.f0
Skipping Oscar.SubfieldLattice_Module.last_M
Skipping Oscar.StraightLinePrograms.last_bla
Skipping Oscar.GaloisGrp.last_N
Test Summary:                                | Pass  Fail  Total     Time
Aqua.jl                                      |    8     1      9  2m53.2s
  Method ambiguity                           |              None     0.0s
  Unbound type parameters                    |              None     0.0s
  Undefined exports                          |    1            1     0.1s
  Compare Project.toml and test/Project.toml |    1            1     0.1s
  Stale dependencies                         |    1            1  1m16.3s
  Compat bounds                              |    4            4     0.5s
  Piracy                                     |              None     0.0s
  Persistent tasks                           |          1      1  1m30.7s

[benlorenz]

This has been happening for a bit, the first occurrence I could find in a quick search is here, https://github.com/oscar-system/Oscar.jl/actions/runs/7489387259/job/20385820568#step:7:2915, 3 weeks ago with julia 1.11.0-DEV.1249.

I don't think this has anything to do with the Aqua tests, it seems to be some issue when loading Oscar as it also happens when the tests are about to start. There is a very similar backtrace in this run https://github.com/oscar-system/Oscar.jl/actions/runs/7686201455/job/20944792824?pr=3189#step:8:360 directly after Testing Running tests....

[fingolfin]

Let's check some recent tests to see if it occurred again, otherwise close it...

[benlorenz]

Still happens, for example in the tests for #3586: https://github.com/oscar-system/Oscar.jl/actions/runs/8638534713/job/23683179472?pr=3586#step:8:1683

So far I have seen this only with the 1.11 versions (while they were still nightly) but not with current 1.12 nightly. Found a 1.12 case as well: https://github.com/oscar-system/Oscar.jl/actions/runs/8554124431/job/23438761668#step:8:379

The backtraces usually point to some CxxWrap initialization, sometimes for Polymake.jl but in the latest case for CxxWrap directly:

[2678] signal 11 (1): Segmentation fault
in expression starting at none:7
jl_object_id__cold at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/builtins.c:441
jl_idset_peek_bp at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/idset.c:34
jl_idset_get at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/idset.c:40
jl_as_global_root at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/staticdata.c:2516 [inlined]
jl_as_global_root at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/staticdata.c:2496
inst_datatype_inner at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/jltypes.c:2077
jl_inst_arg_tuple_type at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/jltypes.c:2265
arg_type_tuple at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/gf.c:2278 [inlined]
jl_lookup_generic_ at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/gf.c:3072 [inlined]
ijl_apply_generic at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/gf.c:3119
initialize_julia_module at /home/runner/.julia/packages/CxxWrap/5IZvn/src/CxxWrap.jl:405
__init__ at /home/runner/.julia/packages/CxxWrap/5IZvn/src/StdLib.jl:25
jfptr___init___20826 at /home/runner/.julia/compiled/v1.11/CxxWrap/WGIJU_20so5.so (unknown line)
jl_apply at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/julia.h:2154 [inlined]
jl_module_run_initializer at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/toplevel.c:76
run_module_init at ./loading.jl:1264
register_restored_modules at ./loading.jl:1252
_include_from_serialized at ./loading.jl:1197
_include_from_serialized at ./loading.jl:1154 [inlined]
#_require_search_from_serialized#1069 at ./loading.jl:1902
_require_search_from_serialized at ./loading.jl:1841
jfptr__require_search_from_serialized_43734.1 at /opt/hostedtoolcache/julia/1.11-nightly/x64/lib/julia/sys.so (unknown line)
_require at ./loading.jl:2347
__require_prelocked at ./loading.jl:2217
jfptr___require_prelocked_68534.1 at /opt/hostedtoolcache/julia/1.11-nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/julia.h:2154 [inlined]
jl_f__call_in_world at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/builtins.c:894
#invoke_in_world#3 at ./essentials.jl:1064 [inlined]
invoke_in_world at ./essentials.jl:1061 [inlined]
_require_prelocked at ./loading.jl:2208 [inlined]
_require_prelocked at ./loading.jl:2207 [inlined]
macro expansion at ./lock.jl:273 [inlined]
require at ./loading.jl:2204
jfptr_require_68443.1 at /opt/hostedtoolcache/julia/1.11-nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/julia.h:2154 [inlined]
do_call at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/interpreter.c:126
eval_value at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/interpreter.c:223
eval_stmt_value at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/interpreter.c:174 [inlined]
eval_body at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/interpreter.c:659
jl_interpret_toplevel_thunk at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/interpreter.c:817
jl_toplevel_eval_flex at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/toplevel.c:943
jl_toplevel_eval_flex at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/toplevel.c:886
ijl_toplevel_eval_in at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/toplevel.c:994
eval at ./boot.jl:428 [inlined]
exec_options at ./client.jl:291
_start at ./client.jl:526
jfptr__start_70792.1 at /opt/hostedtoolcache/julia/1.11-nightly/x64/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/julia.h:2154 [inlined]
true_main at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/jlapi.c:900
jl_repl_entrypoint at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/jlapi.c:1059
main at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/cli/loader_exe.c:58
unknown function (ip: 0x7fb147229d8f)
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
unknown function (ip: 0x4010b8)