Adv/multi file serialization

src/Groups/GAPGroups.jl

@@ -635,7 +635,7 @@ julia> length(small_generating_set(abelian_group(PermGroup, [2,3,4])))
 ```
 """
 @gapattribute function small_generating_set(G::GAPGroup)
-   # We claim that the finiteness check is cheap in Oscar.
+  # We claim that the finiteness check is cheap in Oscar.
    # This does not hold in GAP,
    # and GAP's method selection benefits from the known finiteness flag.
    if G isa MatrixGroup && is_infinite(base_ring(G))

src/Serialization/PolyhedralGeometry.jl

@@ -94,7 +94,22 @@ function save_object(s::SerializerState, lp::LinearProgram{QQFieldElem})
   end
 end
 
+function save_object(s::SerializerState{<: LPSerializer}, lp::LinearProgram{QQFieldElem})
+  lp_filename = basepath(s.serializer) * "-$(objectid(lp)).lp"
+  _internal_save_lp(
+    lp_filename,
+    pm_object(lp.feasible_region),
+    lp.polymake_lp,
+    lp.convention == :min ? true : false)
+
+  save_object(s, basename(lp_filename))
+end
+
 function load_object(s::DeserializerState, ::Type{<:LinearProgram}, field::QQField)
+  if s.obj isa String
+    @warn "LP not loaded properly, please load using serialize_type=Oscar.LPSerializer"
+    return linear_program(cube(1), QQFieldElem[0])
+  end
   coeff_type = elem_type(field)
   fr = load_object(s, Polyhedron, field, :feasible_region)
   conv = load_object(s, String, :convention)
@@ -113,6 +128,14 @@ function load_object(s::DeserializerState, ::Type{<:LinearProgram}, field::QQFie
   return LinearProgram{coeff_type}(fr, lp, Symbol(conv))
 end
 
+function load_object(s::DeserializerState{LPSerializer},
+                     ::Type{<:LinearProgram}, field::QQField)
+  load_node(s) do _
+    lp_filename = dirname(basepath(s.serializer)) * "/$(s.obj)"
+    pm_lp = load_lp(lp_filename)
+  end
+end
+
 ##############################################################################
 @register_serialization_type MixedIntegerLinearProgram uses_params
 

src/Serialization/main.jl

@@ -356,10 +356,10 @@ function register_serialization_type(ex::Any, str::String, uses_id::Bool,
       if !($ex <: Union{Number, String, Bool, Symbol, Vector, Tuple, Matrix, NamedTuple, Dict, Set})
         function Oscar.serialize(s::Oscar.AbstractSerializer, obj::T) where T <: $ex
           Oscar.serialize_type(s, T)
-          Oscar.save(s.io, obj; serializer_type=Oscar.IPCSerializer)
+          Oscar.save(s.io, obj; serializer=Oscar.IPCSerializer())
         end
         function Oscar.deserialize(s::Oscar.AbstractSerializer, ::Type{<:$ex})
-          Oscar.load(s.io; serializer_type=Oscar.IPCSerializer)
+          Oscar.load(s.io; serializer=Oscar.IPCSerializer())
         end
       end
     end)
@@ -511,11 +511,10 @@ julia> load("/tmp/fourtitwo.mrdi")
 """
 function save(io::IO, obj::T; metadata::Union{MetaData, Nothing}=nothing,
               with_attrs::Bool=true,
-              serializer_type::Type{<: OscarSerializer} = JSONSerializer) where T
-
-  s = state(serializer_open(io, serializer_type,
-                            with_attrs ? type_attr_map : Dict{String, Vector{Symbol}}()))
-  save_data_dict(s) do
+              serializer::OscarSerializer = JSONSerializer()) where T
+  s = serializer_open(io, serializer,
+                      with_attrs ? type_attr_map : Dict{String, Vector{Symbol}}())
+  save_data_dict(s) do 
     # write out the namespace first
     save_header(s, get_oscar_serialization_version(), :_ns)
 
@@ -528,21 +527,9 @@ function save(io::IO, obj::T; metadata::Union{MetaData, Nothing}=nothing,
         global_serializer_state.id_to_obj[ref] = obj
       end
       save_object(s, string(ref), :id)
-
     end
 
-    # this should be handled by serializers in a later commit / PR
-    if !isempty(s.refs) && serializer_type == JSONSerializer
-      save_data_dict(s, refs_key) do
-        for id in s.refs
-          ref_obj = global_serializer_state.id_to_obj[id]
-          s.key = Symbol(id)
-          save_data_dict(s) do
-            save_typed_object(s, ref_obj)
-          end
-        end
-      end
-    end
+    handle_refs(s)
 
     if !isnothing(metadata)
       save_json(s, json(metadata), :meta)
@@ -552,13 +539,25 @@ function save(io::IO, obj::T; metadata::Union{MetaData, Nothing}=nothing,
   return nothing
 end
 
-function save(filename::String, obj::Any; metadata::Union{MetaData, Nothing}=nothing,
+function save(filename::String, obj::Any;
+              metadata::Union{MetaData, Nothing}=nothing,
+              serializer_type::Type{<:OscarSerializer}=JSONSerializer,
               with_attrs::Bool=true)
   dir_name = dirname(filename)
   # julia dirname does not return "." for plain filenames without any slashes
   temp_file = tempname(isempty(dir_name) ? pwd() : dir_name)
+
+  if serializer_type <: MultiFileSerializer
+    serializer = serializer_type(splitext(filename)[1])
+  else
+    serializer = serializer_type()
+  end
+
   open(temp_file, "w") do file
-    save(file, obj; metadata=metadata, with_attrs=with_attrs)
+    save(file, obj;
+         metadata=metadata,
+         with_attrs=with_attrs,
+         serializer=serializer)
   end
   Base.Filesystem.rename(temp_file, filename) # atomic "multi process safe"
   return nothing
@@ -623,7 +622,7 @@ true
 """
 function load(io::IO; params::Any = nothing, type::Any = nothing,
               serializer_type=JSONSerializer, with_attrs::Bool=true)
-  s = state(deserializer_open(io, serializer_type, with_attrs))
+  s = deserializer_open(io, serializer_type, with_attrs)
   if haskey(s.obj, :id)
     id = s.obj[:id]
     if haskey(global_serializer_state.id_to_obj, UUID(id))
@@ -720,8 +719,15 @@ function load(io::IO; params::Any = nothing, type::Any = nothing,
 end
 
 function load(filename::String; params::Any = nothing,
-              type::Any = nothing, with_attrs::Bool=true)
+              type::Any = nothing, with_attrs::Bool=true,
+              serializer_type::Type{<:OscarSerializer}=JSONSerializer)
+  if serializer_type <: MultiFileSerializer
+    serializer = serializer_type(splitext(filename)[1])
+  else
+    serializer = serializer_type()
+  end
+
   open(filename) do file
-    return load(file; params=params, type=type)
+    return load(file; params=params, type=type, serializer=serializer)
   end
 end

src/Serialization/serializers.jl

@@ -1,6 +1,22 @@
 using JSON3
 import Base.haskey
 
+################################################################################
+# Serializers
+abstract type OscarSerializer end
+
+struct JSONSerializer <: OscarSerializer end
+
+struct IPCSerializer <: OscarSerializer end
+
+abstract type MultiFileSerializer <: OscarSerializer end
+
+struct LPSerializer <: MultiFileSerializer
+  basepath::String
+end
+
+basepath(serializer::MultiFileSerializer) = serializer.basepath
+
 ################################################################################
 # (de)Serializer States
 
@@ -23,8 +39,8 @@ function reset_global_serializer_state()
 end
 
 # struct which tracks state for (de)serialization
-mutable struct SerializerState
-  # dict to track already serialized objects
+mutable struct SerializerState{T <: OscarSerializer}
+  serializer::T
   new_level_entry::Bool
   # UUIDs that point to the objs in the global state,
   # ideally this would be an ordered set
@@ -137,9 +153,10 @@ function finish_writing(s::SerializerState)
   # nothing to do here
 end
 
-mutable struct DeserializerState
+mutable struct DeserializerState{T <: OscarSerializer}
   # or perhaps Dict{Int,Any} to be resilient against corrupts/malicious files using huge ids
   # the values of refs are objects to be deserialized
+  serializer::T
   obj::Union{Dict{Symbol, Any}, Vector, JSON3.Object, JSON3.Array, BasicTypeUnion}
   key::Union{Symbol, Int, Nothing}
   refs::Union{Dict{Symbol, Any}, JSON3.Object, Nothing}
@@ -200,27 +217,13 @@ function load_params_node(s::DeserializerState)
   end
 end
 
-################################################################################
-# Serializers
-abstract type OscarSerializer end
-
-struct JSONSerializer <: OscarSerializer
-  state::S where S <: Union{SerializerState, DeserializerState}
-end
-
-struct IPCSerializer <: OscarSerializer
-  state::S where S <: Union{SerializerState, DeserializerState}
-end
-
-state(s::OscarSerializer) = s.state
-
 function serializer_open(
   io::IO,
-  T::Type{<: OscarSerializer},
+  serializer::OscarSerializer,
   type_attr_map::S) where S <: Union{Dict{String, Vector{Symbol}}, Nothing}
 
   # some level of handling should be done here at a later date
-  return T(SerializerState(true, UUID[], io, nothing, type_attr_map))
+  return SerializerState(serializer, true, UUID[], io, nothing, type_attr_map)
 end
 
 function deserializer_open(io::IO, T::Type{JSONSerializer}, with_attrs::Bool)
@@ -229,17 +232,34 @@ function deserializer_open(io::IO, T::Type{JSONSerializer}, with_attrs::Bool)
   if haskey(obj, refs_key)
     refs = obj[refs_key]
   end
-  
-  return T(DeserializerState(obj, nothing, refs, with_attrs))
+
+  return DeserializerState(serializer, obj, nothing, refs, type_attr_map)
 end
 
-function deserializer_open(io::IO, T::Type{IPCSerializer}, with_attrs::Bool) 
+function deserializer_open(
+  io::IO,
+  serializer::IPCSerializer,
+  type_attr_map::S) where S <:Union{Dict{String, Vector{Symbol}}, Nothing}
   # Using a JSON3.Object from JSON3 version 1.13.2 causes
   # @everywhere using Oscar
   # to hang. So we use a Dict here for now.
 
   obj = JSON.parse(io, dicttype=Dict{Symbol, Any})
-  return T(DeserializerState(obj, nothing, nothing, with_attrs))
+  return DeserializerState(serializer, obj, nothing, nothing, type_attr_map)
+end
+
+function handle_refs(s::SerializerState)
+  if !isempty(s.refs) 
+    save_data_dict(s, refs_key) do
+      for id in s.refs
+        ref_obj = global_serializer_state.id_to_obj[id]
+        s.key = Symbol(id)
+        save_data_dict(s) do
+          save_typed_object(s, ref_obj)
+        end
+      end
+    end
+  end
 end
 
 function attrs_list(s::SerializerState, T::Type) 

test/Serialization/PolyhedralGeometry.jl

@@ -130,6 +130,19 @@ using Oscar: _integer_variables
         @test objective_function(LP) == objective_function(loaded)
         @test feasible_region(LP) == feasible_region(loaded)
       end
+
+      # currently cant pass the kws to setup tests
+      filename = joinpath(path, "original.json")
+      save(filename, LP; serializer_type=Oscar.LPSerializer)
+      loaded = load(filename; serializer_type=Oscar.LPSerializer)
+      @test feasible_region(LP) == feasible_region(loaded)
+      # should also test warning?
+
+      test_save_load_roundtrip(path, LP) do loaded
+        @test objective_function(LP) == objective_function(loaded)
+        @test feasible_region(LP) == feasible_region(loaded)
+      end
+
     end
 
     @testset "MixedIntegerLinearProgram" begin