Merge pull request #631 from KristofferC/backports-release-0.4

Backports some fixes to 0.4

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.54
+
+ - **Deprecation**: Do not rely on JLD2 to load a compression library. This feature will be removed in a future release. Instead explicitly add `using` statements into your scripts.
+
 ## 0.4.53
  - Experimental: Slicing and inplace updating of array datasets
  - updated CI workflows
@@ -22,7 +26,7 @@
  - fix `Upgrade` for Singleton types
 
 ## 0.4.50
- - Don't hide exception data during loading and saving (#569) 
+ - Don't hide exception data during loading and saving (#569)
 
 ## 0.4.49
  - update compat bounds
@@ -32,7 +36,7 @@
  - fix behaviour for unnormalized strings
  - add missing method for load_attributes
  - clean up `using` statements
- 
+
 ## 0.4.47
  - fix loading structs with more than 256 fields (#558)
 
@@ -71,7 +75,7 @@
  - restrict default Dict encoding to Base implementations
 
 ## 0.4.37
- - Update Dict encoding for latest julia 
+ - Update Dict encoding for latest julia
 
 ## 0.4.36
  - compat bound for TranscodingStreams.jl
@@ -93,22 +97,22 @@
 ## 0.4.31
  - fix UInt32 truncation error for absurdly large array sizes
  - move test-files to a separate repo
- 
+
 ## 0.4.30
  -  allow loading compressed files during precompilation #446 (@marius311)
- 
+
 ## 0.4.29
  - added `Upgrade` feature
- 
+
 ## 0.4.28
  - compatibility to julia v1.9-dev (@eschnett)
- 
+
 ## 0.4.26
  - fix identity relations with custom serialization
 
 ## 0.4.25
  - remove leftover debug statement
- 
+
 ## 0.4.24
  - read-only support for `JLD.jl` files
  - read-only support for many HDF5 files. Most test files of HDF5.jl are covered
@@ -119,14 +123,14 @@
 
 ## 0.4.23
  - Support for `const` fields in mutable structs
- 
+
 ## 0.4.22
  - Fix reconstruction of partially initialized structs
 
 ## 0.4.21
- - Add explicit type mapping 
+ - Add explicit type mapping
 
-## 0.4.20 
+## 0.4.20
  - TTFX improvements
  - Add a comment on jldsave (@BoundaryValueProblems)
 ## 0.4.19

Project.toml

@@ -1,6 +1,6 @@
 name = "JLD2"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.53"
+version = "0.4.54"
 
 [deps]
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"

src/compression.jl

@@ -33,35 +33,18 @@ end
 # Dynamic Package Loading Logic copied from FileIO
 const load_locker = Base.ReentrantLock()
 
-function _findmod(f::Symbol)
-    for (u,v) in Base.loaded_modules
-        (Symbol(v) == f) && return u
-    end
-    nothing
-end
-
-function topimport(modname)
-    @info "Attempting to dynamically load $modname"
-    @eval Base.__toplevel__  import $modname
-    u = _findmod(modname)
-    @eval $modname = Base.loaded_modules[$u]
-end
 
 function checked_import(pkg::Symbol)
     lock(load_locker) do
-        # kludge for test suite
-        if isdefined(Main, pkg)
-            m1 = getfield(Main, pkg)
-            isa(m1, Module) && return false, m1
+        for m in Base.loaded_modules_array()
+            (Symbol(m) == pkg) && return false, m
         end
-        if isdefined(JLD2, pkg)
-            m1 = getfield(JLD2, pkg)
-            isa(m1, Module) && return false, m1
+        @info "Attempting to dynamically load $pkg"
+        @eval Base.__toplevel__  import $pkg
+        for m in Base.loaded_modules_array()
+            (Symbol(m) == pkg) && return true, m
         end
-        m = _findmod(pkg)
-        (m === nothing) || return false, Base.loaded_modules[m]
-        topimport(pkg)
-        return true, Base.loaded_modules[_findmod(pkg)]
+        throw(InternalError("Module $pkg could not be loaded."))
     end
 end
 

src/data/reconstructing_datatypes.jl

@@ -20,12 +20,12 @@ end
 """
     readas(::Type)::Type
 
-**Experimental feature**: 
+**Experimental feature**:
 `JLD2.readas` can be overloaded to override which type a saved type is read as,
 and is used together with custom serialization using [`JLD2.writeas`](@ref).
 
 The typical case is custom serialization of parametric types,
-where not all type parameters are available during reading. 
+where not all type parameters are available during reading.
 Consider the following example for an anonymous function `fun` inside a `Foo`
 ```julia
 struct Foo{F<:Function}
@@ -53,7 +53,7 @@ restart julia, include the definitions again, and call
 `foo::Foo{UndefinedFunction}` and `foo::FooSerialization`
 with and without defining the `JLD2.readas` above, respectively.
 """
-readas(::Any) = nothing # default to nothing to do nothing if no overload is specified. 
+readas(::Any) = nothing # default to nothing to do nothing if no overload is specified.
 
 function _readas(T_custom, T_in)
     T_out = readas(T_custom)::Union{Type,Nothing}
@@ -65,7 +65,7 @@ end
 function jltype(f::JLDFile, sdt::Union{SharedDatatype,CommittedDatatype})
     cdt = get(f.datatype_locations, sdt.header_offset, sdt)
     haskey(f.h5jltype, cdt) && return f.h5jltype[cdt]::ReadRepresentation
-    
+
     dt, attrs = read_shared_datatype(f, cdt)
 
     julia_type_attr = nothing
@@ -223,12 +223,12 @@ function constructrr(f::JLDFile, T::DataType, dt::CompoundDatatype,
             # The on disk representation of T can only be the same as in memory
             # if the offsets are the same, field type on disk (readtype) and in memory (wstype)
             # are the same and if no CustomSerialization is involved
-            samelayout = samelayout && 
-                offsets[i] == fieldoffset(T, i) && 
-                types[i] === wstype && 
+            samelayout = samelayout &&
+                offsets[i] == fieldoffset(T, i) &&
+                types[i] === wstype &&
                 # An OnDiskRepresentation as odr means that something "fixable" went wrong
                 # for this field
-                !(odrs[i] isa OnDiskRepresentation) && 
+                !(odrs[i] isa OnDiskRepresentation) &&
                 !(odrs[i] <: CustomSerialization)
 
             mapped[dtindex] = true
@@ -285,13 +285,13 @@ function constructrr(f::JLDFile, u::Upgrade, dt::CompoundDatatype,
 
     T2 = NamedTuple{tuple(dt.names...), typeof(rodr).parameters[2]}
 
-    return (ReadRepresentation{u.target, CustomSerialization{T2, rodr}}(), false)    
+    return (ReadRepresentation{u.target, CustomSerialization{T2, rodr}}(), false)
 end
 
-function constructrr(f::JLDFile, u::Upgrade, dt::BasicDatatype, 
+function constructrr(f::JLDFile, u::Upgrade, dt::BasicDatatype,
                      attrs::Vector{ReadAttribute},
                      hard_failure::Bool=false)
-    return (ReadRepresentation{u.target, CustomSerialization{NamedTuple{(), Tuple{}},nothing}}(), false)    
+    return (ReadRepresentation{u.target, CustomSerialization{NamedTuple{(), Tuple{}},nothing}}(), false)
 end
 
 function constructrr(f::JLDFile, T::UnionAll, dt::CompoundDatatype,
@@ -334,9 +334,7 @@ function _resolve_type(rr::ReadRepresentation{T,DataTypeODR()},
                        hasparams::Bool,
                        params) where T
     parts = split(mypath, '.')
-    modules = vcat([Main], collect(keys(Base.module_keys)))
-    unique!(modules)
-    for mod in modules
+    for mod in Base.loaded_modules_array()
         resolution_attempt = _resolve_type_singlemodule(rr,
                                                         mod,
                                                         parts,
@@ -689,9 +687,9 @@ function jlconvert(::ReadRepresentation{T, S}, f::JLDFile, ptr::Ptr, header_offs
         offset = offsets[i]
         rtype = types[i]
         odr = odrs[i]
-    
+
         rr = ReadRepresentation{rtype,odr}()
-        if !(jlconvert_canbeuninitialized(rr)) || jlconvert_isinitialized(rr, ptr+offset) 
+        if !(jlconvert_canbeuninitialized(rr)) || jlconvert_isinitialized(rr, ptr+offset)
             res[i] = jlconvert(rr, f, ptr+offset, NULL_REFERENCE)
         end
     end
@@ -721,9 +719,9 @@ end
             offset = offsets[i]
             rtype = types[i]
             odr = odrs[i]
-    
+
             rr = ReadRepresentation{rtype,odr}()
-    
+
             fni = QuoteNode(fn[i])
             ttype = T.types[i]
             if odr === nothing
@@ -745,7 +743,7 @@ end
             end
         end
 
-        push!(args, (:obj))    
+        push!(args, (:obj))
         return blk
     end
     if isbitstype(T)

src/data/specialcased_types.jl

@@ -304,13 +304,15 @@ wconvert(::Type{String}, x::Module) = string(x)
 function rconvert(::Type{Module}, x::String)
     pkg = Symbol(x)
     # Try to find the module
-    # Start with the method used to find compression libraries
-    m =_findmod(pkg)
-    isnothing(m) || return Base.loaded_modules[m]
-    @info "Encountered reference to module $x, but it is not currently loaded."
+    for m in Base.loaded_modules_array()
+        (Symbol(m) == pkg) && return m
+    end
+    @warn "Encountered reference to module $x, but it is not currently loaded."
     return try
-        topimport(pkg)
-        Base.loaded_modules[_findmod(pkg)]
+        @eval Base.__toplevel__  import $pkg
+        for m in Base.loaded_modules_array()
+            (Symbol(m) == pkg) && return m
+        end
     catch
        @warn "Could not load module $x. Returning a dummy module"
        Module(Symbol(x*"_dummy"))

test/modules.jl

@@ -21,7 +21,7 @@ S = BType(x)
 end
 
 @testset "name collisions" begin
-    mods = collect(keys(Base.module_keys))
+    mods = Base.loaded_modules_array()
     # use whichever module would not be found first in a linear search
     M = findfirst(==(A), mods) < findfirst(==(B), mods) ? B : A
     x = M.SameNameType(42)

test/runtests.jl

@@ -1,5 +1,6 @@
 using JLD2, FileIO
 using Test
+using CodecZlib, CodecBzip2, CodecZstd, CodecLz4
 
 function better_success(cmd)
     fn1, _ = mktemp()