[ChunkCodecCore] BREAKING Fix Codec ignoring max_size (#5)

* [ChunkCodecCore] Fix `max_size` being ignored when decoding with a `Codec` * BREAKING make `max_size` a required argument for `try_resize_decode!` * Bump version
nhz2 · Dec 29, 2024 · 954201f · 954201f · nhz2 · Dec 29, 2024
1 parent 3fa6fd8
commit 954201f
Show file tree

Hide file tree

Showing 18 changed files with 76 additions and 59 deletions.
diff --git a/ChunkCodecCore/CHANGELOG.md b/ChunkCodecCore/CHANGELOG.md
@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+## [v0.2.0](https://github.com/nhz2/ChunkCodecs.jl/tree/ChunkCodecCore-v0.2.0) - 2024-12-29
+
+### BREAKING `try_resize_decode!`'s signature is changed. [#5](https://github.com/nhz2/ChunkCodecs.jl/pull/5)
+
+`max_size` is a required positional argument instead of an optional keyword argument.
+
+### Fixed
+
+- When using a `Codec` as a decoder the `max_size` option was ignored. [#5](https://github.com/nhz2/ChunkCodecs.jl/pull/5)
 
 ## [v0.1.1](https://github.com/nhz2/ChunkCodecs.jl/tree/ChunkCodecCore-v0.1.1) - 2024-12-20
 

diff --git a/ChunkCodecCore/Project.toml b/ChunkCodecCore/Project.toml
@@ -1,7 +1,7 @@
 name = "ChunkCodecCore"
 uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
 authors = ["nhz2 <[email protected]>"]
-version = "0.1.1"
+version = "0.2.0"
 
 [compat]
 julia = "1.11"
diff --git a/ChunkCodecCore/src/interface.jl b/ChunkCodecCore/src/interface.jl
@@ -52,7 +52,7 @@ function decode(
     end
     _clamp_size_hint::Int64 = clamp(size_hint, Int64(0), _clamp_max_size)
     dst = zeros(UInt8, _clamp_size_hint)
-    real_dst_size = try_resize_decode!(d, dst, src; max_size=_clamp_max_size)::Union{Nothing, Int64}
+    real_dst_size = try_resize_decode!(d, dst, src, _clamp_max_size)::Union{Nothing, Int64}
     if isnothing(real_dst_size)
         throw(DecodedSizeError(_clamp_max_size, try_find_decoded_size(d, src)))
     end
@@ -173,7 +173,7 @@ Only the initial returned number of bytes are valid output.
 function try_decode! end
 
 """
-    try_resize_decode!(d, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; max_size::Int64=typemax(Int64), kwargs...)::Union{Nothing, Int64}
+    try_resize_decode!(d, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
 
 Try to decode the input `src` into `dst` using decoder `d`.
 
@@ -191,7 +191,7 @@ Precondition: `dst` and `src` do not overlap in memory.
 All of `dst` can be written to or used as scratch space by the decoder.
 Only the initial returned number of bytes are valid output.
 """
-function try_resize_decode!(d, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; max_size::Int64=typemax(Int64), kwargs...)::Union{Nothing, Int64}
+function try_resize_decode!(d, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
     check_in_range(Int64(0):max_size; dst_size=length(dst))
     olb::Int64 = length(dst)
     real_dst_size::Int64 = -1
@@ -238,8 +238,8 @@ end
 
 # allow passing codec to decode
 try_find_decoded_size(c::Codec, src::AbstractVector{UInt8}) = try_find_decoded_size(decode_options(c), src)
-try_decode!(c::Codec, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...) = try_decode!(decode_options(c), dst, src)
-try_resize_decode!(c::Codec, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...) = try_resize_decode!(decode_options(c), dst, src)
+try_decode!(c::Codec, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...) = try_decode!(decode_options(c), dst, src; kwargs...)
+try_resize_decode!(c::Codec, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...) = try_resize_decode!(decode_options(c), dst, src, max_size; kwargs...)
 
 """
     check_contiguous(x::AbstractVector{UInt8})

diff --git a/ChunkCodecCore/src/types.jl b/ChunkCodecCore/src/types.jl
@@ -40,6 +40,6 @@ Required methods for a type `T <: DecodeOptions` to implement:
 
 Optional methods to implement:
 - `is_thread_safe(::T)::Bool`: defaults to `false`.
-- `try_resize_decode!(::T, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; max_size::Int64=typemax(Int64), kwargs...)::Union{Nothing, Int64}`: defaults to using `try_decode!` and `try_find_decoded_size`
+- `try_resize_decode!(::T, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}`: defaults to using `try_decode!` and `try_find_decoded_size`
 """
 abstract type DecodeOptions end
diff --git a/ChunkCodecCore/test/runtests.jl b/ChunkCodecCore/test/runtests.jl
@@ -1,6 +1,9 @@
 using Random: Random
-using ChunkCodecCore: ChunkCodecCore, NoopCodec, NoopEncodeOptions, NoopDecodeOptions, DecodedSizeError, decode
-using ChunkCodecTests: test_codec
+using ChunkCodecCore:
+    ChunkCodecCore,
+    NoopCodec, NoopEncodeOptions, NoopDecodeOptions,
+    DecodedSizeError, decode
+using ChunkCodecTests: test_codec, test_encoder_decoder
 using Aqua: Aqua
 using Test: @test, @testset, @test_throws
 
@@ -10,6 +13,8 @@ Random.seed!(1234)
 
 @testset "noop codec" begin
     test_codec(NoopCodec(), NoopEncodeOptions(), NoopDecodeOptions(); trials=100)
+    # codec can be used as decoder
+    test_encoder_decoder(NoopEncodeOptions(), NoopCodec(); trials=20)
 end
 @testset "errors" begin
     @test sprint(Base.showerror, DecodedSizeError(1, 2)) == "DecodedSizeError: decoded size: 2 is greater than max size: 1"

diff --git a/ChunkCodecLibBlosc/Project.toml b/ChunkCodecLibBlosc/Project.toml
@@ -12,5 +12,5 @@ ChunkCodecCore = {path = "../ChunkCodecCore"}
 
 [compat]
 Blosc_jll = "1"
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 julia = "1.11"
diff --git a/ChunkCodecLibBzip2/Project.toml b/ChunkCodecLibBzip2/Project.toml
@@ -12,5 +12,5 @@ ChunkCodecCore = {path = "../ChunkCodecCore"}
 
 [compat]
 Bzip2_jll = "1"
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 julia = "1.11"
diff --git a/ChunkCodecLibBzip2/src/decode.jl b/ChunkCodecLibBzip2/src/decode.jl
@@ -43,10 +43,10 @@ function try_find_decoded_size(::BZ2DecodeOptions, src::AbstractVector{UInt8})::
 end
 
 function try_decode!(d::BZ2DecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...)::Union{Nothing, Int64}
-    try_resize_decode!(d, dst, src; max_size=Int64(length(dst)))
+    try_resize_decode!(d, dst, src, Int64(length(dst)))
 end
 
-function try_resize_decode!(d::BZ2DecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; max_size::Int64=typemax(Int64), kwargs...)::Union{Nothing, Int64}
+function try_resize_decode!(d::BZ2DecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
     check_in_range(Int64(0):max_size; dst_size=length(dst))
     olb::Int64 = length(dst)
     dst_size::Int64 = olb

diff --git a/ChunkCodecLibLz4/Project.toml b/ChunkCodecLibLz4/Project.toml
@@ -11,6 +11,6 @@ Lz4_jll = "5ced341a-0733-55b8-9ab6-a4889d929147"
 ChunkCodecCore = {path = "../ChunkCodecCore"}
 
 [compat]
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 Lz4_jll = "1.10.0"
 julia = "1.11"
diff --git a/ChunkCodecLibLz4/src/decode.jl b/ChunkCodecLibLz4/src/decode.jl
@@ -36,10 +36,10 @@ function try_find_decoded_size(::LZ4FrameDecodeOptions, src::AbstractVector{UInt
 end
 
 function try_decode!(d::LZ4FrameDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...)::Union{Nothing, Int64}
-    try_resize_decode!(d, dst, src; max_size=Int64(length(dst)))
+    try_resize_decode!(d, dst, src, Int64(length(dst)))
 end
 
-function try_resize_decode!(d::LZ4FrameDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; max_size::Int64=typemax(Int64), kwargs...)::Union{Nothing, Int64}
+function try_resize_decode!(d::LZ4FrameDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
     check_in_range(Int64(0):max_size; dst_size=length(dst))
     olb::Int64 = length(dst)
     dst_size::Int64 = olb

diff --git a/ChunkCodecLibZlib/Project.toml b/ChunkCodecLibZlib/Project.toml
@@ -11,6 +11,6 @@ Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a"
 ChunkCodecCore = {path = "../ChunkCodecCore"}
 
 [compat]
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 Zlib_jll = "1.2.13"
 julia = "1.11"
diff --git a/ChunkCodecLibZlib/src/decode.jl b/ChunkCodecLibZlib/src/decode.jl
@@ -67,10 +67,10 @@ function try_find_decoded_size(::_AllDecodeOptions, src::AbstractVector{UInt8}):
     nothing
 end
 function try_decode!(d::_AllDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...)::Union{Nothing, Int64}
-    try_resize_decode!(d, dst, src; max_size=Int64(length(dst)))
+    try_resize_decode!(d, dst, src, Int64(length(dst)))
 end
 
-function try_resize_decode!(d::_AllDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; max_size::Int64=typemax(Int64), kwargs...)::Union{Nothing, Int64}
+function try_resize_decode!(d::_AllDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
     check_in_range(Int64(0):max_size; dst_size=length(dst))
     olb::Int64 = length(dst)
     dst_size::Int64 = olb

diff --git a/ChunkCodecLibZstd/Project.toml b/ChunkCodecLibZstd/Project.toml
@@ -11,6 +11,6 @@ Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4"
 ChunkCodecCore = {path = "../ChunkCodecCore"}
 
 [compat]
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 Zstd_jll = "1.5.6"
 julia = "1.11"
diff --git a/ChunkCodecTests/Project.toml b/ChunkCodecTests/Project.toml
@@ -11,6 +11,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 ChunkCodecCore = {path = "../ChunkCodecCore"}
 
 [compat]
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 Test = "1"
 julia = "1.11"
diff --git a/ChunkCodecTests/src/ChunkCodecTests.jl b/ChunkCodecTests/src/ChunkCodecTests.jl
@@ -20,15 +20,29 @@ using ChunkCodecCore:
 
 using Test: Test, @test, @test_throws
 
-export test_codec
+export test_codec, test_encoder_decoder, rand_test_data
 
 function test_codec(c::Codec, e::EncodeOptions, d::DecodeOptions; trials=100)
     @test decode_options(c) isa DecodeOptions
+    @test can_concatenate(c) isa Bool
     @test codec(e) == c
     @test codec(d) == c
     @test is_thread_safe(e) isa Bool
     @test is_thread_safe(d) isa Bool
 
+    test_encoder_decoder(e, d; trials)
+
+    # can_concatenate tests
+    if can_concatenate(c)
+        srange = decoded_size_range(e)
+        a = rand(UInt8, 100*step(srange))
+        b = rand(UInt8, 200*step(srange))
+        @test decode(d, [encode(e, a); encode(e, b);]) == [a; b;]
+        @test decode(d, [encode(e, UInt8[]); encode(e, UInt8[]);]) == UInt8[]
+    end
+end
+
+function test_encoder_decoder(e, d; trials=100)
     @test decoded_size_range(e) isa StepRange{Int64, Int64}
 
     srange = decoded_size_range(e)
@@ -48,17 +62,7 @@ function test_codec(c::Codec, e::EncodeOptions, d::DecodeOptions; trials=100)
         rand(first(srange):step(srange):min(last(srange), 2000000), trials);
     ]
     for s in decoded_sizes
-        # generate data
-        local choice = rand(1:4)
-        local data = if choice == 1
-            rand(UInt8, s)
-        elseif choice == 2
-            zeros(UInt8, s)
-        elseif choice == 3
-            ones(UInt8, s)
-        elseif choice == 4
-            rand(0x00:0x0f, s)
-        end
+        local data = rand_test_data(s)
         local e_bound = encode_bound(e, s)
         local encoded = encode(e, data)
         local buffer = rand(UInt8, max(length(encoded)+11, e_bound+11))
@@ -100,42 +104,47 @@ function test_codec(c::Codec, e::EncodeOptions, d::DecodeOptions; trials=100)
             dst = zeros(UInt8, s - 1)
             @test_throws(
                 ArgumentError("dst_size ∈ $(0:-1) must hold. Got\ndst_size => $(s-1)"),
-                try_resize_decode!(d, dst, encoded; max_size=Int64(-1))
+                try_resize_decode!(d, dst, encoded, Int64(-1))
             )
             dst = zeros(UInt8, s - 1)
-            @test try_resize_decode!(d, dst, encoded; max_size=s) == s
+            @test try_resize_decode!(d, dst, encoded, s) == s
             @test length(dst) == s
             @test dst == data
             dst = UInt8[]
-            @test isnothing(try_resize_decode!(d, dst, encoded; max_size=Int64(0)))
+            @test isnothing(try_resize_decode!(d, dst, encoded, Int64(0)))
         end
         if s > 1
             dst = UInt8[]
-            @test isnothing(try_resize_decode!(d, dst, encoded; max_size=Int64(1)))
+            @test isnothing(try_resize_decode!(d, dst, encoded, Int64(1)))
             dst = UInt8[0x01]
-            @test isnothing(try_resize_decode!(d, dst, encoded; max_size=Int64(1)))
+            @test isnothing(try_resize_decode!(d, dst, encoded, Int64(1)))
             @test_throws DecodedSizeError(1, try_find_decoded_size(d, encoded)) decode(d, encoded; max_size=Int64(1))
         end
         dst_buffer = zeros(UInt8, s + 2)
         dst = view(dst_buffer, 1:s+1)
         @test_throws(
             ArgumentError("dst_size ∈ $(0:s) must hold. Got\ndst_size => $(s+1)"),
-            try_resize_decode!(d, dst, encoded; max_size=s),
+            try_resize_decode!(d, dst, encoded, s),
         )
-        @test try_resize_decode!(d, dst, encoded; max_size=s+2) === s
+        @test try_resize_decode!(d, dst, encoded, s+2) === s
         @test length(dst) == s + 1
         @test dst[1:s] == data
         @test dst_buffer[end] == 0x00
 
         @test decode(d, encoded) == data
     end
+end
 
-    # can_concatenate tests
-    if can_concatenate(c)
-        a = rand(UInt8, 100*step(srange))
-        b = rand(UInt8, 200*step(srange))
-        @test decode(d, [encode(e, a); encode(e, b);]) == [a; b;]
-        @test decode(d, [encode(e, UInt8[]); encode(e, UInt8[]);]) == UInt8[]
+function rand_test_data(s::Int64)::Vector{UInt8}
+    choice = rand(1:4)
+    if choice == 1
+        rand(UInt8, s)
+    elseif choice == 2
+        zeros(UInt8, s)
+    elseif choice == 3
+        ones(UInt8, s)
+    elseif choice == 4
+        rand(0x00:0x0f, s)
     end
 end
 

diff --git a/ChunkCodecs/Project.toml b/ChunkCodecs/Project.toml
@@ -21,7 +21,7 @@ ChunkCodecLibZstd = {path = "../ChunkCodecLibZstd"}
 
 [compat]
 ChunkCodecLibBlosc = "0.1"
-ChunkCodecCore = "0.1"
+ChunkCodecCore = "0.2"
 ChunkCodecLibBzip2 = "0.1"
 ChunkCodecLibLz4 = "0.1"
 ChunkCodecLibZlib = "0.1"

diff --git a/test/Project.toml b/test/Project.toml
@@ -7,20 +7,23 @@ ChunkCodecLibBzip2 = "2b723af9-f480-4e8d-a1e4-4a9f5a906122"
 ChunkCodecLibLz4 = "7e9cc85e-5614-42a3-ad86-b78f920b38a5"
 ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8"
 ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398"
+ChunkCodecTests = "06b1ce50-b741-4199-b118-ba5fe1a70fa7"
 ChunkCodecs = "cea5061e-088a-433e-a623-54fa23d14d8f"
 CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Supposition = "5a0628fe-1738-4658-9b6d-0b7605a9755b"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [sources]
-ChunkCodecs = {path = "../ChunkCodecs"}
 ChunkCodecCore = {path = "../ChunkCodecCore"}
 ChunkCodecLibBlosc = {path = "../ChunkCodecLibBlosc"}
 ChunkCodecLibBzip2 = {path = "../ChunkCodecLibBzip2"}
 ChunkCodecLibLz4 = {path = "../ChunkCodecLibLz4"}
 ChunkCodecLibZlib = {path = "../ChunkCodecLibZlib"}
 ChunkCodecLibZstd = {path = "../ChunkCodecLibZstd"}
+ChunkCodecTests = {path = "../ChunkCodecTests"}
+ChunkCodecs = {path = "../ChunkCodecs"}
diff --git a/test/imagecodecs-compat.jl b/test/imagecodecs-compat.jl
@@ -1,5 +1,6 @@
 using PythonCall
 using ChunkCodecs
+using ChunkCodecTests: rand_test_data
 using Test
 
 codecs = [
@@ -28,17 +29,7 @@ codecs = [
         im_dec(x) = pyconvert(Vector, im_dec_funct(x; out=zeros(UInt8, s), im_options[2]...))
         jl_dec(x) = decode(codec(jl_options), x; size_hint=s)
         jl_enc(x) = encode(jl_options, x)
-        # generate data
-        local choice = rand(1:4)
-        local data = if choice == 1
-            rand(UInt8, s)
-        elseif choice == 2
-            zeros(UInt8, s)
-        elseif choice == 3
-            ones(UInt8, s)
-        elseif choice == 4
-            rand(0x00:0x0f, s)
-        end
+        local data = rand_test_data(s)
         has_encode, has_decode = if length(im_options) ≤ 2
             true, true
         else