merging with latest main

.azure/pipelines/azure-pipelines-external-release.yml

@@ -62,12 +62,14 @@ jobs:
   - task: EsrpCodeSigning@5
     displayName: Sign the binaries for nuget and zipped files
     inputs:
-      ConnectedServiceName: 'GarnetCodeSigningOneCert'
-      AppRegistrationClientId: '19bbb452-ce7a-4b3d-bdc4-42a0090e797b'
-      AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
-      AuthAKVName: 'GarnetCodeSignKV'
-      AuthCertName: 'Garnet-CodeSign-AuthCert'
-      AuthSignCertName: 'Garnet-CodeSigning-SigningCertificate'
+      ConnectedServiceName: 'GarnetCodeSignConn'
+      UseMSIAuthentication: false 
+      AppRegistrationClientId: 'ff81d1e2-748b-49e1-a771-b57246668c2b'
+      AppRegistrationTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
+      AuthAKVName: 'GarnetCodeSignKeyVault'
+      AuthSignCertName: 'garnet-codesign-signing-cert'
+      AuthCertName: 'garnet-codesign-auth-cert'
+      ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2'
       FolderPath: .
       Pattern: Garnet*.dll,Tsavorite*.dll,Garnet*.exe,HdrHistogram.dll,native_device.dll,*Lua.dll
       signConfigType: inlineSignParams
@@ -112,12 +114,14 @@ jobs:
   - task: EsrpCodeSigning@5
     displayName: Sign the NuGet Packages 
     inputs:
-      ConnectedServiceName: 'GarnetCodeSigningOneCert'
-      AppRegistrationClientId: '19bbb452-ce7a-4b3d-bdc4-42a0090e797b'
-      AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
-      AuthAKVName: 'GarnetCodeSignKV'
-      AuthCertName: 'Garnet-CodeSign-AuthCert'
-      AuthSignCertName: 'Garnet-CodeSigning-SigningCertificate'
+      ConnectedServiceName: 'GarnetCodeSignConn'
+      UseMSIAuthentication: false 
+      AppRegistrationClientId: 'ff81d1e2-748b-49e1-a771-b57246668c2b'
+      AppRegistrationTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
+      AuthAKVName: 'GarnetCodeSignKeyVault'
+      AuthSignCertName: 'garnet-codesign-signing-cert'
+      AuthCertName: 'garnet-codesign-auth-cert'
+      ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2'
       FolderPath: $(Build.ArtifactStagingDirectory)
       Pattern: Microsoft.Garnet.*.nupkg, garnet-server.*.nupkg
       signConfigType: inlineSignParams

.github/workflows/ci-bdnbenchmark.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Apply filter
-      uses: dorny/paths-filter@v3
+      uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36   #v3  for security reasons have pinned tag (commit SHA) for 3rd party
       id: filter
       with:
         filters: |
@@ -42,7 +42,7 @@ jobs:
         os: [ ubuntu-latest, windows-latest ]
         framework: [ 'net8.0' ]
         configuration: [ 'Release' ]
-        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts', 'Lua.LuaScriptCacheOperations','Lua.LuaRunnerOperations','Operations.CustomOperations', 'Operations.RawStringOperations', 'Operations.ScriptOperations', 'Operations.ModuleOperations', 'Operations.PubSubOperations', 'Network.BasicOperations', 'Network.RawStringOperations' ]
+        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Operations.SortedSetOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts', 'Lua.LuaScriptCacheOperations','Lua.LuaRunnerOperations','Operations.CustomOperations', 'Operations.RawStringOperations', 'Operations.ScriptOperations', 'Operations.ModuleOperations', 'Operations.PubSubOperations', 'Network.BasicOperations', 'Network.RawStringOperations' ]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -76,7 +76,7 @@ jobs:
 
       # Run `github-action-benchmark` action for the Continuous Benchmarking Charts (https://microsoft.github.io/garnet/charts/)
       - name: Store benchmark result for charts
-        uses: benchmark-action/github-action-benchmark@v1
+        uses: benchmark-action/github-action-benchmark@d48d326b4ca9ba73ca0cd0d59f108f9e02a381c7 # v1  for security reasons have pinned tag (commit SHA) for 3rd party
         with:
           name: ${{matrix.test}} (${{matrix.os}}  ${{matrix.framework}} ${{matrix.configuration}})
           tool: 'benchmarkdotnet'

.github/workflows/ci.yml

@@ -12,6 +12,9 @@ env:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
   DOTNET_NOLOGO: true
 
+permissions:
+  contents: read
+
 jobs:
   changes:
     name: Check for changes
@@ -27,7 +30,7 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Apply filter
-      uses: dorny/paths-filter@v3
+      uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36   #v3  for security reasons have pinned tag (commit SHA) for 3rd party
       id: filter
       with:
         filters: |

.github/workflows/deploy-website.yml

@@ -50,7 +50,7 @@ jobs:
       # Popular action to deploy to GitHub Pages:
       # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus
       - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v4
+        uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e    #v4 - for security reasons have pinned tag (commit SHA) for 3rd party
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           # Build output to publish to the `gh-pages` branch:

.github/workflows/docker-linux.yml

@@ -41,7 +41,7 @@ jobs:
       -
         name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@8e1d5461f02b7886d3c1a774bfbd873650445aa2 # was v5 but now v6 with this commit for security reasons have pinned tag (commit SHA) for 3rd party
         with:
           images: ${{ matrix.image }}
           tags: |
@@ -56,21 +56,22 @@ jobs:
             type=raw,value=latest,enable={{is_default_branch}}
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25  # v3 for security reasons have pinned tag (commit SHA) for 3rd party
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3 for security reasons have pinned tag (commit SHA) for 3rd party
       -
         name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3
+        uses: docker/login-action@327cd5a69de6c009b9ce71bce8395f28e651bf99  # was v3 but now v6 with this commit for security reasons have pinned tag (commit SHA) for 3rd party
+
         with:
             registry: ghcr.io
             username: ${{ github.actor }}
             password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v5 for security reasons have pinned tag (commit SHA) for 3rd party
         with:
           file: ${{ matrix.dockerfile }}
           provenance: false

.github/workflows/docker-windows.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@8e1d5461f02b7886d3c1a774bfbd873650445aa2 # was v5 but now v6 with this commit for security reasons have pinned tag (commit SHA) for 3rd party
         with:
           images: ghcr.io/${{ github.repository }}-nanoserver-ltsc2022
           tags: |
@@ -39,7 +39,7 @@ jobs:
       
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3
+        uses: docker/login-action@327cd5a69de6c009b9ce71bce8395f28e651bf99  # was v3 but now v6 with this commit for security reasons have pinned tag (commit SHA) for 3rd party
         with:
             registry: ghcr.io
             username: ${{ github.actor }}

.github/workflows/helm-chart.yml

@@ -28,7 +28,7 @@ jobs:
           git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
 
       - name: Install helm
-        uses: azure/setup-helm@v4
+        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4 for security reasons have pinned tag (commit SHA) for 3rd party
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
@@ -63,7 +63,7 @@ jobs:
           git checkout -- charts/garnet/Chart.yaml
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v7
+        uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f # v7 for security reasons have pinned tag (commit SHA) for 3rd party
         with:
           add-paths: charts/garnet/README.md
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -98,4 +98,3 @@ jobs:
             fi
             helm push "${pkg}" "oci://ghcr.io/${GITHUB_REPOSITORY_OWNER}/helm-charts"
           done
-

.github/workflows/nightly.yml

@@ -8,6 +8,9 @@ env:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
   DOTNET_NOLOGO: true
 
+permissions:
+  contents: read
+
 jobs:
   build-test-garnet:
     name: Garnet

Directory.Build.props

@@ -12,6 +12,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
+	  <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(TF_BUILD)' == 'true' or '$(GITHUB_ACTIONS)' == 'true'">

Dockerfile.ubuntu

@@ -30,11 +30,15 @@ COPY test/testcerts test/testcerts
 WORKDIR /src/main/GarnetServer
 RUN dotnet publish -a $TARGETARCH -c Release -o /app --no-restore --self-contained false -f net8.0 -p:EnableSourceLink=false -p:EnableSourceControlManagerQueries=false
 
+# Separate stage to install libaio1 on amd64 (avoiding QEMU issues on arm64)
+FROM --platform=linux/amd64 ubuntu:22.04 AS builder
+RUN apt-get update && apt-get install -y libaio1
+
 # Final stage/image
 FROM mcr.microsoft.com/dotnet/runtime:8.0-jammy AS runtime
 
-# Install libaio
-RUN apt-get update && apt-get install -y libaio1 && rm -rf /var/lib/apt/lists/*
+# Copy libaio1 from amd64 builder stage (avoiding installation under QEMU)
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libaio.so.1 /usr/lib/x86_64-linux-gnu/
 
 RUN mkdir /data /app \
     && chown -R $APP_UID:$APP_UID /data /app

Version.props

@@ -1,6 +1,6 @@
 <Project>
-	<!-- Versioning property for builds and packages -->
+	<!-- VersionPrefix property for builds and packages -->
 	<PropertyGroup>
-		<VersionPrefix>1.0.54</VersionPrefix>
+		<VersionPrefix>1.0.56</VersionPrefix>
 	</PropertyGroup>
 </Project>

benchmark/BDN.benchmark/Cluster/ClusterContext.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using System.Net;
 using System.Runtime.InteropServices;
 using System.Text;
 using BDN.benchmark.CustomProcs;
@@ -34,7 +35,7 @@ public void SetupSingleInstance(bool disableSlotVerification = false)
             {
                 QuietMode = true,
                 EnableCluster = !disableSlotVerification,
-                Port = port,
+                EndPoint = new IPEndPoint(IPAddress.Loopback, port),
                 CleanClusterConfig = true,
             };
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))

benchmark/BDN.benchmark/Embedded/EmbeddedNetworkHandler.cs

@@ -21,6 +21,12 @@ public EmbeddedNetworkHandler(GarnetServerEmbedded serverHook, EmbeddedNetworkSe
 
         public override string RemoteEndpointName => throw new NotImplementedException();
         public override string LocalEndpointName => throw new NotImplementedException();
+
+        public override bool IsLocalConnection()
+        {
+            return false;
+        }
+
         public override void Dispose()
         {
             DisposeImpl();

benchmark/BDN.benchmark/Embedded/EmbeddedNetworkSender.cs

@@ -48,6 +48,11 @@ public EmbeddedNetworkSender()
 
         public string LocalEndpointName => "";
 
+        public bool IsLocalConnection()
+        {
+            return true;
+        }
+
         /// <summary>
         /// Cleanup this DummyNetworkSender instance
         /// </summary>

benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs

@@ -15,7 +15,7 @@ namespace Embedded.server
     internal sealed class EmbeddedRespServer : GarnetServer
     {
         readonly GarnetServerEmbedded garnetServerEmbedded;
-        readonly SubscribeBroker<SpanByte, SpanByte, IKeySerializer<SpanByte>> subscribeBroker;
+        readonly SubscribeBroker subscribeBroker;
 
         /// <summary>
         /// Creates an EmbeddedRespServer instance
@@ -27,8 +27,7 @@ public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory
         {
             this.garnetServerEmbedded = server;
             this.subscribeBroker = opts.DisablePubSub ? null :
-                new SubscribeBroker<SpanByte, SpanByte, IKeySerializer<SpanByte>>(
-                    new SpanByteKeySerializer(),
+                new SubscribeBroker(
                     null,
                     opts.PubSubPageSizeBytes(),
                     opts.SubscriberRefreshFrequencyMs,

benchmark/BDN.benchmark/Embedded/GarnetServerEmbedded.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Net;
 using System.Net.Security;
 using System.Threading;
 using Garnet.common;
@@ -13,7 +14,7 @@ namespace Embedded.server
 {
     internal class GarnetServerEmbedded : GarnetServerBase, IServerHook
     {
-        public GarnetServerEmbedded() : base("0.0.0.0", 0, 1 << 10)
+        public GarnetServerEmbedded() : base(new IPEndPoint(IPAddress.Loopback, 0), 1 << 10)
         {
         }
 
@@ -36,7 +37,7 @@ public EmbeddedNetworkHandler CreateNetworkHandler(SslClientAuthenticationOption
                             throw new Exception("Unable to add handler to dictionary");
 
                         handler.Start(tlsOptions, remoteEndpointName);
-                        incr_conn_recv();
+                        IncrementConnectionsReceived();
                         return handler;
                     }
                     catch (Exception ex)
@@ -59,7 +60,7 @@ public void DisposeMessageConsumer(INetworkHandler session)
             if (activeHandlers.TryRemove(session, out _))
             {
                 Interlocked.Decrement(ref activeHandlerCount);
-                incr_conn_disp();
+                IncrementConnectionsDisposed();
                 try
                 {
                     session.Session?.Dispose();

benchmark/BDN.benchmark/Lua/LuaParams.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-using BenchmarkDotNet.Code;
 using Garnet.server;
 
 namespace BDN.benchmark.Lua
@@ -14,25 +13,38 @@ public readonly struct LuaParams
         public readonly LuaMemoryManagementMode Mode { get; }
         public readonly bool MemoryLimit { get; }
 
+        public readonly TimeSpan? Timeout { get; }
+
         /// <summary>
         /// Constructor
         /// </summary>
-        public LuaParams(LuaMemoryManagementMode mode, bool memoryLimit)
+        public LuaParams(LuaMemoryManagementMode mode, bool memoryLimit, TimeSpan? timeout = null)
         {
             Mode = mode;
             MemoryLimit = memoryLimit;
+            Timeout = timeout;
         }
 
         /// <summary>
         /// Get the equivalent <see cref="LuaOptions"/>.
         /// </summary>
         public LuaOptions CreateOptions()
-        => new(Mode, MemoryLimit ? "2m" : "");
+        => new(Mode, MemoryLimit ? "2m" : "", Timeout ?? System.Threading.Timeout.InfiniteTimeSpan);
 
         /// <summary>
         /// String representation
         /// </summary>
         public override string ToString()
-        => $"{Mode},{(MemoryLimit ? "Limit" : "None")}";
+        {
+            if (Timeout != null)
+            {
+                return $"{Mode},{(MemoryLimit ? "Limit" : "None")},{(Timeout == System.Threading.Timeout.InfiniteTimeSpan ? "-" : Timeout.ToString())}";
+            }
+            else
+            {
+                // Keep old format for benchmarks that don't care about timeouts
+                return $"{Mode},{(MemoryLimit ? "Limit" : "None")}";
+            }
+        }
     }
 }

benchmark/BDN.benchmark/Lua/LuaScriptCacheOperations.cs

@@ -46,7 +46,7 @@ public void GlobalSetup()
 
             server = new EmbeddedRespServer(new GarnetServerOptions() { EnableLua = true, QuietMode = true, LuaOptions = options });
             storeWrapper = server.StoreWrapper;
-            sessionScriptCache = new SessionScriptCache(storeWrapper, new GarnetNoAuthAuthenticator());
+            sessionScriptCache = new SessionScriptCache(storeWrapper, new GarnetNoAuthAuthenticator(), null);
             session = server.GetRespSession();
 
             outerHitDigest = GC.AllocateUninitializedArray<byte>(SessionScriptCache.SHA1Len, pinned: true);