diff --git a/aws-v1-localstack/src/main/scala/kinesis4cats/localstack/aws/v1/AwsClients.scala b/aws-v1-localstack/src/main/scala/kinesis4cats/localstack/aws/v1/AwsClients.scala index 326e7242..e03dd748 100644 --- a/aws-v1-localstack/src/main/scala/kinesis4cats/localstack/aws/v1/AwsClients.scala +++ b/aws-v1-localstack/src/main/scala/kinesis4cats/localstack/aws/v1/AwsClients.scala @@ -17,8 +17,6 @@ package kinesis4cats.localstack package aws.v1 -import scala.concurrent.duration._ - import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ @@ -28,7 +26,6 @@ import com.amazonaws.services.dynamodbv2._ import com.amazonaws.services.kinesis._ import com.amazonaws.services.kinesis.model._ -import kinesis4cats.compat.retry.RetryPolicies._ import kinesis4cats.compat.retry._ /** Helpers for constructing and leveraging AWS Java Client interfaces with @@ -138,27 +135,21 @@ object AwsClients { */ def createStream[F[_]]( client: AmazonKinesisAsync, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration - )(implicit F: Async[F]): F[Unit] = { - val retryPolicy = constantDelay(describeRetryDuration).join( - limitRetries(describeRetries) - ) + config: TestStreamConfig[F] + )(implicit F: Async[F]): F[Unit] = for { _ <- F.interruptibleMany( client.createStream( new CreateStreamRequest() - .withStreamName(streamName) - .withShardCount(shardCount) + .withStreamName(config.streamName) + .withShardCount(config.shardCount) .withStreamModeDetails( new StreamModeDetails().withStreamMode(StreamMode.PROVISIONED) ) ) ) _ <- retryingOnFailuresAndAllErrors( - retryPolicy, + config.describeRetryPolicy, (x: DescribeStreamSummaryResult) => F.pure( x.getStreamDescriptionSummary().getStreamStatus() === "ACTIVE" @@ -168,12 +159,11 @@ object AwsClients { )( F.interruptibleMany( client.describeStreamSummary( - new DescribeStreamSummaryRequest().withStreamName(streamName) + new DescribeStreamSummaryRequest().withStreamName(config.streamName) ) ) ) } yield () - } /** Deletes a stream and awaits for the stream deletion to be finalized * @@ -193,17 +183,12 @@ object AwsClients { */ def deleteStream[F[_]]( client: AmazonKinesisAsync, - streamName: String, - describeRetries: Int, - describeRetryDuration: FiniteDuration - )(implicit F: Async[F]): F[Unit] = { - val retryPolicy = constantDelay(describeRetryDuration).join( - limitRetries(describeRetries) - ) + config: TestStreamConfig[F] + )(implicit F: Async[F]): F[Unit] = for { - _ <- F.interruptibleMany(client.deleteStream(streamName)) + _ <- F.interruptibleMany(client.deleteStream(config.streamName)) _ <- retryingOnFailuresAndSomeErrors( - retryPolicy, + config.describeRetryPolicy, (x: Either[Throwable, DescribeStreamSummaryResult]) => F.pure( x.swap.exists { @@ -221,12 +206,11 @@ object AwsClients { )( F.interruptibleMany( client.describeStreamSummary( - new DescribeStreamSummaryRequest().withStreamName(streamName) + new DescribeStreamSummaryRequest().withStreamName(config.streamName) ) ).attempt ) } yield () - } /** A resource that does the following: * @@ -256,27 +240,18 @@ object AwsClients { * [[https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/kinesis/AmazonKinesisAsync.html AmazonKinesisAsync]] */ def kinesisStreamResource[F[_]]( - config: LocalstackConfig, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration + localstackConfig: LocalstackConfig, + streamsToCreate: List[TestStreamConfig[F]] )(implicit F: Async[F] ): Resource[F, AmazonKinesisAsync] = for { - client <- kinesisClientResource(config) - result <- Resource.make( - createStream( - client, - streamName, - shardCount, - describeRetries, - describeRetryDuration - ).as(client) - )(client => - deleteStream(client, streamName, describeRetries, describeRetryDuration) + client <- kinesisClientResource(localstackConfig) + _ <- streamsToCreate.traverse_(config => + Resource.make( + createStream(client, config).as(client) + )(client => deleteStream(client, config)) ) - } yield result + } yield client /** A resource that does the following: * @@ -307,22 +282,13 @@ object AwsClients { * [[https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/kinesis/AmazonKinesisAsync.html AmazonKinesisAsync]] */ def kinesisStreamResource[F[_]]( - streamName: String, - shardCount: Int, - prefix: Option[String] = None, - describeRetries: Int = 5, - describeRetryDuration: FiniteDuration = 500.millis + streamsToCreate: List[TestStreamConfig[F]], + prefix: Option[String] = None )(implicit F: Async[F] ): Resource[F, AmazonKinesisAsync] = for { - config <- LocalstackConfig.resource(prefix) - result <- kinesisStreamResource( - config, - streamName, - shardCount, - describeRetries, - describeRetryDuration - ) + localstackConfig <- LocalstackConfig.resource(prefix) + result <- kinesisStreamResource(localstackConfig, streamsToCreate) } yield result /** Builds a diff --git a/aws-v2-localstack/src/main/scala/kinesis4cats/localstack/aws/v2/AwsClients.scala b/aws-v2-localstack/src/main/scala/kinesis4cats/localstack/aws/v2/AwsClients.scala index 08ceb253..6f4668e4 100644 --- a/aws-v2-localstack/src/main/scala/kinesis4cats/localstack/aws/v2/AwsClients.scala +++ b/aws-v2-localstack/src/main/scala/kinesis4cats/localstack/aws/v2/AwsClients.scala @@ -17,8 +17,6 @@ package kinesis4cats.localstack package aws.v2 -import scala.concurrent.duration._ - import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ @@ -32,7 +30,6 @@ import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.awssdk.services.kinesis.model._ import software.amazon.awssdk.utils.AttributeMap -import kinesis4cats.compat.retry.RetryPolicies._ import kinesis4cats.compat.retry._ /** Helpers for constructing and leveraging AWS Java Client interfaces with @@ -166,22 +163,16 @@ object AwsClients { */ def createStream[F[_]]( client: KinesisAsyncClient, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration - )(implicit F: Async[F]): F[Unit] = { - val retryPolicy = constantDelay(describeRetryDuration).join( - limitRetries(describeRetries) - ) + config: TestStreamConfig[F] + )(implicit F: Async[F]): F[Unit] = for { _ <- F.fromCompletableFuture( F.delay( client.createStream( CreateStreamRequest .builder() - .streamName(streamName) - .shardCount(shardCount) + .streamName(config.streamName) + .shardCount(config.shardCount) .streamModeDetails( StreamModeDetails .builder() @@ -193,7 +184,7 @@ object AwsClients { ) ) _ <- retryingOnFailuresAndAllErrors( - retryPolicy, + config.describeRetryPolicy, (x: DescribeStreamSummaryResponse) => F.pure( x.streamDescriptionSummary() @@ -207,14 +198,13 @@ object AwsClients { client.describeStreamSummary( DescribeStreamSummaryRequest .builder() - .streamName(streamName) + .streamName(config.streamName) .build() ) ) ) ) } yield () - } /** Deletes a stream and awaits for the stream deletion to be finalized * @@ -234,23 +224,18 @@ object AwsClients { */ def deleteStream[F[_]]( client: KinesisAsyncClient, - streamName: String, - describeRetries: Int, - describeRetryDuration: FiniteDuration - )(implicit F: Async[F]): F[Unit] = { - val retryPolicy = constantDelay(describeRetryDuration).join( - limitRetries(describeRetries) - ) + config: TestStreamConfig[F] + )(implicit F: Async[F]): F[Unit] = for { _ <- F.fromCompletableFuture( F.delay( client.deleteStream( - DeleteStreamRequest.builder().streamName(streamName).build() + DeleteStreamRequest.builder().streamName(config.streamName).build() ) ) ) _ <- retryingOnFailuresAndSomeErrors( - retryPolicy, + config.describeRetryPolicy, (x: Either[Throwable, DescribeStreamSummaryResponse]) => F.pure( x.swap.exists { @@ -271,14 +256,13 @@ object AwsClients { client.describeStreamSummary( DescribeStreamSummaryRequest .builder() - .streamName(streamName) + .streamName(config.streamName) .build() ) ) ).attempt ) } yield () - } /** A resource that does the following: * @@ -308,27 +292,18 @@ object AwsClients { * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] */ def kinesisStreamResource[F[_]]( - config: LocalstackConfig, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration + localstackConfig: LocalstackConfig, + streamsToCreate: List[TestStreamConfig[F]] )(implicit F: Async[F] ): Resource[F, KinesisAsyncClient] = for { - client <- kinesisClientResource(config) - result <- Resource.make( - createStream( - client, - streamName, - shardCount, - describeRetries, - describeRetryDuration - ).as(client) - )(client => - deleteStream(client, streamName, describeRetries, describeRetryDuration) + client <- kinesisClientResource(localstackConfig) + _ <- streamsToCreate.traverse_(config => + Resource.make(createStream(client, config).as(client))(client => + deleteStream(client, config) + ) ) - } yield result + } yield client /** A resource that does the following: * @@ -359,22 +334,13 @@ object AwsClients { * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] */ def kinesisStreamResource[F[_]]( - streamName: String, - shardCount: Int, - prefix: Option[String] = None, - describeRetries: Int = 5, - describeRetryDuration: FiniteDuration = 500.millis + streamsToCreate: List[TestStreamConfig[F]], + prefix: Option[String] = None )(implicit F: Async[F] ): Resource[F, KinesisAsyncClient] = for { - config <- LocalstackConfig.resource(prefix) - result <- kinesisStreamResource( - config, - streamName, - shardCount, - describeRetries, - describeRetryDuration - ) + localstackConfig <- LocalstackConfig.resource(prefix) + result <- kinesisStreamResource(localstackConfig, streamsToCreate) } yield result /** Builds a diff --git a/build.sbt b/build.sbt index 2fd1598f..e449ec3d 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,10 @@ import laika.rewrite.link._ lazy val compat = projectMatrix .settings( - description := "Code to maintain compatability across major scala versions" + description := "Code to maintain compatability across major scala versions", + scalacOptions --= Seq("-deprecation", "-Xlint:deprecation", "-Xsource:3"), + Compile / doc / sources := Seq.empty, + Compile / packageDoc / publishArtifact := false ) .jvmPlatform(allScalaVersions) .nativePlatform(allScalaVersions) @@ -475,7 +478,6 @@ lazy val unidocs = projectMatrix moduleName := name.value, ScalaUnidoc / unidoc / unidocProjectFilter := inProjects( List( - compat, shared, `shared-circe`, `shared-ciris`, diff --git a/docs/client/circe.md b/docs/client/circe.md index e0292939..d621b2d2 100644 --- a/docs/client/circe.md +++ b/docs/client/circe.md @@ -12,10 +12,9 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-client-logging-ci ```scala mdoc:compile-only import cats.effect._ -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.client.KinesisClient import kinesis4cats.client.logging.instances.circe._ -KinesisClient[IO](KinesisAsyncClient.builder().build(), kinesisClientCirceEncoders) +KinesisClient.Builder.default[IO].withLogEncoders(kinesisClientCirceEncoders).build ``` diff --git a/docs/client/getting-started.md b/docs/client/getting-started.md index 079e6da5..d2439205 100644 --- a/docs/client/getting-started.md +++ b/docs/client/getting-started.md @@ -13,14 +13,13 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-client" % "@VERSI ```scala mdoc:compile-only import cats.effect._ import software.amazon.awssdk.core.SdkBytes -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.awssdk.services.kinesis.model._ import kinesis4cats.client.KinesisClient object MyApp extends IOApp { override def run(args: List[String]) = - KinesisClient[IO](KinesisAsyncClient.builder().build()).use(client => + KinesisClient.Builder.default[IO].build.use(client => for { _ <- client.createStream( CreateStreamRequest @@ -58,28 +57,27 @@ This module provides an implementation of that interface, backed by the @:source ```scala mdoc:compile-only import cats.data.NonEmptyList import cats.effect._ -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.client.producer.KinesisProducer import kinesis4cats.producer._ import kinesis4cats.models.StreamNameOrArn object MyApp extends IOApp { - override def run(args: List[String]) = - KinesisProducer.instance[IO]( - Producer.Config.default(StreamNameOrArn.Name("my-stream")), - KinesisAsyncClient.builder().build() - ).use(producer => - for { - _ <- producer.put( - NonEmptyList.of( - Record("my-data".getBytes(), "some-partition-key"), - Record("my-data-2".getBytes(), "some-partition-key-2"), - Record("my-data-3".getBytes(), "some-partition-key-3"), - ) - ) - } yield ExitCode.Success - ) + override def run(args: List[String]) = + KinesisProducer.Builder + .default[IO](StreamNameOrArn.Name("my-stream")) + .build + .use(producer => + for { + _ <- producer.put( + NonEmptyList.of( + Record("my-data".getBytes(), "some-partition-key"), + Record("my-data-2".getBytes(), "some-partition-key-2"), + Record("my-data-3".getBytes(), "some-partition-key-3"), + ) + ) + } yield ExitCode.Success + ) } ``` @@ -89,28 +87,26 @@ This package provides a [KPL-like](https://github.com/awslabs/amazon-kinesis-pro ```scala mdoc:compile-only import cats.effect._ -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.client.producer.fs2.FS2KinesisProducer import kinesis4cats.producer._ -import kinesis4cats.producer.fs2._ import kinesis4cats.models.StreamNameOrArn object MyApp extends IOApp { override def run(args: List[String]) = - FS2KinesisProducer.instance[IO]( - FS2Producer.Config.default(StreamNameOrArn.Name("my-stream")), - KinesisAsyncClient.builder().build() - ).use(producer => + FS2KinesisProducer.Builder + .default[IO](StreamNameOrArn.Name("my-stream")) + .build + .use(producer => for { - _ <- producer.put( - Record("my-data".getBytes(), "some-partition-key") + _ <- producer.put( + Record("my-data".getBytes(), "some-partition-key") ) - _ <- producer.put( - Record("my-data-2".getBytes(), "some-partition-key-2") + _ <- producer.put( + Record("my-data-2".getBytes(), "some-partition-key-2") ) - _ <- producer.put( - Record("my-data-3".getBytes(), "some-partition-key-3") + _ <- producer.put( + Record("my-data-3".getBytes(), "some-partition-key-3") ) } yield ExitCode.Success ) diff --git a/docs/client/localstack.md b/docs/client/localstack.md index 2a8557c9..7aa4e37c 100644 --- a/docs/client/localstack.md +++ b/docs/client/localstack.md @@ -12,21 +12,44 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kinesis-client-lo ```scala mdoc:compile-only import cats.effect.IO +import cats.effect.syntax.all._ import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.client.producer.fs2.localstack.LocalstackFS2KinesisProducer +import kinesis4cats.models.StreamNameOrArn +import kinesis4cats.localstack.TestStreamConfig // Load a KinesisClient as a Resource -LocalstackKinesisClient.clientResource[IO]() +LocalstackKinesisClient.Builder + .default[IO]() + .toResource + .flatMap(_.build) // Load a KinesisClient as a Resource. -// Also creates and deletes a stream during it's usage. Useful for tests. -LocalstackKinesisClient.streamResource[IO]("my-stream", 1) +// Also creates and deletes streams during it's usage. Useful for tests. +LocalstackKinesisClient.Builder + .default[IO]() + .toResource + .flatMap(x => + x.withStreamsToCreate( + List( + TestStreamConfig.default[IO]("my-stream", 1), + TestStreamConfig.default[IO]("my-stream-2", 1), + ) + ) + .build + ) // Load a KinesisProducer as a resource -LocalstackKinesisProducer.resource[IO]("my-stream") +LocalstackKinesisProducer.Builder + .default[IO](StreamNameOrArn.Name("my-stream")) + .toResource + .flatMap(_.build) // Load a FS2KinesisProducer as a resource -LocalstackFS2KinesisProducer.resource[IO]("my-stream") +LocalstackFS2KinesisProducer.Builder + .default[IO](StreamNameOrArn.Name("my-stream")) + .toResource + .flatMap(_.build) ``` diff --git a/docs/kcl/circe.md b/docs/kcl/circe.md index 7fb4d729..9fbe2dd1 100644 --- a/docs/kcl/circe.md +++ b/docs/kcl/circe.md @@ -13,9 +13,6 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kcl-logging-circe ```scala mdoc:compile-only import cats.effect._ import cats.syntax.all._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.kcl._ @@ -24,25 +21,17 @@ import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { override def run(args: List[String]) = for { - kinesisClient <- Resource.fromAutoCloseable( - IO(KinesisAsyncClient.builder().build()) - ) - dynamoClient <- Resource.fromAutoCloseable( - IO(DynamoDbAsyncClient.builder().build()) - ) - cloudWatchClient <- Resource.fromAutoCloseable( - IO(CloudWatchAsyncClient.builder().build()) - ) - consumer <- KCLConsumer.configsBuilder[IO]( - kinesisClient, - dynamoClient, - cloudWatchClient, - new SingleStreamTracker("my-stream"), + consumerBuilder <- KCLConsumer.Builder.default[IO]( + new SingleStreamTracker("my-stream"), "my-app-name", - encoders = kclCirceEncoders - )((records: List[CommittableRecord[IO]]) => - records.traverse_(r => IO.println(r.data.asString)) - )() + ) + consumer <- consumerBuilder + .withCallback( + (records: List[CommittableRecord[IO]]) => + records.traverse_(r => IO.println(r.data.asString)) + ) + .configure(_.withLogEncoders(kclCirceEncoders)) + .build _ <- consumer.run() } yield () } diff --git a/docs/kcl/ciris.md b/docs/kcl/ciris.md index 8421923b..0ac8ca7d 100644 --- a/docs/kcl/ciris.md +++ b/docs/kcl/ciris.md @@ -13,9 +13,6 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kcl-ciris" % "@VE ```scala mdoc:compile-only import cats.effect._ import cats.syntax.all._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.kcl._ import kinesis4cats.kcl.ciris.KCLCiris @@ -23,21 +20,9 @@ import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { override def run(args: List[String]) = for { - kinesisClient <- Resource.fromAutoCloseable( - IO(KinesisAsyncClient.builder().build()) - ) - dynamoClient <- Resource.fromAutoCloseable( - IO(DynamoDbAsyncClient.builder().build()) - ) - cloudWatchClient <- Resource.fromAutoCloseable( - IO(CloudWatchAsyncClient.builder().build()) - ) - consumer <- KCLCiris.consumer[IO]( - kinesisClient, - dynamoClient, - cloudWatchClient - ){ case records: List[CommittableRecord[IO]] => - records.traverse_(r => IO.println(r.data.asString)) + consumer <- KCLCiris.consumer[IO](){ + case records: List[CommittableRecord[IO]] => + records.traverse_(r => IO.println(r.data.asString)) } _ <- consumer.run() } yield () @@ -191,19 +176,13 @@ Standard environment variables and system properties for configuring a @:source( ```scala mdoc:compile-only import cats.effect._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.kcl.fs2.ciris.KCLCirisFS2 import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { override def run(args: List[String]) = for { - kinesisClient <- Resource.fromAutoCloseable(IO(KinesisAsyncClient.builder().build())) - dynamoClient <- Resource.fromAutoCloseable(IO(DynamoDbAsyncClient.builder().build())) - cloudWatchClient <- Resource.fromAutoCloseable(IO(CloudWatchAsyncClient.builder().build())) - consumer <- KCLCirisFS2.consumer[IO](kinesisClient, dynamoClient, cloudWatchClient) + consumer <- KCLCirisFS2.consumer[IO]() _ <- consumer.stream() .flatMap(stream => stream diff --git a/docs/kcl/getting-started.md b/docs/kcl/getting-started.md index c749ee6b..f4720b85 100644 --- a/docs/kcl/getting-started.md +++ b/docs/kcl/getting-started.md @@ -13,9 +13,6 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kcl" % "@VERSION@ ```scala mdoc:compile-only import cats.effect._ import cats.syntax.all._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.kcl._ @@ -23,24 +20,14 @@ import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { override def run(args: List[String]) = for { - kinesisClient <- Resource.fromAutoCloseable( - IO(KinesisAsyncClient.builder().build()) + consumerBuilder <- KCLConsumer.Builder.default[IO]( + new SingleStreamTracker("my-stream"), + "my-app-name", ) - dynamoClient <- Resource.fromAutoCloseable( - IO(DynamoDbAsyncClient.builder().build()) - ) - cloudWatchClient <- Resource.fromAutoCloseable( - IO(CloudWatchAsyncClient.builder().build()) - ) - consumer <- KCLConsumer.configsBuilder[IO]( - kinesisClient, - dynamoClient, - cloudWatchClient, - new SingleStreamTracker("my-stream"), - "my-app-name" - )((records: List[CommittableRecord[IO]]) => - records.traverse_(r => IO.println(r.data.asString)) - )() + consumer <- consumerBuilder.withCallback( + (records: List[CommittableRecord[IO]]) => + records.traverse_(r => IO.println(r.data.asString)) + ).build _ <- consumer.run() } yield () } @@ -56,9 +43,6 @@ It is not recommended to use this in production as scaling the application becom import cats.effect._ import cats.effect.syntax.all._ import cats.syntax.all._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.common._ import kinesis4cats.client._ @@ -68,35 +52,26 @@ import kinesis4cats.kcl.multistream._ import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { - override def run(args: List[String]) = for { - kinesisClient <- KinesisClient[IO]( - KinesisAsyncClient.builder().build() - ) - dynamoClient <- Resource.fromAutoCloseable( - IO(DynamoDbAsyncClient.builder().build()) - ) - cloudWatchClient <- Resource.fromAutoCloseable( - IO(CloudWatchAsyncClient.builder().build()) - ) - streamArn1 = StreamArn(AwsRegion.US_EAST_1, "my-stream-1", "123456789012") - streamArn2 = StreamArn(AwsRegion.US_EAST_1, "my-stream-2", "123456789012") - position = InitialPositionInStreamExtended + override def run(args: List[String]) = { + val streamArn1 = StreamArn(AwsRegion.US_EAST_1, "my-stream-1", "123456789012") + val streamArn2 = StreamArn(AwsRegion.US_EAST_1, "my-stream-2", "123456789012") + val position = InitialPositionInStreamExtended .newInitialPosition(InitialPositionInStream.TRIM_HORIZON) - tracker <- MultiStreamTracker.noLeaseDeletionFromArns[IO]( - kinesisClient, - Map(streamArn1 -> position, streamArn2 -> position) - ).toResource - consumer <- KCLConsumer.configsBuilder[IO]( - kinesisClient.client, - dynamoClient, - cloudWatchClient, - tracker, - "my-app-name" - )((records: List[CommittableRecord[IO]]) => - records.traverse_(r => IO.println(r.data.asString)) - )() - _ <- consumer.run() - } yield () + for { + kinesisClient <- KinesisClient.Builder.default[IO].build + tracker <- MultiStreamTracker.noLeaseDeletionFromArns[IO]( + kinesisClient, + Map(streamArn1 -> position, streamArn2 -> position) + ).toResource + consumerBuilder <- KCLConsumer.Builder + .default[IO](tracker, "my-app-name") + consumer <- consumerBuilder.withCallback( + (records: List[CommittableRecord[IO]]) => + records.traverse_(r => IO.println(r.data.asString)) + ).build + _ <- consumer.run() + } yield () + } } ``` @@ -108,9 +83,6 @@ This package intends to be an enriched wrapper for the [KCL](https://docs.aws.am ```scala mdoc:compile-only import cats.effect._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.kcl.fs2.KCLConsumerFS2 @@ -118,22 +90,11 @@ import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { override def run(args: List[String]) = for { - kinesisClient <- Resource.fromAutoCloseable( - IO(KinesisAsyncClient.builder().build()) - ) - dynamoClient <- Resource.fromAutoCloseable( - IO(DynamoDbAsyncClient.builder().build()) - ) - cloudWatchClient <- Resource.fromAutoCloseable( - IO(CloudWatchAsyncClient.builder().build()) - ) - consumer <- KCLConsumerFS2.configsBuilder[IO]( - kinesisClient, - dynamoClient, - cloudWatchClient, + consumerBuilder <- KCLConsumerFS2.Builder.default[IO]( new SingleStreamTracker("my-stream"), - "my-app-name" - )() + "my-app-name", + ) + consumer <- consumerBuilder.build _ <- consumer .stream() .flatMap(stream => diff --git a/docs/kcl/http4s.md b/docs/kcl/http4s.md index e21368d4..7db1c3a8 100644 --- a/docs/kcl/http4s.md +++ b/docs/kcl/http4s.md @@ -17,9 +17,6 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kcl-http4s" % "@V import cats.effect._ import cats.syntax.all._ import com.comcast.ip4s._ -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.kcl._ @@ -28,24 +25,14 @@ import kinesis4cats.syntax.bytebuffer._ object MyApp extends ResourceApp.Forever { override def run(args: List[String]) = for { - kinesisClient <- Resource.fromAutoCloseable( - IO(KinesisAsyncClient.builder().build()) - ) - dynamoClient <- Resource.fromAutoCloseable( - IO(DynamoDbAsyncClient.builder().build()) - ) - cloudWatchClient <- Resource.fromAutoCloseable( - IO(CloudWatchAsyncClient.builder().build()) - ) - consumer <- KCLConsumer.configsBuilder[IO]( - kinesisClient, - dynamoClient, - cloudWatchClient, + consumerBuilder <- KCLConsumer.Builder.default[IO]( new SingleStreamTracker("my-stream"), - "my-app-name" - )((records: List[CommittableRecord[IO]]) => - records.traverse_(r => IO.println(r.data.asString)) - )() + "my-app-name", + ) + consumer <- consumerBuilder.withCallback( + (records: List[CommittableRecord[IO]]) => + records.traverse_(r => IO.println(r.data.asString)) + ).build _ <- KCLService.server[IO](consumer, port"8080", host"0.0.0.0") } yield () } diff --git a/docs/kcl/localstack.md b/docs/kcl/localstack.md index 9a0468eb..670e0e85 100644 --- a/docs/kcl/localstack.md +++ b/docs/kcl/localstack.md @@ -22,20 +22,26 @@ import kinesis4cats.syntax.bytebuffer._ val processRecords = (records: List[CommittableRecord[IO]]) => records.traverse_(record => IO.println(record.data.asString)) +// Builds a KCLConsumer as a Resource. +LocalstackKCLConsumer.Builder.default[IO]( + new SingleStreamTracker("my-stream"), + "my-app-name" +).flatMap(_.withCallback(processRecords).build) + // Runs a KCLConsumer as a Resource. Resource contains a Deferred value, //which completes when the consumer has begun to process records. -LocalstackKCLConsumer.kclConsumer[IO]( +LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker("my-stream"), "my-app-name" -)(processRecords) +).flatMap(_.withCallback(processRecords).build) // Runs a KCLConsumer as a Resource. Resource contains 2 things: // - A Deferred value, which completes when the consumer has begun to process records. // - A results Queue, which contains records received by the consumer -LocalstackKCLConsumer.kclConsumerWithResults[IO]( +LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker("my-stream"), "my-app-name" -)(processRecords) +).flatMap(_.withCallback(processRecords).build) ``` ## Usage - FS2 @@ -47,8 +53,8 @@ import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.kcl.fs2.localstack.LocalstackKCLConsumerFS2 // Runs a KCLConsumerFS2 as a Resource, which contains FS2 Streaming methods. -LocalstackKCLConsumerFS2.kclConsumer[IO]( +LocalstackKCLConsumerFS2.Builder.default[IO]( new SingleStreamTracker("my-stream"), "my-app-name" -) +).flatMap(_.build) ``` diff --git a/docs/kpl/circe.md b/docs/kpl/circe.md index 31eba45c..19b2c55f 100644 --- a/docs/kpl/circe.md +++ b/docs/kpl/circe.md @@ -13,8 +13,8 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kpl-logging-circe ```scala mdoc:compile-only import cats.effect._ -import kinesis4cats.kpl.logging.instances.circe +import kinesis4cats.kpl.logging.instances.circe._ import kinesis4cats.kpl.KPLProducer -KPLProducer[IO](encoders = circe.kplProducer) + KPLProducer.Builder.default[IO].withLogEncoders(kplCirceEncoders).build ``` diff --git a/docs/kpl/ciris.md b/docs/kpl/ciris.md index bbf6e05c..6ed6ecbd 100644 --- a/docs/kpl/ciris.md +++ b/docs/kpl/ciris.md @@ -60,7 +60,8 @@ All of the configuration below is optional. | `KPL_THREADING_MODEL`| `kpl.threading.model` | PER_REQUEST | Sets the threading model that the native process will use. Valid values are PER_REQUEST, POOLED | | `KPL_THREAD_POOL_SIZE`| `kpl.thread.pool.size` | | Sets the maximum number of threads that the native process' thread pool will be configured with. Must be a non-negative number. | | `KPL_USER_RECORD_TIMEOUT`| `kpl.user.record.timeout` | | Set the value when the user submitted records will be timed out at the Java layer. Valid values are durations (e.g. `1 second`) | - +| `KPL_GRACEFUL_SHUTDOWN_FLUSH_ATTEMPTS`| `kpl.graceful.shutdown.flush.attempts` | 5 | Set the amount of times that the KPL will perform a flush() during its graceful shutdown process. | +| `KPL_GRACEFUL_SHUTDOWN_FLUSH_INTERVAL`| `kpl.graceful.shutdown.flush.attempts` | 500 millis | Set the duration between flush() commands during its graceful shutdown process. | ### Glue The KPL can be configured to leverage the [AWS Glue](https://docs.aws.amazon.com/streams/latest/dev/kpl-with-schemaregistry.html) schema registry. There are some available configuration values for this. diff --git a/docs/kpl/getting-started.md b/docs/kpl/getting-started.md index 76f352d0..3dde47c0 100644 --- a/docs/kpl/getting-started.md +++ b/docs/kpl/getting-started.md @@ -20,7 +20,7 @@ import kinesis4cats.kpl.KPLProducer object MyApp extends IOApp { override def run(args: List[String]) = { - KPLProducer[IO]().use(kpl => + KPLProducer.Builder.default[IO].build.use(kpl => for { _ <- kpl.put( new UserRecord( diff --git a/docs/kpl/localstack.md b/docs/kpl/localstack.md index 55a10893..6e671fd2 100644 --- a/docs/kpl/localstack.md +++ b/docs/kpl/localstack.md @@ -12,13 +12,28 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-kpl-localstack" % ```scala mdoc:compile-only import cats.effect.IO +import cats.effect.syntax.all._ +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.kpl.localstack.LocalstackKPLProducer // Load a KPLProducer as a resource -LocalstackKPLProducer.producer[IO]() +LocalstackKPLProducer.Builder + .default[IO]() + .toResource + .flatMap(_.build) // Load a KPLProducer as a resource. -// Also creates and deletes a stream during it's usage. Useful for tests. -LocalstackKPLProducer.producerWithStream[IO]("my-stream", 1) +// Also creates and deletes streams during it's usage. Useful for tests. +LocalstackKPLProducer.Builder + .default[IO]() + .toResource + .flatMap(x => + x.withStreamsToCreate( + List( + TestStreamConfig.default[IO]("my-stream", 1), + TestStreamConfig.default[IO]("my-stream-2", 1) + ) + ).build + ) ``` diff --git a/docs/localstack/aws-v1.md b/docs/localstack/aws-v1.md index 10f6165f..52fd1f90 100644 --- a/docs/localstack/aws-v1.md +++ b/docs/localstack/aws-v1.md @@ -14,6 +14,7 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-aws-v1-localstack import cats.effect.IO import kinesis4cats.localstack.aws.v1.AwsClients +import kinesis4cats.localstack.TestStreamConfig // Load an AmazonKinesisAsync as an effect AwsClients.kinesisClient[IO]() @@ -22,6 +23,11 @@ AwsClients.kinesisClient[IO]() AwsClients.kinesisClientResource[IO]() // Load a AmazonKinesisAsync as a resource. -// Also creates and deletes a stream during it's usage. Useful for tests. -AwsClients.kinesisStreamResource[IO]("my-stream", 1) +// Also creates and deletes streams during it's usage. Useful for tests. +AwsClients.kinesisStreamResource[IO]( + List( + TestStreamConfig.default[IO]("my-stream", 1), + TestStreamConfig.default[IO]("my-stream-2", 1) + ) +) ``` diff --git a/docs/localstack/aws-v2.md b/docs/localstack/aws-v2.md index e69e8381..347567bb 100644 --- a/docs/localstack/aws-v2.md +++ b/docs/localstack/aws-v2.md @@ -14,6 +14,7 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-aws-v2-localstack import cats.effect.IO import kinesis4cats.localstack.aws.v1.AwsClients +import kinesis4cats.localstack.TestStreamConfig // Load a KinesisAsyncClient as an effect AwsClients.kinesisClient[IO]() @@ -22,6 +23,11 @@ AwsClients.kinesisClient[IO]() AwsClients.kinesisClientResource[IO]() // Load a KinesisAsyncClient as a resource. -// Also creates and deletes a stream during it's usage. Useful for tests. -AwsClients.kinesisStreamResource[IO]("my-stream", 1) +// Also creates and deletes streams during it's usage. Useful for tests. +AwsClients.kinesisStreamResource[IO]( + List( + TestStreamConfig.default[IO]("my-stream", 1), + TestStreamConfig.default[IO]("my-stream-2", 1) + ) +) ``` diff --git a/docs/smithy4s/circe.md b/docs/smithy4s/circe.md index f810dd2f..539e56a6 100644 --- a/docs/smithy4s/circe.md +++ b/docs/smithy4s/circe.md @@ -15,18 +15,14 @@ import org.http4s.blaze.client.BlazeClientBuilder import org.typelevel.log4cats.slf4j.Slf4jLogger import smithy4s.aws._ -import kinesis4cats.smithy4s.client.producer.KinesisProducer +import kinesis4cats.smithy4s.client.KinesisClient import kinesis4cats.smithy4s.client.logging.instances.circe._ -import kinesis4cats.producer._ -import kinesis4cats.models.StreamNameOrArn -BlazeClientBuilder[IO].resource.flatMap(client => - KinesisProducer[IO]( - Producer.Config.default(StreamNameOrArn.Name("my-stream")), - client, - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO], - encoders = kinesisProducerCirceEncoders[IO] - ) - ) +BlazeClientBuilder[IO].resource.flatMap(underlying => + KinesisClient.Builder + .default[IO](underlying, AwsRegion.US_EAST_1) + .withLogger(Slf4jLogger.getLogger) + .withLogEncoders(kinesisClientCirceEncoders[IO]) + .build +) ``` diff --git a/docs/smithy4s/getting-started.md b/docs/smithy4s/getting-started.md index c5523d7a..e077a4b3 100644 --- a/docs/smithy4s/getting-started.md +++ b/docs/smithy4s/getting-started.md @@ -32,11 +32,10 @@ import kinesis4cats.smithy4s.client.KinesisClient object MyApp extends IOApp { override def run(args: List[String]) = (for { underlying <- BlazeClientBuilder[IO].resource - client <- KinesisClient[IO]( - underlying, - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] - ) + client <- KinesisClient.Builder + .default[IO](underlying, AwsRegion.US_EAST_1) + .withLogger(Slf4jLogger.getLogger) + .build } yield client).use(client => for { _ <- client.createStream(StreamName("my-stream"), Some(1)) @@ -77,12 +76,14 @@ import kinesis4cats.models.StreamNameOrArn object MyApp extends IOApp { override def run(args: List[String]) = BlazeClientBuilder[IO].resource.flatMap(client => - KinesisProducer[IO]( - Producer.Config.default(StreamNameOrArn.Name("my-stream")), - client, - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] - ) + KinesisProducer.Builder + .default[IO]( + StreamNameOrArn.Name("my-stream"), + client, + AwsRegion.US_EAST_1 + ) + .withLogger(Slf4jLogger.getLogger) + .build ).use(producer => for { _ <- producer.put( @@ -110,19 +111,20 @@ import org.typelevel.log4cats.slf4j.Slf4jLogger import smithy4s.aws._ import kinesis4cats.smithy4s.client.producer.fs2.FS2KinesisProducer -import kinesis4cats.producer.fs2._ import kinesis4cats.producer._ import kinesis4cats.models.StreamNameOrArn object MyApp extends IOApp { override def run(args: List[String]) = BlazeClientBuilder[IO].resource.flatMap(client => - FS2KinesisProducer[IO]( - FS2Producer.Config.default(StreamNameOrArn.Name("my-stream")), - client, - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] - ) + FS2KinesisProducer.Builder + .default[IO]( + StreamNameOrArn.Name("my-stream"), + client, + AwsRegion.US_EAST_1 + ) + .withLogger(Slf4jLogger.getLogger) + .build ).use(producer => for { _ <- producer.put( diff --git a/docs/smithy4s/localstack.md b/docs/smithy4s/localstack.md index 9f5b126c..e3b5ac05 100644 --- a/docs/smithy4s/localstack.md +++ b/docs/smithy4s/localstack.md @@ -12,10 +12,12 @@ libraryDependencies += "io.github.etspaceman" %% "kinesis4cats-smithy4s-client-l ```scala mdoc:compile-only import cats.effect._ +import cats.effect.syntax.all._ import org.http4s.blaze.client.BlazeClientBuilder import org.typelevel.log4cats.slf4j.Slf4jLogger import smithy4s.aws._ +import kinesis4cats.models.StreamNameOrArn import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient import kinesis4cats.smithy4s.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.smithy4s.client.producer.fs2.localstack.LocalstackFS2KinesisProducer @@ -24,11 +26,12 @@ val kinesisClientResource = for { underlying <- BlazeClientBuilder[IO] .withCheckEndpointAuthentication(false) .resource - client <- LocalstackKinesisClient.clientResource[IO]( - underlying, - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] - ) + client <- LocalstackKinesisClient.Builder + .default[IO](underlying, AwsRegion.US_EAST_1) + .toResource + .flatMap(x => + x.withLogger(Slf4jLogger.getLogger).build + ) } yield client // Load a KinesisProducer as a Resource @@ -36,12 +39,16 @@ val kinesisProducerResource = for { underlying <- BlazeClientBuilder[IO] .withCheckEndpointAuthentication(false) .resource - producer <- LocalstackKinesisProducer.resource[IO]( - underlying, - "my-stream", - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] - ) + producer <- LocalstackKinesisProducer.Builder + .default[IO]( + underlying, + AwsRegion.US_EAST_1, + StreamNameOrArn.Name("my-stream") + ) + .toResource + .flatMap(x => + x.withLogger(Slf4jLogger.getLogger).build + ) } yield producer // Load a FS2KinesisProducer as a Resource @@ -49,11 +56,15 @@ val fs2KinesisProducerResource = for { underlying <- BlazeClientBuilder[IO] .withCheckEndpointAuthentication(false) .resource - producer <- LocalstackFS2KinesisProducer.resource[IO]( - underlying, - "my-stream", - IO.pure(AwsRegion.US_EAST_1), - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] - ) + producer <- LocalstackFS2KinesisProducer.Builder + .default[IO]( + underlying, + AwsRegion.US_EAST_1, + StreamNameOrArn.Name("my-stream") + ) + .toResource + .flatMap(x => + x.withLogger(Slf4jLogger.getLogger).build + ) } yield producer ``` diff --git a/integration-tests/src/main/scalajvm/kinesis4cats/kcl/http4s/TestKCLService.scala b/integration-tests/src/main/scalajvm/kinesis4cats/kcl/http4s/TestKCLService.scala index 9fc148c7..c1ddc12b 100644 --- a/integration-tests/src/main/scalajvm/kinesis4cats/kcl/http4s/TestKCLService.scala +++ b/integration-tests/src/main/scalajvm/kinesis4cats/kcl/http4s/TestKCLService.scala @@ -29,11 +29,12 @@ import kinesis4cats.kcl.localstack.LocalstackKCLConsumer object TestKCLService extends ResourceApp.Forever { override def run(args: List[String]): Resource[IO, Unit] = for { streamName <- CirisReader.read[String](List("test", "stream")).resource[IO] - configAndResults <- LocalstackKCLConsumer.kclConfigWithResults[IO]( - new SingleStreamTracker(streamName), - s"test-kcl-service-spec-${Utils.randomUUIDString}" - )((_: List[CommittableRecord[IO]]) => IO.unit) - consumer = new KCLConsumer[IO](configAndResults.kclConfig) + consumer <- LocalstackKCLConsumer.Builder + .default[IO]( + new SingleStreamTracker(streamName), + s"test-kcl-service-spec-${Utils.randomUUIDString}" + ) + .flatMap(_.build) _ <- KCLService.server[IO](consumer, port"8080", host"0.0.0.0") } yield () diff --git a/integration-tests/src/test/scala/kinesis4cats/producer/ProducerSpec.scala b/integration-tests/src/test/scala/kinesis4cats/producer/ProducerSpec.scala index 0830af25..30b1a701 100644 --- a/integration-tests/src/test/scala/kinesis4cats/producer/ProducerSpec.scala +++ b/integration-tests/src/test/scala/kinesis4cats/producer/ProducerSpec.scala @@ -40,7 +40,6 @@ private[kinesis4cats] abstract class ProducerSpec[PutReq, PutRes, A] def aAsBytes(a: A): Array[Byte] def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[ProducerSpec.Resources[IO, PutReq, PutRes, A]]] @@ -48,36 +47,35 @@ private[kinesis4cats] abstract class ProducerSpec[PutReq, PutRes, A] def appName = streamName - fixture(3, appName).test("It should produce records end to end") { - resources => - for { - data <- IO(Arbitrary.arbitrary[TestData].take(50).toList) - records = NonEmptyList.fromListUnsafe( - data.map(x => - Record( - x.asJson.noSpaces.getBytes(), - Utils.randomUUIDString, - None, - None - ) + fixture(appName).test("It should produce records end to end") { resources => + for { + data <- IO(Arbitrary.arbitrary[TestData].take(50).toList) + records = NonEmptyList.fromListUnsafe( + data.map(x => + Record( + x.asJson.noSpaces.getBytes(), + Utils.randomUUIDString, + None, + None ) ) - _ <- resources.producer.put(records) - retryPolicy = limitRetries[IO](30).join(constantDelay(1.second)) - size <- retryingOnFailures( - retryPolicy, - (x: Int) => IO(x === 50), - noop[IO, Int] - )(resources.resultsQueue.size) - _ <- IO(assertEquals(size, 50)) - results <- resources.resultsQueue.tryTakeN(None) - resultRecords <- results.traverse { x => - IO.fromEither(decode[TestData](new String(aAsBytes(x)))) - } - } yield assert( - resultRecords.forall(data.contains), - s"res: ${resultRecords}\nexp: ${data}" ) + _ <- resources.producer.put(records) + retryPolicy = limitRetries[IO](30).join(constantDelay(1.second)) + size <- retryingOnFailures( + retryPolicy, + (x: Int) => IO(x === 50), + noop[IO, Int] + )(resources.resultsQueue.size) + _ <- IO(assertEquals(size, 50)) + results <- resources.resultsQueue.tryTakeN(None) + resultRecords <- results.traverse { x => + IO.fromEither(decode[TestData](new String(aAsBytes(x)))) + } + } yield assert( + resultRecords.forall(data.contains), + s"res: ${resultRecords}\nexp: ${data}" + ) } } diff --git a/integration-tests/src/test/scala/kinesis4cats/producer/fs2/FS2ProducerSpec.scala b/integration-tests/src/test/scala/kinesis4cats/producer/fs2/FS2ProducerSpec.scala index 8efe73ea..a289e7dd 100644 --- a/integration-tests/src/test/scala/kinesis4cats/producer/fs2/FS2ProducerSpec.scala +++ b/integration-tests/src/test/scala/kinesis4cats/producer/fs2/FS2ProducerSpec.scala @@ -41,7 +41,6 @@ private[kinesis4cats] abstract class FS2ProducerSpec[PutReq, PutRes, A] def aAsBytes(a: A): Array[Byte] def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[FS2ProducerSpec.Resources[IO, PutReq, PutRes, A]]] @@ -49,36 +48,35 @@ private[kinesis4cats] abstract class FS2ProducerSpec[PutReq, PutRes, A] def appName = streamName - fixture(3, appName).test("It should produce records end to end") { - resources => - for { - data <- IO(Arbitrary.arbitrary[TestData].take(50).toList) - records = NonEmptyList.fromListUnsafe( - data.map(x => - Record( - x.asJson.noSpaces.getBytes(), - Utils.randomUUIDString, - None, - None - ) + fixture(appName).test("It should produce records end to end") { resources => + for { + data <- IO(Arbitrary.arbitrary[TestData].take(50).toList) + records = NonEmptyList.fromListUnsafe( + data.map(x => + Record( + x.asJson.noSpaces.getBytes(), + Utils.randomUUIDString, + None, + None ) ) - _ <- records.traverse(resources.producer.put) - retryPolicy = limitRetries[IO](30).join(constantDelay(1.second)) - size <- retryingOnFailures( - retryPolicy, - (x: Int) => IO(x === 50), - noop[IO, Int] - )(resources.resultsQueue.size) - _ <- IO(assertEquals(size, 50)) - results <- resources.resultsQueue.tryTakeN(None) - resultRecords <- results.traverse { x => - IO.fromEither(decode[TestData](new String(aAsBytes(x)))) - } - } yield assert( - resultRecords.forall(data.contains), - s"res: ${resultRecords}\nexp: ${data}" ) + _ <- records.traverse(resources.producer.put) + retryPolicy = limitRetries[IO](30).join(constantDelay(1.second)) + size <- retryingOnFailures( + retryPolicy, + (x: Int) => IO(x === 50), + noop[IO, Int] + )(resources.resultsQueue.size) + _ <- IO(assertEquals(size, 50)) + results <- resources.resultsQueue.tryTakeN(None) + resultRecords <- results.traverse { x => + IO.fromEither(decode[TestData](new String(aAsBytes(x)))) + } + } yield assert( + resultRecords.forall(data.contains), + s"res: ${resultRecords}\nexp: ${data}" + ) } } diff --git a/integration-tests/src/test/scala/kinesis4cats/smithy4s/client/KinesisClientSpec.scala b/integration-tests/src/test/scala/kinesis4cats/smithy4s/client/KinesisClientSpec.scala index 318117a6..e668fced 100644 --- a/integration-tests/src/test/scala/kinesis4cats/smithy4s/client/KinesisClientSpec.scala +++ b/integration-tests/src/test/scala/kinesis4cats/smithy4s/client/KinesisClientSpec.scala @@ -22,6 +22,7 @@ import scala.concurrent.duration._ import _root_.smithy4s.ByteArray import _root_.smithy4s.aws.AwsRegion import cats.effect._ +import cats.effect.syntax.all._ import cats.syntax.all._ import com.amazonaws.kinesis._ import fs2.io.net.tls.TLSContext @@ -51,11 +52,10 @@ abstract class KinesisClientSpec extends munit.CatsEffectSuite { .withTLSContext(tlsContext) .withoutCheckEndpointAuthentication .build - client <- LocalstackKinesisClient.clientResource[IO]( - underlying, - IO.pure(region), - loggerF = (f: Async[IO]) => f.pure(new ConsoleLogger[IO]) - ) + builder <- LocalstackKinesisClient.Builder + .default(underlying, region) + .toResource + client <- builder.withLogger(new ConsoleLogger[IO]).build } yield client ) diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/client/CloudWatchClientSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/client/CloudWatchClientSpec.scala index c82d6e28..e22faced 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/client/CloudWatchClientSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/client/CloudWatchClientSpec.scala @@ -18,6 +18,7 @@ package kinesis4cats package client import cats.effect.kernel.Clock +import cats.effect.syntax.all._ import cats.effect.{IO, SyncIO} import software.amazon.awssdk.services.cloudwatch.model._ @@ -27,7 +28,10 @@ import kinesis4cats.client.localstack.LocalstackCloudWatchClient class CloudWatchClientSpec extends munit.CatsEffectSuite { def fixture: SyncIO[FunFixture[CloudWatchClient[IO]]] = ResourceFunFixture( - LocalstackCloudWatchClient.clientResource[IO]() + LocalstackCloudWatchClient.Builder + .default[IO]() + .toResource + .flatMap(_.build) ) val tableName = s"cloudwatch-client-spec-${Utils.randomUUIDString}" diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/client/DynamoClientSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/client/DynamoClientSpec.scala index 7073c625..0bd17b37 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/client/DynamoClientSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/client/DynamoClientSpec.scala @@ -19,6 +19,7 @@ package client import scala.jdk.CollectionConverters._ +import cats.effect.syntax.all._ import cats.effect.{IO, SyncIO} import software.amazon.awssdk.services.dynamodb.model._ @@ -28,7 +29,7 @@ import kinesis4cats.client.localstack.LocalstackDynamoClient class DynamoClientSpec extends munit.CatsEffectSuite { def fixture: SyncIO[FunFixture[DynamoClient[IO]]] = ResourceFunFixture( - LocalstackDynamoClient.clientResource[IO]() + LocalstackDynamoClient.Builder.default[IO]().toResource.flatMap(_.build) ) val tableName = s"dynamo-client-spec-${Utils.randomUUIDString}" diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/client/KinesisClientSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/client/KinesisClientSpec.scala index 71ec298e..7431b31a 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/client/KinesisClientSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/client/KinesisClientSpec.scala @@ -19,6 +19,7 @@ package client import scala.jdk.CollectionConverters._ +import cats.effect.syntax.all._ import cats.effect.{IO, SyncIO} import cats.syntax.all._ import io.circe.parser._ @@ -35,7 +36,7 @@ import kinesis4cats.syntax.scalacheck._ class KinesisClientSpec extends munit.CatsEffectSuite { def fixture: SyncIO[FunFixture[KinesisClient[IO]]] = ResourceFunFixture( - LocalstackKinesisClient.clientResource[IO]() + LocalstackKinesisClient.Builder.default[IO]().toResource.flatMap(_.build) ) val streamName = s"kinesis-client-spec-${Utils.randomUUIDString}" diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerNoShardMapSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerNoShardMapSpec.scala index c431258e..8b0f96be 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerNoShardMapSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerNoShardMapSpec.scala @@ -26,10 +26,10 @@ import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.Utils import kinesis4cats.client.KinesisClient -import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.ProducerSpec @@ -46,26 +46,26 @@ class KinesisProducerNoShardMapSpec s"kinesis-client-producer-spec-${Utils.randomUUIDString}" override def producerResource : Resource[IO, Producer[IO, PutRecordsRequest, PutRecordsResponse]] = - LocalstackKinesisProducer.resource[IO]( - streamName, - shardMapF = ( - _: KinesisClient[IO], - _: StreamNameOrArn, - _: Async[IO] - ) => - IO.pure( - ShardMapCache - .ListShardsError( - new RuntimeException("Expected Exception") - ) - .asLeft - ) - ) + LocalstackKinesisProducer.Builder + .default[IO](StreamNameOrArn.Name(streamName)) + .toResource + .flatMap(x => + x.withStreamsToCreate( + List(TestStreamConfig.default(streamName, 3)) + ).withShardMapF((_: KinesisClient[IO], _: StreamNameOrArn) => + IO.pure( + ShardMapCache + .ListShardsError( + new RuntimeException("Expected Exception") + ) + .asLeft + ) + ).build + ) override def aAsBytes(a: CommittableRecord[IO]): Array[Byte] = a.data.asArray override def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[ProducerSpec.Resources[ IO, @@ -74,18 +74,19 @@ class KinesisProducerNoShardMapSpec CommittableRecord[IO] ]]] = ResourceFunFixture( for { - _ <- LocalstackKinesisClient.streamResource[IO](streamName, shardCount) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( - new SingleStreamTracker( - StreamIdentifier.singleStreamInstance(streamName), - InitialPositionInStreamExtended.newInitialPosition( - InitialPositionInStream.TRIM_HORIZON - ) - ), - appName - )((_: List[CommittableRecord[IO]]) => IO.unit) - _ <- deferredWithResults.deferred.get.toResource producer <- producerResource + builder <- LocalstackKCLConsumer.Builder + .default[IO]( + new SingleStreamTracker( + StreamIdentifier.singleStreamInstance(streamName), + InitialPositionInStreamExtended.newInitialPosition( + InitialPositionInStream.TRIM_HORIZON + ) + ), + appName + ) + deferredWithResults <- builder.runWithResults() + _ <- deferredWithResults.deferred.get.toResource } yield ProducerSpec.Resources(deferredWithResults.resultsQueue, producer) ) diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerSpec.scala index dbb3c049..9a5e3a03 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/KinesisProducerSpec.scala @@ -24,10 +24,11 @@ import software.amazon.kinesis.common._ import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.Utils -import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig +import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.ProducerSpec import kinesis4cats.syntax.bytebuffer._ @@ -42,12 +43,15 @@ class KinesisProducerSpec s"kinesis-client-producer-spec-${Utils.randomUUIDString}" override def producerResource : Resource[IO, Producer[IO, PutRecordsRequest, PutRecordsResponse]] = - LocalstackKinesisProducer.resource[IO](streamName) + LocalstackKinesisProducer.Builder + .default[IO](StreamNameOrArn.Name(streamName)) + .map(_.withStreamsToCreate(List(TestStreamConfig.default(streamName, 3)))) + .toResource + .flatMap(_.build) override def aAsBytes(a: CommittableRecord[IO]): Array[Byte] = a.data.asArray override def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[ProducerSpec.Resources[ IO, @@ -56,18 +60,19 @@ class KinesisProducerSpec CommittableRecord[IO] ]]] = ResourceFunFixture( for { - _ <- LocalstackKinesisClient.streamResource[IO](streamName, shardCount) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( - new SingleStreamTracker( - StreamIdentifier.singleStreamInstance(streamName), - InitialPositionInStreamExtended.newInitialPosition( - InitialPositionInStream.TRIM_HORIZON - ) - ), - appName - )((_: List[CommittableRecord[IO]]) => IO.unit) - _ <- deferredWithResults.deferred.get.toResource producer <- producerResource + builder <- LocalstackKCLConsumer.Builder + .default[IO]( + new SingleStreamTracker( + StreamIdentifier.singleStreamInstance(streamName), + InitialPositionInStreamExtended.newInitialPosition( + InitialPositionInStream.TRIM_HORIZON + ) + ), + appName + ) + deferredWithResults <- builder.runWithResults() + _ <- deferredWithResults.deferred.get.toResource } yield ProducerSpec.Resources(deferredWithResults.resultsQueue, producer) ) diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/fs2/FS2KinesisProducerSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/fs2/FS2KinesisProducerSpec.scala index 9d3f39c7..2381bf3d 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/fs2/FS2KinesisProducerSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/client/producer/fs2/FS2KinesisProducerSpec.scala @@ -24,10 +24,11 @@ import software.amazon.kinesis.common._ import software.amazon.kinesis.processor.SingleStreamTracker import kinesis4cats.Utils -import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.client.producer.fs2.localstack.LocalstackFS2KinesisProducer import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig +import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.fs2.FS2Producer import kinesis4cats.producer.fs2.FS2ProducerSpec import kinesis4cats.syntax.bytebuffer._ @@ -42,12 +43,15 @@ class KinesisFS2ProducerSpec s"kinesis-client-fs2-producer-spec-${Utils.randomUUIDString}" override def producerResource : Resource[IO, FS2Producer[IO, PutRecordsRequest, PutRecordsResponse]] = - LocalstackFS2KinesisProducer.resource[IO](streamName) + LocalstackFS2KinesisProducer.Builder + .default[IO](StreamNameOrArn.Name(streamName)) + .map(_.withStreamsToCreate(List(TestStreamConfig.default(streamName, 3)))) + .toResource + .flatMap(_.build) override def aAsBytes(a: CommittableRecord[IO]): Array[Byte] = a.data.asArray override def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[FS2ProducerSpec.Resources[ IO, @@ -56,8 +60,8 @@ class KinesisFS2ProducerSpec CommittableRecord[IO] ]]] = ResourceFunFixture( for { - _ <- LocalstackKinesisClient.streamResource[IO](streamName, shardCount) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( + producer <- producerResource + builder <- LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker( StreamIdentifier.singleStreamInstance(streamName), InitialPositionInStreamExtended.newInitialPosition( @@ -65,9 +69,9 @@ class KinesisFS2ProducerSpec ) ), appName - )((_: List[CommittableRecord[IO]]) => IO.unit) + ) + deferredWithResults <- builder.runWithResults() _ <- deferredWithResults.deferred.get.toResource - producer <- producerResource } yield FS2ProducerSpec.Resources( deferredWithResults.resultsQueue, producer diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/KCLConsumerSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/KCLConsumerSpec.scala index 17ee2e22..fcdc5a8c 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/KCLConsumerSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/KCLConsumerSpec.scala @@ -21,6 +21,7 @@ import scala.concurrent.duration._ import cats.effect.Deferred import cats.effect.std.Queue +import cats.effect.syntax.all._ import cats.effect.{IO, Resource, SyncIO} import cats.syntax.all._ import io.circe.parser._ @@ -37,6 +38,7 @@ import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.compat.retry import kinesis4cats.compat.retry.RetryPolicies._ import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.syntax.bytebuffer._ import kinesis4cats.syntax.scalacheck._ @@ -90,8 +92,15 @@ object KCLConsumerSpec { shardCount: Int, appName: String ): Resource[IO, Resources[IO]] = for { - client <- LocalstackKinesisClient.streamResource[IO](streamName, shardCount) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( + client <- LocalstackKinesisClient.Builder + .default[IO]() + .toResource + .flatMap( + _.withStreamsToCreate( + List(TestStreamConfig.default(streamName, shardCount)) + ).build + ) + builder <- LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker( StreamIdentifier.singleStreamInstance(streamName), InitialPositionInStreamExtended.newInitialPosition( @@ -99,7 +108,8 @@ object KCLConsumerSpec { ) ), appName - )((_: List[CommittableRecord[IO]]) => IO.unit) + ) + deferredWithResults <- builder.runWithResults() } yield Resources( client, deferredWithResults.deferred, diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/KCLConsumerFS2Spec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/KCLConsumerFS2Spec.scala index 4fabc7e3..9b3f3675 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/KCLConsumerFS2Spec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/KCLConsumerFS2Spec.scala @@ -22,6 +22,7 @@ import scala.concurrent.duration._ import _root_.fs2.Stream import cats.effect.Deferred +import cats.effect.syntax.all._ import cats.effect.{IO, Resource, SyncIO} import cats.syntax.all._ import io.circe.parser._ @@ -36,6 +37,7 @@ import kinesis4cats.Utils import kinesis4cats.client.KinesisClient import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.kcl.fs2.localstack.LocalstackKCLConsumerFS2 +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.syntax.bytebuffer._ import kinesis4cats.syntax.scalacheck._ @@ -87,8 +89,15 @@ object KCLConsumerFS2Spec { shardCount: Int, appName: String ): Resource[IO, Resources[IO]] = for { - client <- LocalstackKinesisClient.streamResource[IO](streamName, shardCount) - consumer <- LocalstackKCLConsumerFS2.kclConsumer[IO]( + client <- LocalstackKinesisClient.Builder + .default[IO]() + .toResource + .flatMap( + _.withStreamsToCreate( + List(TestStreamConfig.default(streamName, shardCount)) + ).build + ) + builder <- LocalstackKCLConsumerFS2.Builder.default[IO]( new SingleStreamTracker( StreamIdentifier.singleStreamInstance(streamName), InitialPositionInStreamExtended.newInitialPosition( @@ -97,6 +106,7 @@ object KCLConsumerFS2Spec { ), appName ) + consumer <- builder.build streamAndDeferred <- consumer.streamWithDeferredListener() } yield Resources( client, diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/multistream/KCLConsumerFS2MultiSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/multistream/KCLConsumerFS2MultiSpec.scala index ddeefc2b..136af6d5 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/multistream/KCLConsumerFS2MultiSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/fs2/multistream/KCLConsumerFS2MultiSpec.scala @@ -38,6 +38,7 @@ import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.fs2.KCLConsumerFS2 import kinesis4cats.kcl.fs2.localstack.LocalstackKCLConsumerFS2 import kinesis4cats.kcl.multistream.MultiStreamTracker +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.{AwsRegion, StreamArn} import kinesis4cats.syntax.bytebuffer._ import kinesis4cats.syntax.scalacheck._ @@ -121,23 +122,32 @@ object KCLConsumerFS2MultiSpec { shardCount: Int, appName: String ): Resource[IO, Resources[IO]] = for { - _ <- LocalstackKinesisClient - .streamResource[IO](streamArn1.streamName, shardCount) + + client <- LocalstackKinesisClient.Builder + .default[IO]() + .toResource + .flatMap( + _.withStreamsToCreate( + List( + TestStreamConfig.default(streamArn1.streamName, shardCount), + TestStreamConfig.default(streamArn2.streamName, shardCount) + ) + ).build + ) position = InitialPositionInStreamExtended.newInitialPosition( InitialPositionInStream.TRIM_HORIZON ) - client <- LocalstackKinesisClient - .streamResource[IO](streamArn2.streamName, shardCount) tracker <- MultiStreamTracker .noLeaseDeletionFromArns[IO]( client, Map(streamArn1 -> position, streamArn2 -> position) ) .toResource - consumer <- LocalstackKCLConsumerFS2.kclConsumer[IO]( + builder <- LocalstackKCLConsumerFS2.Builder.default[IO]( tracker, appName ) + consumer <- builder.build streamAndDeferred <- consumer.streamWithDeferredListener() } yield Resources( client, diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/multistream/KCLConsumerMultiSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/multistream/KCLConsumerMultiSpec.scala index eeda6c96..c55517c2 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/kcl/multistream/KCLConsumerMultiSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/kcl/multistream/KCLConsumerMultiSpec.scala @@ -38,6 +38,7 @@ import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.compat.retry.RetryPolicies._ import kinesis4cats.compat.retry._ import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.{AwsRegion, StreamArn} import kinesis4cats.syntax.bytebuffer._ import kinesis4cats.syntax.scalacheck._ @@ -118,23 +119,28 @@ object KCLConsumerMultiSpec { shardCount: Int, appName: String ): Resource[IO, Resources[IO]] = for { - _ <- LocalstackKinesisClient - .streamResource[IO](streamArn1.streamName, shardCount) + client <- LocalstackKinesisClient.Builder + .default[IO]() + .toResource + .flatMap( + _.withStreamsToCreate( + List( + TestStreamConfig.default(streamArn1.streamName, shardCount), + TestStreamConfig.default(streamArn2.streamName, shardCount) + ) + ).build + ) position = InitialPositionInStreamExtended.newInitialPosition( InitialPositionInStream.TRIM_HORIZON ) - client <- LocalstackKinesisClient - .streamResource[IO](streamArn2.streamName, shardCount) tracker <- MultiStreamTracker .noLeaseDeletionFromArns[IO]( client, Map(streamArn1 -> position, streamArn2 -> position) ) .toResource - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( - tracker, - appName - )((_: List[CommittableRecord[IO]]) => IO.unit) + builder <- LocalstackKCLConsumer.Builder.default[IO](tracker, appName) + deferredWithResults <- builder.runWithResults() } yield Resources( client, deferredWithResults.deferred, diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/kpl/KPLProducerSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/kpl/KPLProducerSpec.scala index 559a96f1..c3e59cec 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/kpl/KPLProducerSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/kpl/KPLProducerSpec.scala @@ -21,6 +21,7 @@ import scala.concurrent.duration._ import java.nio.ByteBuffer +import cats.effect.syntax.all._ import cats.effect.{IO, SyncIO} import com.amazonaws.services.kinesis.producer._ import io.circe.syntax._ @@ -28,6 +29,7 @@ import org.scalacheck.Arbitrary import kinesis4cats.Utils import kinesis4cats.kpl.localstack.LocalstackKPLProducer +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.syntax.scalacheck._ abstract class KPLProducerSpec @@ -40,10 +42,14 @@ abstract class KPLProducerSpec streamName: String, shardCount: Int ): SyncIO[FunFixture[KPLProducer[IO]]] = ResourceFunFixture( - LocalstackKPLProducer.producerWithStream[IO]( - streamName, - shardCount - ) + LocalstackKPLProducer.Builder + .default[IO]() + .toResource + .flatMap( + _.withStreamsToCreate( + List(TestStreamConfig.default(streamName, shardCount)) + ).build + ) ) val streamName = s"kpl-producer-spec-${Utils.randomUUIDString}" diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/KinesisClientJVMSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/KinesisClientJVMSpec.scala index 5770e0f1..0d22d748 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/KinesisClientJVMSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/KinesisClientJVMSpec.scala @@ -16,9 +16,9 @@ package kinesis4cats.smithy4s.client -import cats.effect.Async import cats.effect.IO import cats.effect.SyncIO +import cats.effect.syntax.all._ import com.amazonaws.kinesis.Kinesis import org.http4s.blaze.client.BlazeClientBuilder @@ -33,11 +33,10 @@ class KinesisClientJVMSpec extends KinesisClientSpec { underlying <- BlazeClientBuilder[IO] .withSslContext(SSL.context) .resource - client <- LocalstackKinesisClient.clientResource[IO]( - underlying, - IO.pure(region), - loggerF = (f: Async[IO]) => f.pure(new ConsoleLogger[IO]) - ) + builder <- LocalstackKinesisClient.Builder + .default(underlying, region) + .toResource + client <- builder.withLogger(new ConsoleLogger[IO]).build } yield client ) } diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerNoShardMapSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerNoShardMapSpec.scala index 139011d1..67ca570e 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerNoShardMapSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerNoShardMapSpec.scala @@ -31,12 +31,12 @@ import kinesis4cats.SSL import kinesis4cats.Utils import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.ProducerSpec import kinesis4cats.producer.ShardMapCache import kinesis4cats.smithy4s.client.KinesisClient -import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient import kinesis4cats.smithy4s.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.syntax.bytebuffer._ @@ -52,22 +52,24 @@ class KinesisProducerNoShardMapSpec def http4sClientResource = BlazeClientBuilder[IO].withSslContext(SSL.context).resource - lazy val region = IO.pure(AwsRegion.US_EAST_1) + lazy val region = AwsRegion.US_EAST_1 override def producerResource : Resource[IO, Producer[IO, PutRecordsInput, PutRecordsOutput]] = for { http4sClient <- http4sClientResource - producer <- LocalstackKinesisProducer - .resource[IO]( - http4sClient, - streamName, - region, - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO], - shardMapF = ( + builder <- LocalstackKinesisProducer.Builder + .default(http4sClient, region, StreamNameOrArn.Name(streamName)) + .toResource + producer <- builder + .withLogger(Slf4jLogger.getLogger[IO]) + .withStreamsToCreate( + List(TestStreamConfig.default(streamName, 3)) + ) + .withShardMapF( + ( _: KinesisClient[IO], - _: StreamNameOrArn, - _: Async[IO] + _: StreamNameOrArn ) => IO.pure( ShardMapCache @@ -77,12 +79,12 @@ class KinesisProducerNoShardMapSpec .asLeft ) ) + .build } yield producer override def aAsBytes(a: CommittableRecord[IO]): Array[Byte] = a.data.asArray override def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[ProducerSpec.Resources[ IO, @@ -91,16 +93,8 @@ class KinesisProducerNoShardMapSpec CommittableRecord[IO] ]]] = ResourceFunFixture( for { - http4sClient <- http4sClientResource - _ <- LocalstackKinesisClient - .streamResource[IO]( - http4sClient, - region, - streamName, - shardCount, - loggerF = (f: Async[IO]) => Slf4jLogger.create[IO](f, implicitly) - ) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( + producer <- producerResource + builder <- LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker( StreamIdentifier.singleStreamInstance(streamName), InitialPositionInStreamExtended.newInitialPosition( @@ -108,9 +102,9 @@ class KinesisProducerNoShardMapSpec ) ), appName - )((_: List[CommittableRecord[IO]]) => IO.unit) + ) + deferredWithResults <- builder.runWithResults() _ <- deferredWithResults.deferred.get.toResource - producer <- producerResource } yield ProducerSpec.Resources(deferredWithResults.resultsQueue, producer) ) diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerSpec.scala index b1576da3..3cc09609 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/KinesisProducerSpec.scala @@ -30,9 +30,10 @@ import kinesis4cats.SSL import kinesis4cats.Utils import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig +import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.ProducerSpec -import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient import kinesis4cats.smithy4s.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.syntax.bytebuffer._ @@ -48,25 +49,26 @@ class KinesisProducerSpec def http4sClientResource = BlazeClientBuilder[IO].withSslContext(SSL.context).resource - lazy val region = IO.pure(AwsRegion.US_EAST_1) + lazy val region = AwsRegion.US_EAST_1 override def producerResource : Resource[IO, Producer[IO, PutRecordsInput, PutRecordsOutput]] = for { http4sClient <- http4sClientResource - producer <- LocalstackKinesisProducer - .resource[IO]( - http4sClient, - streamName, - region, - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] + builder <- LocalstackKinesisProducer.Builder + .default(http4sClient, region, StreamNameOrArn.Name(streamName)) + .toResource + producer <- builder + .withLogger(Slf4jLogger.getLogger[IO]) + .withStreamsToCreate( + List(TestStreamConfig.default(streamName, 3)) ) + .build } yield producer override def aAsBytes(a: CommittableRecord[IO]): Array[Byte] = a.data.asArray override def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[ProducerSpec.Resources[ IO, @@ -75,16 +77,8 @@ class KinesisProducerSpec CommittableRecord[IO] ]]] = ResourceFunFixture( for { - http4sClient <- http4sClientResource - _ <- LocalstackKinesisClient - .streamResource[IO]( - http4sClient, - region, - streamName, - shardCount, - loggerF = (f: Async[IO]) => Slf4jLogger.create[IO](f, implicitly) - ) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( + producer <- producerResource + builder <- LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker( StreamIdentifier.singleStreamInstance(streamName), InitialPositionInStreamExtended.newInitialPosition( @@ -92,9 +86,9 @@ class KinesisProducerSpec ) ), appName - )((_: List[CommittableRecord[IO]]) => IO.unit) + ) + deferredWithResults <- builder.runWithResults() _ <- deferredWithResults.deferred.get.toResource - producer <- producerResource } yield ProducerSpec.Resources(deferredWithResults.resultsQueue, producer) ) diff --git a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducerSpec.scala b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducerSpec.scala index c820a338..f0f4cf1e 100644 --- a/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducerSpec.scala +++ b/integration-tests/src/test/scalajvm/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducerSpec.scala @@ -30,9 +30,10 @@ import kinesis4cats.SSL import kinesis4cats.Utils import kinesis4cats.kcl.CommittableRecord import kinesis4cats.kcl.localstack.LocalstackKCLConsumer +import kinesis4cats.localstack.TestStreamConfig +import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.fs2.FS2Producer import kinesis4cats.producer.fs2.FS2ProducerSpec -import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient import kinesis4cats.smithy4s.client.producer.fs2.localstack.LocalstackFS2KinesisProducer import kinesis4cats.syntax.bytebuffer._ @@ -48,25 +49,25 @@ class FS2KinesisProducerSpec def http4sClientResource = BlazeClientBuilder[IO].withSslContext(SSL.context).resource - lazy val region = IO.pure(AwsRegion.US_EAST_1) + lazy val region = AwsRegion.US_EAST_1 override def producerResource : Resource[IO, FS2Producer[IO, PutRecordsInput, PutRecordsOutput]] = for { http4sClient <- http4sClientResource - producer <- LocalstackFS2KinesisProducer - .resource[IO]( - http4sClient, - streamName, - region, - loggerF = (_: Async[IO]) => Slf4jLogger.create[IO] + producer <- LocalstackFS2KinesisProducer.Builder + .default[IO](http4sClient, region, StreamNameOrArn.Name(streamName)) + .toResource + .flatMap( + _.withLogger(Slf4jLogger.getLogger[IO]) + .withStreamsToCreate(List(TestStreamConfig.default(streamName, 3))) + .build ) } yield producer override def aAsBytes(a: CommittableRecord[IO]): Array[Byte] = a.data.asArray override def fixture( - shardCount: Int, appName: String ): SyncIO[FunFixture[FS2ProducerSpec.Resources[ IO, @@ -75,16 +76,8 @@ class FS2KinesisProducerSpec CommittableRecord[IO] ]]] = ResourceFunFixture( for { - http4sClient <- http4sClientResource - _ <- LocalstackKinesisClient - .streamResource[IO]( - http4sClient, - region, - streamName, - shardCount, - loggerF = (f: Async[IO]) => Slf4jLogger.create[IO](f, implicitly) - ) - deferredWithResults <- LocalstackKCLConsumer.kclConsumerWithResults( + producer <- producerResource + builder <- LocalstackKCLConsumer.Builder.default[IO]( new SingleStreamTracker( StreamIdentifier.singleStreamInstance(streamName), InitialPositionInStreamExtended.newInitialPosition( @@ -92,9 +85,9 @@ class FS2KinesisProducerSpec ) ), appName - )((_: List[CommittableRecord[IO]]) => IO.unit) + ) + deferredWithResults <- builder.runWithResults() _ <- deferredWithResults.deferred.get.toResource - producer <- producerResource } yield FS2ProducerSpec.Resources( deferredWithResults.resultsQueue, producer diff --git a/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/KCLCiris.scala b/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/KCLCiris.scala index 550b2b4c..a0c5c7b5 100644 --- a/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/KCLCiris.scala +++ b/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/KCLCiris.scala @@ -107,9 +107,11 @@ object KCLCiris { * [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] */ def consumer[F[_]]( - kinesisClient: KinesisAsyncClient, - dynamoClient: DynamoDbAsyncClient, - cloudwatchClient: CloudWatchAsyncClient, + kinesisClient: => KinesisAsyncClient = KinesisAsyncClient.builder().build, + dynamoClient: => DynamoDbAsyncClient = + DynamoDbAsyncClient.builder().build, + cloudWatchClient: => CloudWatchAsyncClient = + CloudWatchAsyncClient.builder().build, prefix: Option[String] = None, shardPrioritization: Option[ShardPrioritization] = None, workerStateChangeListener: Option[WorkerStateChangeListener] = None, @@ -124,28 +126,44 @@ object KCLCiris { metricsFactory: Option[MetricsFactory] = None, glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = None, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show + encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show, + managedClients: Boolean = true )(cb: List[CommittableRecord[F]] => F[Unit])(implicit F: Async[F] - ): Resource[F, KCLConsumer[F]] = kclConfig[F]( - kinesisClient, - dynamoClient, - cloudwatchClient, - prefix, - shardPrioritization, - workerStateChangeListener, - coordinatorFactory, - customShardDetectorProvider, - tableCreatorCallback, - hierarchicalShardSyncer, - leaseManagementFactory, - leaseExecutorService, - aggregatorUtil, - taskExecutionListener, - metricsFactory, - glueSchemaRegistryDeserializer, - encoders - )(cb).map(new KCLConsumer[F](_)) + ): Resource[F, KCLConsumer[F]] = for { + kClient <- + if (managedClients) + Resource.fromAutoCloseable( + F.delay(kinesisClient) + ) + else Resource.pure[F, KinesisAsyncClient](kinesisClient) + dClient <- + if (managedClients) Resource.fromAutoCloseable(F.delay(dynamoClient)) + else Resource.pure[F, DynamoDbAsyncClient](dynamoClient) + cClient <- + if (managedClients) + Resource.fromAutoCloseable(F.delay(cloudWatchClient)) + else Resource.pure[F, CloudWatchAsyncClient](cloudWatchClient) + consumer <- kclConfig[F]( + kClient, + dClient, + cClient, + prefix, + shardPrioritization, + workerStateChangeListener, + coordinatorFactory, + customShardDetectorProvider, + tableCreatorCallback, + hierarchicalShardSyncer, + leaseManagementFactory, + leaseExecutorService, + aggregatorUtil, + taskExecutionListener, + metricsFactory, + glueSchemaRegistryDeserializer, + encoders + )(cb).map(new KCLConsumer[F](_)) + } yield consumer /** Reads environment variables and system properties to load a * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] @@ -198,25 +216,24 @@ object KCLCiris { * [[cats.effect.Resource Resource]] containing the * [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] */ - def kclConfig[F[_]]( + private[kinesis4cats] def kclConfig[F[_]]( kinesisClient: KinesisAsyncClient, dynamoClient: DynamoDbAsyncClient, cloudwatchClient: CloudWatchAsyncClient, - prefix: Option[String] = None, - shardPrioritization: Option[ShardPrioritization] = None, - workerStateChangeListener: Option[WorkerStateChangeListener] = None, - coordinatorFactory: Option[CoordinatorFactory] = None, - customShardDetectorProvider: Option[StreamConfig => ShardDetector] = None, - tableCreatorCallback: Option[TableCreatorCallback] = None, - hierarchicalShardSyncer: Option[HierarchicalShardSyncer] = None, - leaseManagementFactory: Option[LeaseManagementFactory] = None, - leaseExecutorService: Option[ExecutorService] = None, - aggregatorUtil: Option[AggregatorUtil] = None, - taskExecutionListener: Option[TaskExecutionListener] = None, - metricsFactory: Option[MetricsFactory] = None, - glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = - None, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show + prefix: Option[String], + shardPrioritization: Option[ShardPrioritization], + workerStateChangeListener: Option[WorkerStateChangeListener], + coordinatorFactory: Option[CoordinatorFactory], + customShardDetectorProvider: Option[StreamConfig => ShardDetector], + tableCreatorCallback: Option[TableCreatorCallback], + hierarchicalShardSyncer: Option[HierarchicalShardSyncer], + leaseManagementFactory: Option[LeaseManagementFactory], + leaseExecutorService: Option[ExecutorService], + aggregatorUtil: Option[AggregatorUtil], + taskExecutionListener: Option[TaskExecutionListener], + metricsFactory: Option[MetricsFactory], + glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer], + encoders: RecordProcessor.LogEncoders )(cb: List[CommittableRecord[F]] => F[Unit])(implicit F: Async[F] ): Resource[F, KCLConsumer.Config[F]] = for { @@ -266,8 +283,8 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of the * app name string */ - def readAppName( - prefix: Option[String] = None + private[kinesis4cats] def readAppName( + prefix: Option[String] ): ConfigValue[Effect, String] = CirisReader.read[String](List("kcl", "app", "name"), prefix) @@ -279,8 +296,8 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of the * stream name string */ - def readStreamName( - prefix: Option[String] = None + private[kinesis4cats] def readStreamName( + prefix: Option[String] ): ConfigValue[Effect, String] = CirisReader.read[String](List("kcl", "stream", "name"), prefix) @@ -292,8 +309,8 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of the * initial position */ - def readInitialPosition( - prefix: Option[String] = None + private[kinesis4cats] def readInitialPosition( + prefix: Option[String] ): ConfigValue[Effect, InitialPositionInStreamExtended] = CirisReader.readDefaulted[InitialPositionInStreamExtended]( List("kcl", "initial", "position"), @@ -312,21 +329,9 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[[[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] */ - def read: ConfigValue[Effect, CheckpointConfig] = + private[kinesis4cats] def read: ConfigValue[Effect, CheckpointConfig] = ConfigValue.default(new CheckpointConfig()) - /** Reads the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] - * into an [[cats.effect.Async Async]] - * - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[[[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] - */ - def load[F[_]](implicit F: Async[F]) = read.load[F] - /** Reads the * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] * into a [[cats.effect.Resource Resource]] @@ -337,7 +342,8 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[[[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] */ - def resource[F[_]](implicit F: Async[F]) = read.resource[F] + private[kinesis4cats] def resource[F[_]](implicit F: Async[F]) = + read.resource[F] } object Coordinator { @@ -357,11 +363,11 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] */ - def read( - prefix: Option[String] = None, - shardPrioritization: Option[ShardPrioritization] = None, - workerStateChangeListener: Option[WorkerStateChangeListener] = None, - coordinatorFactory: Option[CoordinatorFactory] = None + private[kinesis4cats] def read( + prefix: Option[String], + shardPrioritization: Option[ShardPrioritization], + workerStateChangeListener: Option[WorkerStateChangeListener], + coordinatorFactory: Option[CoordinatorFactory] ): ConfigValue[Effect, CoordinatorConfig] = for { appName <- Common.readAppName(prefix) @@ -440,36 +446,6 @@ object KCLCiris { ) .maybeTransform(coordinatorFactory)(_.coordinatorFactory(_)) - /** Reads the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] - * into an [[cats.effect.Async Async]] - * - * @param prefix - * Optional prefix to apply to configuration loaders. Default None - * @param shardPrioritization - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardPrioritization.java ShardPrioritization]] - * @param workerStateChangeListener - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/WorkerStateChangeListener.java WorkerStateChangeListener]] - * @param coordinatorFactory - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java CoordinatorFactory]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] - */ - def load[F[_]]( - prefix: Option[String] = None, - shardPrioritization: Option[ShardPrioritization] = None, - workerStateChangeListener: Option[WorkerStateChangeListener] = None, - coordinatorFactory: Option[CoordinatorFactory] = None - )(implicit F: Async[F]): F[CoordinatorConfig] = read( - prefix, - shardPrioritization, - workerStateChangeListener, - coordinatorFactory - ).load[F] - /** Reads the * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] * into a [[cats.effect.Resource Resource]] @@ -488,11 +464,11 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] */ - def resource[F[_]]( - prefix: Option[String] = None, - shardPrioritization: Option[ShardPrioritization] = None, - workerStateChangeListener: Option[WorkerStateChangeListener] = None, - coordinatorFactory: Option[CoordinatorFactory] = None + private[kinesis4cats] def resource[F[_]]( + prefix: Option[String], + shardPrioritization: Option[ShardPrioritization], + workerStateChangeListener: Option[WorkerStateChangeListener], + coordinatorFactory: Option[CoordinatorFactory] )(implicit F: Async[F]): Resource[F, CoordinatorConfig] = read( prefix, shardPrioritization, @@ -529,16 +505,15 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] */ - def read( + private[kinesis4cats] def read( dynamoClient: DynamoDbAsyncClient, kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None, - customShardDetectorProvider: Option[StreamConfig => ShardDetector] = - None, - tableCreatorCallback: Option[TableCreatorCallback] = None, - hierarchicalShardSyncer: Option[HierarchicalShardSyncer] = None, - leaseManagementFactory: Option[LeaseManagementFactory] = None, - executorService: Option[ExecutorService] = None + prefix: Option[String], + customShardDetectorProvider: Option[StreamConfig => ShardDetector], + tableCreatorCallback: Option[TableCreatorCallback], + hierarchicalShardSyncer: Option[HierarchicalShardSyncer], + leaseManagementFactory: Option[LeaseManagementFactory], + executorService: Option[ExecutorService] ): ConfigValue[Effect, LeaseManagementConfig] = for { tableName <- CirisReader .read[String](List("kcl", "lease", "table", "name"), prefix) @@ -729,56 +704,6 @@ object KCLCiris { .maybeTransform(executorService)(_.executorService(_)) .maybeTransform(leaseManagementFactory)(_.leaseManagementFactory(_)) - /** Reads the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] - * as an [[cats.effect.Async Async]] - * - * @param dynamoClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/dynamodb/DynamoDbAsyncClient.html DynamoDbAsyncClient]] - * @param kinesisClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param prefix - * Optional prefix to apply to configuration loaders. Default None - * @param customShardDetectorProvider - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/StreamConfig.java StreamConfig]] - * \=> - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java ShardDetector]] - * @param tableCreatorCallback - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallback.java TableCreatorCallback]] - * @param hierarchicalShardSyncer - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/HierarchicalShardSyncer.java HierarchicalShardSyncer]] - * @param leaseManagementFactory - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java LeaseManagementFactory]] - * @param leaseExecutorService - * [[https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ExecutorService.html ExecutorService]] - * for the lease management - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] - */ - def load[F[_]]( - dynamoClient: DynamoDbAsyncClient, - kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None, - customShardDetectorProvider: Option[StreamConfig => ShardDetector] = - None, - tableCreatorCallback: Option[TableCreatorCallback] = None, - hierarchicalShardSyncer: Option[HierarchicalShardSyncer] = None, - leaseManagementFactory: Option[LeaseManagementFactory] = None, - executorService: Option[ExecutorService] = None - )(implicit F: Async[F]): F[LeaseManagementConfig] = read( - dynamoClient, - kinesisClient, - prefix, - customShardDetectorProvider, - tableCreatorCallback, - hierarchicalShardSyncer, - leaseManagementFactory, - executorService - ).load[F] - /** Reads the * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] * as a [[cats.effect.Resource Resource]] @@ -808,16 +733,15 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] */ - def resource[F[_]]( + private[kinesis4cats] def resource[F[_]]( dynamoClient: DynamoDbAsyncClient, kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None, - customShardDetectorProvider: Option[StreamConfig => ShardDetector] = - None, - tableCreatorCallback: Option[TableCreatorCallback] = None, - hierarchicalShardSyncer: Option[HierarchicalShardSyncer] = None, - leaseManagementFactory: Option[LeaseManagementFactory] = None, - executorService: Option[ExecutorService] = None + prefix: Option[String], + customShardDetectorProvider: Option[StreamConfig => ShardDetector], + tableCreatorCallback: Option[TableCreatorCallback], + hierarchicalShardSyncer: Option[HierarchicalShardSyncer], + leaseManagementFactory: Option[LeaseManagementFactory], + executorService: Option[ExecutorService] )(implicit F: Async[F]): Resource[F, LeaseManagementConfig] = read( dynamoClient, kinesisClient, @@ -845,10 +769,10 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] */ - def read( - prefix: Option[String] = None, - aggregatorUtil: Option[AggregatorUtil] = None, - taskExecutionListener: Option[TaskExecutionListener] = None + private[kinesis4cats] def read( + prefix: Option[String], + aggregatorUtil: Option[AggregatorUtil], + taskExecutionListener: Option[TaskExecutionListener] ): ConfigValue[Effect, LifecycleConfig] = for { logWarningForTaskAfter <- CirisReader .readOptional[Duration]( @@ -886,29 +810,6 @@ object KCLCiris { .maybeTransform(aggregatorUtil)(_.aggregatorUtil(_)) .maybeTransform(taskExecutionListener)(_.taskExecutionListener(_)) - /** Reads the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] - * as an [[cats.effect.Async Async]] - * - * @param prefix - * Optional prefix to apply to configuration loaders. Default None - * @param aggregatorUtil - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/AggregatorUtil.java AggregatorUtil]] - * @param taskExecutionListener - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/TaskExecutionListener.java TaskExecutionListener]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] - */ - def load[F[_]]( - prefix: Option[String] = None, - aggregatorUtil: Option[AggregatorUtil] = None, - taskExecutionListener: Option[TaskExecutionListener] = None - )(implicit F: Async[F]): F[LifecycleConfig] = - read(prefix, aggregatorUtil, taskExecutionListener).load[F] - /** Reads the * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] * as a [[cats.effect.Resource Resource]] @@ -925,10 +826,10 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] */ - def resource[F[_]]( - prefix: Option[String] = None, - aggregatorUtil: Option[AggregatorUtil] = None, - taskExecutionListener: Option[TaskExecutionListener] = None + private[kinesis4cats] def resource[F[_]]( + prefix: Option[String], + aggregatorUtil: Option[AggregatorUtil], + taskExecutionListener: Option[TaskExecutionListener] )(implicit F: Async[F]): Resource[F, LifecycleConfig] = read(prefix, aggregatorUtil, taskExecutionListener).resource[F] } @@ -948,10 +849,10 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] */ - def read( + private[kinesis4cats] def read( cloudwatchClient: CloudWatchAsyncClient, - prefix: Option[String] = None, - metricsFactory: Option[MetricsFactory] = None + prefix: Option[String], + metricsFactory: Option[MetricsFactory] ): ConfigValue[Effect, MetricsConfig] = for { namespace <- CirisReader .read[String]( @@ -993,29 +894,6 @@ object KCLCiris { .maybeTransform(publisherFlushBuffer)(_.publisherFlushBuffer(_)) .maybeTransform(metricsFactory)(_.metricsFactory(_)) - /** Reads the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] - * into an [[cats.effect.Async Async]] - * - * @param cloudWatchClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cloudwatch/CloudWatchClient.html CloudWatchClient]] - * @param prefix - * Optional prefix to apply to configuration loaders. Default None - * @param metricsFactory - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsFactory.java MetricsFactory]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] - */ - def load[F[_]]( - cloudwatchClient: CloudWatchAsyncClient, - prefix: Option[String] = None, - metricsFactory: Option[MetricsFactory] = None - )(implicit F: Async[F]): F[MetricsConfig] = - read(cloudwatchClient, prefix, metricsFactory).load[F] - /** Reads the * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] * into a [[cats.effect.Resource Resource]] @@ -1032,10 +910,10 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] */ - def resource[F[_]]( + private[kinesis4cats] def resource[F[_]]( cloudwatchClient: CloudWatchAsyncClient, - prefix: Option[String] = None, - metricsFactory: Option[MetricsFactory] = None + prefix: Option[String], + metricsFactory: Option[MetricsFactory] )(implicit F: Async[F]): Resource[F, MetricsConfig] = read(cloudwatchClient, prefix, metricsFactory).resource[F] } @@ -1053,9 +931,9 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/polling/PollingConfig.java PollingConfig]] */ - def readPollingConfig( + private[kinesis4cats] def readPollingConfig( kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None + prefix: Option[String] ): ConfigValue[Effect, PollingConfig] = for { streamName <- Common.readStreamName(prefix) maxRecords <- CirisReader.readOptional[Int]( @@ -1139,9 +1017,9 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/fanout/FanOutConfig.java FanOutConfig]] */ - def readFanOutConfig( + private[kinesis4cats] def readFanOutConfig( kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None + prefix: Option[String] ): ConfigValue[Effect, FanOutConfig] = for { streamName <- Common.readStreamName(prefix) appName <- Common.readAppName(prefix) @@ -1228,11 +1106,10 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] */ - def read( + private[kinesis4cats] def read( kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None, - glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = - None + prefix: Option[String], + glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] ): ConfigValue[Effect, RetrievalConfig] = for { appName <- Common.readAppName(prefix) streamName <- Common.readStreamName(prefix) @@ -1278,30 +1155,6 @@ object KCLCiris { _.glueSchemaRegistryDeserializer(_) ) - /** Reads the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] - * as an [[cats.effect.Async Async]] - * - * @param kinesisClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param prefix - * Optional prefix to apply to configuration loaders. Default None - * @param glueSchemaRegistryDeserializer - * [[https://github.com/awslabs/aws-glue-schema-registry/blob/master/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/GlueSchemaRegistryDeserializer.java GlueSchemaRegistryDeserializer]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] - */ - def load[F[_]]( - kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None, - glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = - None - )(implicit F: Async[F]): F[RetrievalConfig] = - read(kinesisClient, prefix, glueSchemaRegistryDeserializer).load[F] - /** Reads the * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] * as a [[cats.effect.Resource Resource]] @@ -1318,11 +1171,10 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] */ - def resource[F[_]]( + private[kinesis4cats] def resource[F[_]]( kinesisClient: KinesisAsyncClient, - prefix: Option[String] = None, - glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = - None + prefix: Option[String], + glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] )(implicit F: Async[F]): Resource[F, RetrievalConfig] = read(kinesisClient, prefix, glueSchemaRegistryDeserializer).resource[F] } @@ -1338,8 +1190,8 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[kinesis4cats.kcl.RecordProcessor.Config RecordProcessor.Config]] */ - def readRecordProcessorConfig( - prefix: Option[String] = None + private[kinesis4cats] def readRecordProcessorConfig( + prefix: Option[String] ): ConfigValue[Effect, RecordProcessor.Config] = for { shardEndTimeout <- CirisReader.readOptional[FiniteDuration]( List("kcl", "processor", "shard", "end", "timeout"), @@ -1376,8 +1228,8 @@ object KCLCiris { * [[https://cir.is/api/ciris/ConfigDecoder.html ConfigDecoder]] of * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] */ - def read( - prefix: Option[String] = None + private[kinesis4cats] def read( + prefix: Option[String] ): ConfigValue[Effect, KCLConsumer.ProcessConfig] = for { recordProcessor <- readRecordProcessorConfig(prefix) callProcessRecordsEvenForEmptyRecordList <- CirisReader @@ -1406,22 +1258,6 @@ object KCLCiris { callProcessRecordsEvenForEmptyRecordList ) - /** Reads the - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * as an [[cats.effect.Async Async]] - * - * @param prefix - * Optional prefix to apply to configuration loaders. Default None - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Async Async]] of - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - */ - def load[F[_]]( - prefix: Option[String] = None - )(implicit F: Async[F]): F[KCLConsumer.ProcessConfig] = read(prefix).load[F] - /** Reads the * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] * as a [[cats.effect.Resource Resource]] @@ -1434,8 +1270,8 @@ object KCLCiris { * [[cats.effect.Resource Resource]] of * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] */ - def resource[F[_]]( - prefix: Option[String] = None + private[kinesis4cats] def resource[F[_]]( + prefix: Option[String] )(implicit F: Async[F]): Resource[F, KCLConsumer.ProcessConfig] = read( prefix ).resource[F] diff --git a/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/RetrievalType.scala b/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/RetrievalType.scala index 9cafc601..61440a58 100644 --- a/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/RetrievalType.scala +++ b/kcl-ciris/src/main/scala/kinesis4cats/kcl/ciris/RetrievalType.scala @@ -25,11 +25,11 @@ import ciris.{ConfigDecoder, ConfigError} * @param value * Underlying value */ -sealed abstract class RetrievalType(val value: String) +private[kinesis4cats] sealed abstract class RetrievalType(val value: String) -object RetrievalType { - case object Polling extends RetrievalType("polling") - case object FanOut extends RetrievalType("fanout") +private[kinesis4cats] object RetrievalType { + private[kinesis4cats] case object Polling extends RetrievalType("polling") + private[kinesis4cats] case object FanOut extends RetrievalType("fanout") implicit val retrievalTypeConfigDecoder : ConfigDecoder[String, RetrievalType] = ConfigDecoder[String].mapEither { diff --git a/kcl-ciris/src/main/scala/kinesis4cats/kcl/fs2/ciris/KCLCirisFS2.scala b/kcl-ciris/src/main/scala/kinesis4cats/kcl/fs2/ciris/KCLCirisFS2.scala index 388ed800..74fdb1ce 100644 --- a/kcl-ciris/src/main/scala/kinesis4cats/kcl/fs2/ciris/KCLCirisFS2.scala +++ b/kcl-ciris/src/main/scala/kinesis4cats/kcl/fs2/ciris/KCLCirisFS2.scala @@ -23,6 +23,8 @@ import java.util.concurrent.ExecutorService import _root_.ciris._ import cats.Parallel +import cats.effect.std.Queue +import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializer import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient @@ -38,6 +40,8 @@ import software.amazon.kinesis.retrieval.AggregatorUtil import kinesis4cats.ciris.CirisReader import kinesis4cats.instances.ciris._ +import kinesis4cats.kcl.CommittableRecord +import kinesis4cats.kcl.KCLConsumer import kinesis4cats.kcl.RecordProcessor import kinesis4cats.kcl.ciris.KCLCiris @@ -101,9 +105,11 @@ object KCLCirisFS2 { * [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] */ def consumer[F[_]]( - kinesisClient: KinesisAsyncClient, - dynamoClient: DynamoDbAsyncClient, - cloudwatchClient: CloudWatchAsyncClient, + kinesisClient: => KinesisAsyncClient = KinesisAsyncClient.builder().build, + dynamoClient: => DynamoDbAsyncClient = + DynamoDbAsyncClient.builder().build, + cloudWatchClient: => CloudWatchAsyncClient = + CloudWatchAsyncClient.builder().build, prefix: Option[String] = None, shardPrioritization: Option[ShardPrioritization] = None, workerStateChangeListener: Option[WorkerStateChangeListener] = None, @@ -118,29 +124,45 @@ object KCLCirisFS2 { metricsFactory: Option[MetricsFactory] = None, glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = None, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show + encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show, + managedClients: Boolean = true )(implicit F: Async[F], P: Parallel[F] - ): Resource[F, KCLConsumerFS2[F]] = kclConfig( - kinesisClient, - dynamoClient, - cloudwatchClient, - prefix, - shardPrioritization, - workerStateChangeListener, - coordinatorFactory, - customShardDetectorProvider, - tableCreatorCallback, - hierarchicalShardSyncer, - leaseManagementFactory, - leaseExecutorService, - aggregatorUtil, - taskExecutionListener, - metricsFactory, - glueSchemaRegistryDeserializer, - encoders - ).map(new KCLConsumerFS2[F](_)) + ): Resource[F, KCLConsumerFS2[F]] = for { + kClient <- + if (managedClients) + Resource.fromAutoCloseable( + F.delay(kinesisClient) + ) + else Resource.pure[F, KinesisAsyncClient](kinesisClient) + dClient <- + if (managedClients) Resource.fromAutoCloseable(F.delay(dynamoClient)) + else Resource.pure[F, DynamoDbAsyncClient](dynamoClient) + cClient <- + if (managedClients) + Resource.fromAutoCloseable(F.delay(cloudWatchClient)) + else Resource.pure[F, CloudWatchAsyncClient](cloudWatchClient) + config <- kclConfig( + kClient, + dClient, + cClient, + prefix, + shardPrioritization, + workerStateChangeListener, + coordinatorFactory, + customShardDetectorProvider, + tableCreatorCallback, + hierarchicalShardSyncer, + leaseManagementFactory, + leaseExecutorService, + aggregatorUtil, + taskExecutionListener, + metricsFactory, + glueSchemaRegistryDeserializer, + encoders + ) + } yield new KCLConsumerFS2[F](config) /** Reads environment variables and system properties to load a * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] @@ -193,25 +215,24 @@ object KCLCirisFS2 { * [[cats.effect.Resource Resource]] containing the * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] */ - def kclConfig[F[_]]( + private[kinesis4cats] def kclConfig[F[_]]( kinesisClient: KinesisAsyncClient, dynamoClient: DynamoDbAsyncClient, cloudwatchClient: CloudWatchAsyncClient, - prefix: Option[String] = None, - shardPrioritization: Option[ShardPrioritization] = None, - workerStateChangeListener: Option[WorkerStateChangeListener] = None, - coordinatorFactory: Option[CoordinatorFactory] = None, - customShardDetectorProvider: Option[StreamConfig => ShardDetector] = None, - tableCreatorCallback: Option[TableCreatorCallback] = None, - hierarchicalShardSyncer: Option[HierarchicalShardSyncer] = None, - leaseManagementFactory: Option[LeaseManagementFactory] = None, - leaseExecutorService: Option[ExecutorService] = None, - aggregatorUtil: Option[AggregatorUtil] = None, - taskExecutionListener: Option[TaskExecutionListener] = None, - metricsFactory: Option[MetricsFactory] = None, - glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer] = - None, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show + prefix: Option[String], + shardPrioritization: Option[ShardPrioritization], + workerStateChangeListener: Option[WorkerStateChangeListener], + coordinatorFactory: Option[CoordinatorFactory], + customShardDetectorProvider: Option[StreamConfig => ShardDetector], + tableCreatorCallback: Option[TableCreatorCallback], + hierarchicalShardSyncer: Option[HierarchicalShardSyncer], + leaseManagementFactory: Option[LeaseManagementFactory], + leaseExecutorService: Option[ExecutorService], + aggregatorUtil: Option[AggregatorUtil], + taskExecutionListener: Option[TaskExecutionListener], + metricsFactory: Option[MetricsFactory], + glueSchemaRegistryDeserializer: Option[GlueSchemaRegistryDeserializer], + encoders: RecordProcessor.LogEncoders )(implicit F: Async[F] ): Resource[F, KCLConsumerFS2.Config[F]] = for { @@ -247,24 +268,25 @@ object KCLCirisFS2 { retrievalConfig <- KCLCiris.Retrieval .resource[F](kinesisClient, prefix, glueSchemaRegistryDeserializer) processConfig <- KCLCiris.Processor.resource[F](prefix) - config <- KCLConsumerFS2.Config.create[F]( + queue <- Queue + .bounded[F, CommittableRecord[F]](fs2Config.queueSize) + .toResource + underlying <- KCLConsumer.Config.create[F]( checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig, metricsConfig, retrievalConfig, - fs2Config, processConfig.copy(recordProcessorConfig = processConfig.recordProcessorConfig.copy(autoCommit = autoCommit) ), encoders - ) - - } yield config + )(KCLConsumerFS2.callback(queue)) + } yield KCLConsumerFS2.Config(underlying, queue, fs2Config) - def readFS2Config( - prefix: Option[String] = None + private[kinesis4cats] def readFS2Config( + prefix: Option[String] ): ConfigValue[Effect, KCLConsumerFS2.FS2Config] = for { queueSize <- CirisReader .readDefaulted[Int](List("kcl", "fs2", "queue", "size"), 100, prefix) diff --git a/kcl-ciris/src/test/scala/kinesis4cats/kcl/ciris/KCLCirisSpec.scala b/kcl-ciris/src/test/scala/kinesis4cats/kcl/ciris/KCLCirisSpec.scala index 71111789..3d36529f 100644 --- a/kcl-ciris/src/test/scala/kinesis4cats/kcl/ciris/KCLCirisSpec.scala +++ b/kcl-ciris/src/test/scala/kinesis4cats/kcl/ciris/KCLCirisSpec.scala @@ -45,8 +45,12 @@ class KCLCirisSpec extends munit.CatsEffectSuite { "It should load the environment variables the same as system properties for CoordinatorConfig" ) { for { - configEnv <- KCLCiris.Coordinator.load[IO](prefix = Some("env")) - configProp <- KCLCiris.Coordinator.load[IO](prefix = Some("prop")) + configEnv <- KCLCiris.Coordinator + .read(Some("env"), None, None, None) + .load[IO] + configProp <- KCLCiris.Coordinator + .read(Some("prop"), None, None, None) + .load[IO] expected = new CoordinatorConfig(BuildInfo.kclAppName) .safeTransform(BuildInfo.kclCoordinatorMaxInitializationAttempts.toInt)( _.maxInitializationAttempts(_) @@ -91,9 +95,29 @@ class KCLCirisSpec extends munit.CatsEffectSuite { dynamoClient <- AwsClients.dynamoClient[IO]() kinesisClient <- AwsClients.kinesisClient[IO]() configEnv <- KCLCiris.Lease - .load[IO](dynamoClient, kinesisClient, prefix = Some("env")) + .read( + dynamoClient, + kinesisClient, + Some("env"), + None, + None, + None, + None, + None + ) + .load[IO] configProp <- KCLCiris.Lease - .load[IO](dynamoClient, kinesisClient, prefix = Some("prop")) + .read( + dynamoClient, + kinesisClient, + Some("prop"), + None, + None, + None, + None, + None + ) + .load[IO] expected = new LeaseManagementConfig( BuildInfo.kclLeaseTableName, dynamoClient, @@ -170,8 +194,8 @@ class KCLCirisSpec extends munit.CatsEffectSuite { "It should load the environment variables the same as system properties for LifecycleConfig" ) { for { - configEnv <- KCLCiris.Lifecycle.load[IO](prefix = Some("env")) - configProp <- KCLCiris.Lifecycle.load[IO](prefix = Some("prop")) + configEnv <- KCLCiris.Lifecycle.read(Some("env"), None, None).load[IO] + configProp <- KCLCiris.Lifecycle.read(Some("prop"), None, None).load[IO] expected = new LifecycleConfig() .logWarningForTaskAfterMillis( java.lang.Long @@ -207,9 +231,11 @@ class KCLCirisSpec extends munit.CatsEffectSuite { for { cloudwatchClient <- AwsClients.cloudwatchClient[IO]() configEnv <- KCLCiris.Metrics - .load[IO](cloudwatchClient, prefix = Some("env")) + .read(cloudwatchClient, Some("env"), None) + .load[IO] configProp <- KCLCiris.Metrics - .load[IO](cloudwatchClient, prefix = Some("prop")) + .read(cloudwatchClient, Some("prop"), None) + .load[IO] expected = new MetricsConfig( cloudwatchClient, BuildInfo.kclMetricsNamespace @@ -247,13 +273,17 @@ class KCLCirisSpec extends munit.CatsEffectSuite { for { kinesisClient <- AwsClients.kinesisClient[IO]() fanoutConfigEnv <- KCLCiris.Retrieval - .load[IO](kinesisClient, prefix = Some("FANOUT_ENV")) + .read(kinesisClient, Some("FANOUT_ENV"), None) + .load[IO] fanoutConfigProp <- KCLCiris.Retrieval - .load[IO](kinesisClient, prefix = Some("fanout.prop")) + .read(kinesisClient, Some("fanout.prop"), None) + .load[IO] pollingConfigEnv <- KCLCiris.Retrieval - .load[IO](kinesisClient, prefix = Some("POLLING_ENV")) + .read(kinesisClient, Some("POLLING_ENV"), None) + .load[IO] pollingConfigProp <- KCLCiris.Retrieval - .load[IO](kinesisClient, prefix = Some("polling.prop")) + .read(kinesisClient, Some("polling.prop"), None) + .load[IO] pollingExpected = new RetrievalConfig( kinesisClient, new SingleStreamTracker( @@ -380,8 +410,8 @@ class KCLCirisSpec extends munit.CatsEffectSuite { "It should load the environment variables the same as system properties for ProcessorConfig" ) { for { - configEnv <- KCLCiris.Processor.load[IO](prefix = Some("env")) - configProp <- KCLCiris.Processor.load[IO](prefix = Some("prop")) + configEnv <- KCLCiris.Processor.read(prefix = Some("env")) + configProp <- KCLCiris.Processor.read(prefix = Some("prop")) expected = KCLConsumer.ProcessConfig( BuildInfo.kclProcessorRaiseOnError.toBoolean, RecordProcessor.Config( diff --git a/kcl-localstack/src/main/scala/kinesis4cats/kcl/fs2/localstack/LocalstackKCLConsumerFS2.scala b/kcl-localstack/src/main/scala/kinesis4cats/kcl/fs2/localstack/LocalstackKCLConsumerFS2.scala index e7deea60..d1eca8f9 100644 --- a/kcl-localstack/src/main/scala/kinesis4cats/kcl/fs2/localstack/LocalstackKCLConsumerFS2.scala +++ b/kcl-localstack/src/main/scala/kinesis4cats/kcl/fs2/localstack/LocalstackKCLConsumerFS2.scala @@ -19,211 +19,85 @@ package fs2 package localstack import cats.Parallel -import cats.effect.std.Queue -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import software.amazon.kinesis.processor.StreamTracker +import software.amazon.kinesis.retrieval.polling.PollingConfig -import kinesis4cats.Utils -import kinesis4cats.kcl.localstack.LocalstackKCLConsumer import kinesis4cats.localstack.LocalstackConfig +import kinesis4cats.localstack.aws.v2.AwsClients /** Helpers for constructing and leveraging the KCL with Localstack via FS2. */ object LocalstackKCLConsumerFS2 { - /** Creates a - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] that - * is compliant with Localstack. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param workerId - * Unique identifier for the worker. Typically a UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] - */ - def kclConfig[F[_]]( - config: LocalstackConfig, - streamTracker: StreamTracker, - appName: String, - workerId: String, - processConfig: KCLConsumer.ProcessConfig, - encoders: RecordProcessor.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, KCLConsumerFS2.Config[F]] = for { - queue <- Queue.bounded[F, CommittableRecord[F]](100).toResource - underlying <- LocalstackKCLConsumer.kclConfig( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - )(KCLConsumerFS2.callback(queue)) - } yield KCLConsumerFS2 - .Config[F](underlying, queue, KCLConsumerFS2.FS2Config.default) + final case class Builder[F[_]] private ( + kclBuilder: KCLConsumerFS2.Builder[F] + )(implicit F: Async[F]) { + def configure(f: KCLConsumerFS2.Builder[F] => KCLConsumerFS2.Builder[F]) = + copy( + kclBuilder = f(kclBuilder) + ) - /** Creates a - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] that - * is compliant with Localstack. - * - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param workerId - * Unique identifier for the worker. Default is a random UUID - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * Default is TRIM_HORIZON - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * Default is `ProcessConfig.default` with autoCommit set to false - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] - */ - def kclConfig[F[_]]( - streamTracker: StreamTracker, - appName: String, - prefix: Option[String] = None, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = - KCLConsumerFS2.defaultProcessConfig, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KCLConsumerFS2.Config[F]] = for { - config <- LocalstackConfig.resource(prefix) - result <- kclConfig( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - ) - } yield result + def build: Resource[F, KCLConsumerFS2[F]] = kclBuilder.build + } - /** Runs a [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] that is - * compliant with Localstack. Also exposes a - * [[cats.effect.Deferred Deferred]] that will complete when the consumer has - * started processing records. Useful for allowing tests time for the - * consumer to start before processing the stream. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param workerId - * Unique identifier for the worker. Typically a UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2]] in a - * [[cats.effect.Resource Resource]] - */ - def kclConsumer[F[_]]( - config: LocalstackConfig, - streamTracker: StreamTracker, - appName: String, - workerId: String, - processConfig: KCLConsumer.ProcessConfig, - encoders: RecordProcessor.LogEncoders - )(implicit - F: Async[F], - P: Parallel[F] - ): Resource[F, KCLConsumerFS2[F]] = - kclConfig( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - ).map( - new KCLConsumerFS2[F](_) - ) + object Builder { + def default[F[_]]( + localstackConfig: LocalstackConfig, + streamTracker: StreamTracker, + appName: String + )(implicit F: Async[F], P: Parallel[F]): Resource[F, Builder[F]] = for { + kinesisClient <- AwsClients.kinesisClientResource(localstackConfig) + cloudWatchClient <- AwsClients.cloudwatchClientResource(localstackConfig) + dynamoClient <- AwsClients.dynamoClientResource(localstackConfig) + default <- KCLConsumerFS2.Builder.default( + streamTracker, + appName, + kinesisClient, + dynamoClient, + cloudWatchClient, + false + ) + retrievalConfig = + if (streamTracker.isMultiStream()) new PollingConfig(kinesisClient) + else + new PollingConfig( + streamTracker.streamConfigList.get(0).streamIdentifier.streamName, + kinesisClient + ) + initial = default + .configure(x => + x.configureLeaseManagementConfig(_.shardSyncIntervalMillis(1000L)) + .configureCoordinatorConfig(_.parentShardPollIntervalMillis(1000L)) + .configureRetrievalConfig( + _.retrievalSpecificConfig(retrievalConfig) + .retrievalFactory(retrievalConfig.retrievalFactory()) + ) + ) + kclBuilder = + if (streamTracker.isMultiStream()) initial + else + initial.configure( + _.configureLeaseManagementConfig( + _.initialPositionInStream( + streamTracker.streamConfigList + .get(0) + .initialPositionInStreamExtended() + ) + ) + ) + } yield Builder(kclBuilder) - /** Runs a [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] that is - * compliant with Localstack. Also exposes a - * [[cats.effect.Deferred Deferred]] that will complete when the consumer has - * started processing records. Useful for allowing tests time for the - * consumer to start before processing the stream. - * - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param workerId - * Unique identifier for the worker. Default to a random UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]]. - * Default to TRIM_HORIZON - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * Default is `ProcessConfig.default` with autoCommit set to false - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] in a - * [[cats.effect.Resource Resource]] - */ - def kclConsumer[F[_]]( - streamTracker: StreamTracker, - appName: String, - prefix: Option[String] = None, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = - KCLConsumerFS2.defaultProcessConfig, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(implicit - F: Async[F], - P: Parallel[F] - ): Resource[F, KCLConsumerFS2[F]] = for { - config <- LocalstackConfig.resource(prefix) - result <- kclConsumer( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - ) - } yield result + def default[F[_]]( + streamTracker: StreamTracker, + appName: String, + prefix: Option[String] = None + )(implicit F: Async[F], P: Parallel[F]): Resource[F, Builder[F]] = + LocalstackConfig + .resource[F](prefix) + .flatMap(default(_, streamTracker, appName)) + + @annotation.unused + def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kcl-localstack/src/main/scala/kinesis4cats/kcl/localstack/LocalstackKCLConsumer.scala b/kcl-localstack/src/main/scala/kinesis4cats/kcl/localstack/LocalstackKCLConsumer.scala index bf5bfa72..8332b1b1 100644 --- a/kcl-localstack/src/main/scala/kinesis4cats/kcl/localstack/LocalstackKCLConsumer.scala +++ b/kcl-localstack/src/main/scala/kinesis4cats/kcl/localstack/LocalstackKCLConsumer.scala @@ -21,16 +21,9 @@ import cats.effect.std.Queue import cats.effect.syntax.all._ import cats.effect.{Async, Deferred, Resource} import cats.syntax.all._ -import software.amazon.kinesis.checkpoint.CheckpointConfig -import software.amazon.kinesis.coordinator.CoordinatorConfig -import software.amazon.kinesis.leases.LeaseManagementConfig -import software.amazon.kinesis.lifecycle.LifecycleConfig -import software.amazon.kinesis.metrics.MetricsConfig import software.amazon.kinesis.processor.StreamTracker -import software.amazon.kinesis.retrieval.RetrievalConfig import software.amazon.kinesis.retrieval.polling.PollingConfig -import kinesis4cats.Utils import kinesis4cats.localstack.LocalstackConfig import kinesis4cats.localstack.aws.v2.AwsClients @@ -48,458 +41,93 @@ object LocalstackKCLConsumer { resultsQueue: Queue[F, CommittableRecord[F]] ) - /** Creates a [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] that - * is compliant with Localstack. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param workerId - * Unique identifier for the worker. Typically a UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param cb - * User-defined callback function for processing records - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] - */ - def kclConfig[F[_]]( - config: LocalstackConfig, - streamTracker: StreamTracker, - appName: String, - workerId: String, - processConfig: KCLConsumer.ProcessConfig, - encoders: RecordProcessor.LogEncoders - )( - cb: List[CommittableRecord[F]] => F[Unit] - )(implicit F: Async[F]): Resource[F, KCLConsumer.Config[F]] = for { - kinesisClient <- AwsClients.kinesisClientResource(config) - cloudwatchClient <- AwsClients.cloudwatchClientResource(config) - dynamoClient <- AwsClients.dynamoClientResource(config) - initialLeaseManagementConfig = new LeaseManagementConfig( - appName, - dynamoClient, - kinesisClient, - workerId - ).shardSyncIntervalMillis(1000L) - retrievalConfig = - if (streamTracker.isMultiStream()) new PollingConfig(kinesisClient) - else - new PollingConfig( - streamTracker.streamConfigList.get(0).streamIdentifier.streamName, - kinesisClient - ) - result <- KCLConsumer.Config.create[F]( - new CheckpointConfig(), - new CoordinatorConfig(appName).parentShardPollIntervalMillis(1000L), - if (streamTracker.isMultiStream()) initialLeaseManagementConfig - else - initialLeaseManagementConfig.initialPositionInStream( - streamTracker.streamConfigList - .get(0) - .initialPositionInStreamExtended() - ), - new LifecycleConfig(), - new MetricsConfig(cloudwatchClient, appName), - new RetrievalConfig(kinesisClient, streamTracker, appName) - .retrievalSpecificConfig(retrievalConfig) - .retrievalFactory(retrievalConfig.retrievalFactory()), - processConfig = processConfig, - encoders - )(cb) - } yield result - - /** Creates a [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] that - * is compliant with Localstack. - * - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param workerId - * Unique identifier for the worker. Default is a random UUID - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * Default is TRIM_HORIZON - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * Default is `ProcessConfig.default` - * @param cb - * User-defined callback function for processing records - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] - */ - def kclConfig[F[_]]( - streamTracker: StreamTracker, - appName: String, - prefix: Option[String] = None, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = - KCLConsumer.ProcessConfig.default, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, KCLConsumer.Config[F]] = for { - config <- LocalstackConfig.resource(prefix) - result <- kclConfig( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - )(cb) - } yield result - - /** Creates a [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] that - * is compliant with Localstack. Also creates a results - * [[cats.effect.std.Queue queue]] for the consumer to stick results into. - * Helpful when confirming data that has been produced to a stream. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param workerId - * Unique identifier for the worker. Typically a UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param resultsQueueSize - * Bounded size of the [[cats.effect.std.Queue Queue]] - * @param cb - * User-defined callback function for processing records. This will run - * after the records are enqueued into the results queue - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.localstack.LocalstackKCLConsumer.ConfigWithResults ConfigWithResults]] - */ - def kclConfigWithResults[F[_]]( - config: LocalstackConfig, - streamTracker: StreamTracker, - appName: String, - workerId: String, - processConfig: KCLConsumer.ProcessConfig, - resultsQueueSize: Int, - encoders: RecordProcessor.LogEncoders - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, ConfigWithResults[F]] = for { - resultsQueue <- Queue - .bounded[F, CommittableRecord[F]](resultsQueueSize) - .toResource - kclConf <- kclConfig( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - )((recs: List[CommittableRecord[F]]) => - resultsQueue.tryOfferN(recs) >> cb(recs) + final case class Builder[F[_]] private ( + kclBuilder: KCLConsumer.Builder[F] + )(implicit F: Async[F]) { + def configure( + f: KCLConsumer.Builder[F] => KCLConsumer.Builder[F] + ): Builder[F] = copy( + kclBuilder = f(kclBuilder) ) - } yield ConfigWithResults(kclConf, resultsQueue) - - /** Creates a [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] that - * is compliant with Localstack. Also creates a results - * [[cats.effect.std.Queue queue]] for the consumer to stick results into. - * Helpful when confirming data that has been produced to a stream. - * - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param workerId - * Unique identifier for the worker. Default to random UUID - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * Default is `ProcessConfig.default` - * @param resultsQueueSize - * Bounded size of the [[cats.effect.std.Queue Queue]]. Default to 50. - * @param cb - * User-defined callback function for processing records. This will run - * after the records are enqueued into the results queue - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.localstack.LocalstackKCLConsumer.ConfigWithResults ConfigWithResults]] - */ - def kclConfigWithResults[F[_]]( - streamTracker: StreamTracker, - appName: String, - prefix: Option[String] = None, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = - KCLConsumer.ProcessConfig.default, - resultsQueueSize: Int = 50, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, ConfigWithResults[F]] = for { - config <- LocalstackConfig.resource(prefix) - result <- kclConfigWithResults( - config, - streamTracker, - appName, - workerId, - processConfig, - resultsQueueSize, - encoders - )(cb) - } yield result - - /** Runs a [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] that is compliant with - * Localstack. Also exposes a [[cats.effect.Deferred Deferred]] that will - * complete when the consumer has started processing records. Useful for - * allowing tests time for the consumer to start before processing the - * stream. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param workerId - * Unique identifier for the worker. Typically a UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param cb - * User-defined callback function for processing records - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[cats.effect.Deferred Deferred]] in a - * [[cats.effect.Resource Resource]], which completes when the consumer has - * started processing records - */ - def kclConsumer[F[_]]( - config: LocalstackConfig, - streamTracker: StreamTracker, - appName: String, - workerId: String, - processConfig: KCLConsumer.ProcessConfig, - encoders: RecordProcessor.LogEncoders - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, Deferred[F, Unit]] = for { - config <- kclConfig( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - )(cb) - consumer = new KCLConsumer(config) - deferred <- consumer.runWithDeferredListener() - } yield deferred - - /** Runs a [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] that is compliant with - * Localstack. Also exposes a [[cats.effect.Deferred Deferred]] that will - * complete when the consumer has started processing records. Useful for - * allowing tests time for the consumer to start before processing the - * stream. - * - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param workerId - * Unique identifier for the worker. Default to a random UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]]. - * Default to TRIM_HORIZON - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * Default is `ProcessConfig.default` - * @param cb - * User-defined callback function for processing records - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[cats.effect.Deferred Deferred]] in a - * [[cats.effect.Resource Resource]], which completes when the consumer has - * started processing records - */ - def kclConsumer[F[_]]( - streamTracker: StreamTracker, - appName: String, - prefix: Option[String] = None, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = - KCLConsumer.ProcessConfig.default, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, Deferred[F, Unit]] = for { - config <- LocalstackConfig.resource(prefix) - result <- kclConsumer( - config, - streamTracker, - appName, - workerId, - processConfig, - encoders - )(cb) - } yield result + def withCallback(f: List[CommittableRecord[F]] => F[Unit]): Builder[F] = + copy(kclBuilder = kclBuilder.withCallback(f)) + def build: Resource[F, KCLConsumer[F]] = kclBuilder.build + def run: Resource[F, Deferred[F, Unit]] = + build.flatMap(_.runWithDeferredListener()) + def runWithResults( + resultsQueueSize: Int = 50 + ): Resource[F, DeferredWithResults[F]] = for { + resultsQueue <- Queue + .bounded[F, CommittableRecord[F]](resultsQueueSize) + .toResource + consumer <- kclBuilder + .configure(x => + x.withCallback((recs: List[CommittableRecord[F]]) => + resultsQueue.tryOfferN(recs) >> x.callback(recs) + ) + ) + .build + deferred <- consumer.runWithDeferredListener() + } yield DeferredWithResults(deferred, resultsQueue) + } - /** Runs a [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] that is compliant with - * Localstack. Exposes a [[cats.effect.Deferred Deferred]] that will complete - * when the consumer has started processing records, as well as a - * [[cats.effect.std.Queue Queue]] for tracking the received records. Useful - * for allowing tests time for the consumer to start before processing the - * stream, and testing those records that have been received. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param workerId - * Unique identifier for the worker. Typically a UUID. - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param resultsQueueSize - * Bounded size of the [[cats.effect.std.Queue Queue]]. - * @param cb - * User-defined callback function for processing records - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.localstack.LocalstackKCLConsumer.DeferredWithResults DeferredWithResults]] - * in a [[cats.effect.Resource Resource]], which completes when the - * consumer has started processing records - */ - def kclConsumerWithResults[F[_]]( - config: LocalstackConfig, - streamTracker: StreamTracker, - appName: String, - workerId: String, - processConfig: KCLConsumer.ProcessConfig, - resultsQueueSize: Int, - encoders: RecordProcessor.LogEncoders - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, DeferredWithResults[F]] = for { - configWithResults <- kclConfigWithResults( - config, - streamTracker, - appName, - workerId, - processConfig, - resultsQueueSize, - encoders - )(cb) - consumer = new KCLConsumer(configWithResults.kclConfig) - deferred <- consumer.runWithDeferredListener() - } yield DeferredWithResults(deferred, configWithResults.resultsQueue) + object Builder { + def default[F[_]]( + localstackConfig: LocalstackConfig, + streamTracker: StreamTracker, + appName: String + )(implicit F: Async[F]): Resource[F, Builder[F]] = for { + kinesisClient <- AwsClients.kinesisClientResource(localstackConfig) + cloudWatchClient <- AwsClients.cloudwatchClientResource(localstackConfig) + dynamoClient <- AwsClients.dynamoClientResource(localstackConfig) + default <- KCLConsumer.Builder.default( + streamTracker, + appName, + kinesisClient, + dynamoClient, + cloudWatchClient, + false + ) + retrievalConfig = + if (streamTracker.isMultiStream()) new PollingConfig(kinesisClient) + else + new PollingConfig( + streamTracker.streamConfigList.get(0).streamIdentifier.streamName, + kinesisClient + ) + initial = default + .configure(x => + x.configureLeaseManagementConfig(_.shardSyncIntervalMillis(1000L)) + .configureCoordinatorConfig(_.parentShardPollIntervalMillis(1000L)) + .configureRetrievalConfig( + _.retrievalSpecificConfig(retrievalConfig) + .retrievalFactory(retrievalConfig.retrievalFactory()) + ) + ) + kclBuilder = + if (streamTracker.isMultiStream()) initial + else + initial.configure( + _.configureLeaseManagementConfig( + _.initialPositionInStream( + streamTracker.streamConfigList + .get(0) + .initialPositionInStreamExtended() + ) + ) + ) + } yield Builder(kclBuilder) - /** Runs a [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] that is compliant with - * Localstack. Exposes a [[cats.effect.Deferred Deferred]] that will complete - * when the consumer has started processing records, as well as a - * [[cats.effect.std.Queue Queue]] for tracking the received records. Useful - * for allowing tests time for the consumer to start before processing the - * stream, and testing those records that have been received. - * - * @param streamTracker - * Name of stream to consume - * @param appName - * Application name for the consumer. Used for the dynamodb table name as - * well as the metrics namespace. - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param workerId - * Unique identifier for the worker. Default to a random UUID - * @param position - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java InitialPositionInStreamExtended]]. - * Default to TRIM_HORIZON. - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * Default is `ProcessConfig.default` - * @param resultsQueueSize - * Bounded size of the [[cats.effect.std.Queue Queue]]. Default to 50. - * @param cb - * User-defined callback function for processing records - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * @return - * [[kinesis4cats.kcl.localstack.LocalstackKCLConsumer.DeferredWithResults DeferredWithResults]] - * in a [[cats.effect.Resource Resource]], which completes when the - * consumer has started processing records - */ - def kclConsumerWithResults[F[_]]( - streamTracker: StreamTracker, - appName: String, - prefix: Option[String] = None, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = - KCLConsumer.ProcessConfig.default, - resultsQueueSize: Int = 50, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, DeferredWithResults[F]] = for { - configWithResults <- kclConfigWithResults( - streamTracker, - appName, - prefix, - workerId, - processConfig, - resultsQueueSize, - encoders - )(cb) - consumer = new KCLConsumer(configWithResults.kclConfig) - deferred <- consumer.runWithDeferredListener() - } yield DeferredWithResults(deferred, configWithResults.resultsQueue) + def default[F[_]]( + streamTracker: StreamTracker, + appName: String, + prefix: Option[String] = None + )(implicit F: Async[F]): Resource[F, Builder[F]] = + LocalstackConfig + .resource[F](prefix) + .flatMap(default(_, streamTracker, appName)) + @annotation.unused + def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kcl/src/main/scala/kinesis4cats/kcl/KCLConsumer.scala b/kcl/src/main/scala/kinesis4cats/kcl/KCLConsumer.scala index c377b09e..895a08b4 100644 --- a/kcl/src/main/scala/kinesis4cats/kcl/KCLConsumer.scala +++ b/kcl/src/main/scala/kinesis4cats/kcl/KCLConsumer.scala @@ -24,7 +24,6 @@ import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.checkpoint.CheckpointConfig -import software.amazon.kinesis.common.ConfigsBuilder import software.amazon.kinesis.coordinator.WorkerStateChangeListener.WorkerState import software.amazon.kinesis.coordinator._ import software.amazon.kinesis.leases.LeaseManagementConfig @@ -122,49 +121,79 @@ class KCLConsumer[F[_]] private[kinesis4cats] ( object KCLConsumer { - /** Low-level constructor for the - * [[kinesis4cats.kcl.KCLConsumer KCLConsumer]]. - * - * @param checkpointConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] - * @param coordinatorConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] - * @param leaseManagementConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] - * @param lifecycleConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] - * @param metricsConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] - * @param retrievalConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param cb - * Function to process - * [[kinesis4cats.kcl.CommittableRecord CommittableRecords]] received from - * Kinesis - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] - */ - def apply[F[_]]( + final case class BuilderConfig[F[_]] private[kinesis4cats] ( checkpointConfig: CheckpointConfig, coordinatorConfig: CoordinatorConfig, leaseManagementConfig: LeaseManagementConfig, lifecycleConfig: LifecycleConfig, metricsConfig: MetricsConfig, retrievalConfig: RetrievalConfig, - processConfig: ProcessConfig = ProcessConfig.default, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(cb: List[CommittableRecord[F]] => F[Unit])(implicit - F: Async[F] - ): Resource[F, KCLConsumer[F]] = Config - .create( + processConfig: ProcessConfig, + encoders: RecordProcessor.LogEncoders, + callback: List[CommittableRecord[F]] => F[Unit] + ) { + def withLogEncoders( + encoders: RecordProcessor.LogEncoders + ): BuilderConfig[F] = + copy(encoders = encoders) + def withCheckpointConfig( + checkpointConfig: CheckpointConfig + ): BuilderConfig[F] = + copy(checkpointConfig = checkpointConfig) + def configureCheckpointConfig( + f: CheckpointConfig => CheckpointConfig + ): BuilderConfig[F] = + copy(checkpointConfig = f(checkpointConfig)) + def withCoordinatorConfig( + coordinatorConfig: CoordinatorConfig + ): BuilderConfig[F] = + copy(coordinatorConfig = coordinatorConfig) + def configureCoordinatorConfig( + f: CoordinatorConfig => CoordinatorConfig + ): BuilderConfig[F] = + copy(coordinatorConfig = f(coordinatorConfig)) + def withLeaseManagementConfig( + leaseManagementConfig: LeaseManagementConfig + ): BuilderConfig[F] = + copy(leaseManagementConfig = leaseManagementConfig) + def configureLeaseManagementConfig( + f: LeaseManagementConfig => LeaseManagementConfig + ): BuilderConfig[F] = + copy(leaseManagementConfig = f(leaseManagementConfig)) + def withLifecycleConfig( + lifecycleConfig: LifecycleConfig + ): BuilderConfig[F] = + copy(lifecycleConfig = lifecycleConfig) + def configureLifecycleConfig( + f: LifecycleConfig => LifecycleConfig + ): BuilderConfig[F] = + copy(lifecycleConfig = f(lifecycleConfig)) + def withMetricsConfig(metricsConfig: MetricsConfig): BuilderConfig[F] = + copy(metricsConfig = metricsConfig) + def configureMetricsConfig( + f: MetricsConfig => MetricsConfig + ): BuilderConfig[F] = + copy(metricsConfig = f(metricsConfig)) + def withRetrievalConfig( + retrievalConfig: RetrievalConfig + ): BuilderConfig[F] = + copy(retrievalConfig = retrievalConfig) + def configureRetrievalConfig( + f: RetrievalConfig => RetrievalConfig + ): BuilderConfig[F] = + copy(retrievalConfig = f(retrievalConfig)) + def withProcessConfig(processConfig: ProcessConfig): BuilderConfig[F] = + copy(processConfig = processConfig) + def configureProcessConfig( + f: ProcessConfig => ProcessConfig + ): BuilderConfig[F] = + copy(processConfig = f(processConfig)) + def withCallback( + callback: List[CommittableRecord[F]] => F[Unit] + ): BuilderConfig[F] = + copy(callback = callback) + + def build(implicit F: Async[F]): Resource[F, Config[F]] = Config.create[F]( checkpointConfig, coordinatorConfig, leaseManagementConfig, @@ -173,79 +202,72 @@ object KCLConsumer { retrievalConfig, processConfig, encoders - )(cb) - .map(new KCLConsumer[F](_)) + )(callback) + } - /** Constructor for the [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] that - * leverages the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ConfigsBuilder.java ConfigsBuilder]] - * from the KCL. This is a simpler entry-point for creating the - * configuration, and provides a transform function to add any custom - * configuration that was not covered by the default - * - * @param kinesisClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param dynamoClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/dynamodb/DynamoDbAsyncClient.html DynamoDbAsyncClient]] - * @param cloudWatchClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cloudwatch/CloudWatchClient.html CloudWatchClient]] - * @param streamTracker - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/StreamTracker.java StreamTracker]] - * to use, which defines the name of the stream(s) and the initial position - * within them - * @param appName - * Name of the application. Usually also the dynamo table name for - * checkpoints - * @param workerId - * Unique identifier for a single instance of this consumer. Default is a - * random UUID. - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param cb - * Function to process - * [[kinesis4cats.kcl.CommittableRecord CommittableRecords]] received from - * Kinesis - * @param tfn - * Function to update the - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]]. Useful for - * overriding defaults. - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] - * @return - */ - def configsBuilder[F[_]]( - kinesisClient: KinesisAsyncClient, - dynamoClient: DynamoDbAsyncClient, - cloudWatchClient: CloudWatchAsyncClient, - streamTracker: StreamTracker, - appName: String, - workerId: String = Utils.randomUUIDString, - processConfig: ProcessConfig = ProcessConfig.default, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )( - cb: List[CommittableRecord[F]] => F[Unit] - )( - tfn: Config[F] => Config[F] = (x: Config[F]) => x - )(implicit - F: Async[F] - ): Resource[F, KCLConsumer[F]] = Config - .configsBuilder( - kinesisClient, - dynamoClient, - cloudWatchClient, - streamTracker, - appName, - workerId, - processConfig, - encoders - )(cb)(tfn) - .map(new KCLConsumer[F](_)) + final case class Builder[F[_]] private ( + config: BuilderConfig[F] + )(implicit F: Async[F]) { + + def configure(f: BuilderConfig[F] => BuilderConfig[F]): Builder[F] = copy( + config = f(config) + ) + + def withCallback( + callback: List[CommittableRecord[F]] => F[Unit] + ): Builder[F] = + copy(config = config.withCallback(callback)) + + def build: Resource[F, KCLConsumer[F]] = + config.build.map(new KCLConsumer[F](_)) + } + + object Builder { + + def default[F[_]]( + streamTracker: StreamTracker, + appName: String, + kinesisClient: => KinesisAsyncClient = + KinesisAsyncClient.builder().build(), + dynamoClient: => DynamoDbAsyncClient = + DynamoDbAsyncClient.builder().build(), + cloudWatchClient: => CloudWatchAsyncClient = + CloudWatchAsyncClient.builder().build(), + managedClients: Boolean = true + )(implicit + F: Async[F] + ): Resource[F, Builder[F]] = for { + kClient <- + if (managedClients) + Resource.fromAutoCloseable( + F.delay(kinesisClient) + ) + else Resource.pure[F, KinesisAsyncClient](kinesisClient) + dClient <- + if (managedClients) Resource.fromAutoCloseable(F.delay(dynamoClient)) + else Resource.pure[F, DynamoDbAsyncClient](dynamoClient) + cClient <- + if (managedClients) + Resource.fromAutoCloseable(F.delay(cloudWatchClient)) + else Resource.pure[F, CloudWatchAsyncClient](cloudWatchClient) + workerId = Utils.randomUUIDString + } yield Builder( + BuilderConfig( + new CheckpointConfig(), + new CoordinatorConfig(appName), + new LeaseManagementConfig(appName, dClient, kClient, workerId), + new LifecycleConfig(), + new MetricsConfig(cClient, appName), + new RetrievalConfig(kClient, streamTracker, appName), + ProcessConfig.default, + RecordProcessor.LogEncoders.show, + (_: List[CommittableRecord[F]]) => F.unit + ) + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** Config class for the [[kinesis4cats.kcl.KCLConsumer KCLConsumer]] * @@ -272,7 +294,7 @@ object KCLConsumer { * [[https://github.com/awslabs/amazon-kinesis-client/issues/10 issue]] for * more information. */ - final case class Config[F[_]] private ( + final case class Config[F[_]] private[kinesis4cats] ( checkpointConfig: CheckpointConfig, coordinatorConfig: CoordinatorConfig, leaseManagementConfig: LeaseManagementConfig, @@ -379,96 +401,6 @@ object KCLConsumer { deferredException, processConfig.raiseOnError ) - - /** Constructor for the - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] that - * leverages the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ConfigsBuilder.java ConfigsBuilder]] - * from the KCL. This is a simpler entry-point for creating the - * configuration, and provides a transform function to add any custom - * configuration that was not covered by the default - * - * @param kinesisClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param dynamoClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/dynamodb/DynamoDbAsyncClient.html DynamoDbAsyncClient]] - * @param cloudWatchClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cloudwatch/CloudWatchClient.html CloudWatchClient]] - * @param streamTracker - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/StreamTracker.java StreamTracker]] - * to use, which defines the name of the stream(s) and the initial - * position within them - * @param appName - * Name of the application. Usually also the dynamo table name for - * checkpoints - * @param workerId - * Unique identifier for a single instance of this consumer. Default is a - * random UUID. - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param cb - * Function to process - * [[kinesis4cats.kcl.CommittableRecord CommittableRecords]] received - * from Kinesis - * @param tfn - * Function to update the - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]]. Useful for - * overriding defaults. - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]] - * @return - */ - def configsBuilder[F[_]]( - kinesisClient: KinesisAsyncClient, - dynamoClient: DynamoDbAsyncClient, - cloudWatchClient: CloudWatchAsyncClient, - streamTracker: StreamTracker, - appName: String, - workerId: String = Utils.randomUUIDString, - processConfig: ProcessConfig = ProcessConfig.default, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )( - cb: List[CommittableRecord[F]] => F[Unit] - )( - tfn: Config[F] => Config[F] = (x: Config[F]) => x - )(implicit - F: Async[F] - ): Resource[F, Config[F]] = for { - deferredException <- Resource.eval(Deferred[F, Throwable]) - processorFactory <- RecordProcessor.Factory[F]( - processConfig.recordProcessorConfig, - deferredException, - processConfig.raiseOnError, - encoders - )(cb) - confBuilder = new ConfigsBuilder( - streamTracker, - appName, - kinesisClient, - dynamoClient, - cloudWatchClient, - workerId, - processorFactory - ) - } yield tfn( - Config( - confBuilder.checkpointConfig(), - confBuilder.coordinatorConfig(), - confBuilder.leaseManagementConfig(), - confBuilder.lifecycleConfig(), - confBuilder.metricsConfig(), - confBuilder.processorConfig(), - confBuilder.retrievalConfig(), - deferredException, - processConfig.raiseOnError - ) - ) } /** Runs a [[https://github.com/awslabs/amazon-kinesis-client KCL Consumer]] diff --git a/kcl/src/main/scala/kinesis4cats/kcl/fs2/KCLConsumerFS2.scala b/kcl/src/main/scala/kinesis4cats/kcl/fs2/KCLConsumerFS2.scala index 2862cbc9..6328a55f 100644 --- a/kcl/src/main/scala/kinesis4cats/kcl/fs2/KCLConsumerFS2.scala +++ b/kcl/src/main/scala/kinesis4cats/kcl/fs2/KCLConsumerFS2.scala @@ -197,146 +197,74 @@ object KCLConsumerFS2 { ): List[CommittableRecord[F]] => F[Unit] = (records: List[CommittableRecord[F]]) => records.traverse_(queue.offer) - /** Low-level constructor for - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]]. - * - * @param checkpointConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] - * @param coordinatorConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] - * @param leaseManagementConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] - * @param lifecycleConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] - * @param metricsConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] - * @param retrievalConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] - * @param queueSize - * Size of the underlying queue for the FS2 stream. If the queue fills up, - * backpressure on the processors will occur. Default 100 - * @param commitMaxChunk - * Max records to be received in the commitRecords [[fs2.Pipe Pipe]] before - * a commit is run. Default is 1000 - * @param commitMaxWait - * Max duration to wait in commitRecords [[fs2.Pipe Pipe]] before a commit - * is run. Default is 10 seconds - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] - */ - def apply[F[_]]( - checkpointConfig: CheckpointConfig, - coordinatorConfig: CoordinatorConfig, - leaseManagementConfig: LeaseManagementConfig, - lifecycleConfig: LifecycleConfig, - metricsConfig: MetricsConfig, - retrievalConfig: RetrievalConfig, - fs2Config: FS2Config = FS2Config.default, - processConfig: KCLConsumer.ProcessConfig = defaultProcessConfig, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(implicit - F: Async[F], - P: Parallel[F] - ): Resource[F, KCLConsumerFS2[F]] = Config - .create( - checkpointConfig, - coordinatorConfig, - leaseManagementConfig, - lifecycleConfig, - metricsConfig, - retrievalConfig, - fs2Config, - processConfig, - encoders + final case class Builder[F[_]] private ( + config: KCLConsumer.BuilderConfig[F], + fs2Config: FS2Config + )(implicit F: Async[F], P: Parallel[F]) { + def configure( + f: KCLConsumer.BuilderConfig[F] => KCLConsumer.BuilderConfig[F] + ): Builder[F] = + copy(config = f(config)) + def configureFs2Config(f: FS2Config => FS2Config): Builder[F] = + copy(fs2Config = f(fs2Config)) + def withFs2Config(fs2Config: FS2Config): Builder[F] = + copy(fs2Config = fs2Config) + + def build: Resource[F, KCLConsumerFS2[F]] = for { + queue <- Queue + .bounded[F, CommittableRecord[F]](fs2Config.queueSize) + .toResource + underlying <- config.withCallback(callback(queue)).build + } yield new KCLConsumerFS2[F](Config(underlying, queue, fs2Config)) + } + + object Builder { + def default[F[_]]( + streamTracker: StreamTracker, + appName: String, + kinesisClient: => KinesisAsyncClient = + KinesisAsyncClient.builder().build(), + dynamoClient: => DynamoDbAsyncClient = + DynamoDbAsyncClient.builder().build(), + cloudWatchClient: => CloudWatchAsyncClient = + CloudWatchAsyncClient.builder().build(), + managedClients: Boolean = true + )(implicit + F: Async[F], + P: Parallel[F] + ): Resource[F, Builder[F]] = for { + kClient <- + if (managedClients) + Resource.fromAutoCloseable( + F.delay(kinesisClient) + ) + else Resource.pure[F, KinesisAsyncClient](kinesisClient) + dClient <- + if (managedClients) Resource.fromAutoCloseable(F.delay(dynamoClient)) + else Resource.pure[F, DynamoDbAsyncClient](dynamoClient) + cClient <- + if (managedClients) + Resource.fromAutoCloseable(F.delay(cloudWatchClient)) + else Resource.pure[F, CloudWatchAsyncClient](cloudWatchClient) + workerId = Utils.randomUUIDString + } yield Builder( + KCLConsumer.BuilderConfig( + new CheckpointConfig(), + new CoordinatorConfig(appName), + new LeaseManagementConfig(appName, dClient, kClient, workerId), + new LifecycleConfig(), + new MetricsConfig(cClient, appName), + new RetrievalConfig(kClient, streamTracker, appName), + defaultProcessConfig, + RecordProcessor.LogEncoders.show, + (_: List[CommittableRecord[F]]) => F.unit + ), + FS2Config.default ) - .map(new KCLConsumerFS2[F](_)) - /** Constructor for the [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] - * that leverages the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ConfigsBuilder.java ConfigsBuilder]] - * from the KCL. This is a simpler entry-point for creating the - * configuration, and provides a transform function to add any custom - * configuration that was not covered by the default - * - * @param kinesisClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param dynamoClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/dynamodb/DynamoDbAsyncClient.html DynamoDbAsyncClient]] - * @param cloudWatchClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cloudwatch/CloudWatchClient.html CloudWatchClient]] - * @param streamTracker - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/StreamTracker.java StreamTracker]] - * to use, which defines the name of the stream(s) and the initial position - * within them - * @param appName - * Name of the application. Usually also the dynamo table name for - * checkpoints - * @param queueSize - * Size of the underlying queue for the FS2 stream. If the queue fills up, - * backpressure on the processors will occur. Default 100 - * @param commitMaxChunk - * Max records to be received in the commitRecords [[fs2.Pipe Pipe]] before - * a commit is run. Default is 1000 - * @param commitMaxWait - * Max duration to wait in commitRecords [[fs2.Pipe Pipe]] before a commit - * is run. Default is 10 seconds - * @param workerId - * Unique identifier for a single instance of this consumer. Default is a - * random UUID. - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param tfn - * Function to update the - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]]. Useful for - * overriding defaults. - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] - */ - def configsBuilder[F[_]]( - kinesisClient: KinesisAsyncClient, - dynamoClient: DynamoDbAsyncClient, - cloudWatchClient: CloudWatchAsyncClient, - streamTracker: StreamTracker, - appName: String, - fs2Config: FS2Config = FS2Config.default, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = defaultProcessConfig, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )( - tfn: kinesis4cats.kcl.KCLConsumer.Config[ - F - ] => kinesis4cats.kcl.KCLConsumer.Config[F] = - (x: kinesis4cats.kcl.KCLConsumer.Config[F]) => x - )(implicit - F: Async[F], - P: Parallel[F] - ): Resource[F, KCLConsumerFS2[F]] = Config - .configsBuilder( - kinesisClient, - dynamoClient, - cloudWatchClient, - streamTracker, - appName, - fs2Config, - workerId, - processConfig, - encoders - )(tfn) - .map(new KCLConsumerFS2[F](_)) + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** Configuration for the * [[kinesis4cats.kcl.fs2.KCLConsumerFS2 KCLConsumerFS2]] @@ -357,154 +285,12 @@ object KCLConsumerFS2 { * @param maxCommitRetryDuration * Delay between retries of commits */ - final case class Config[F[_]]( + final case class Config[F[_]] private[kinesis4cats] ( underlying: kinesis4cats.kcl.KCLConsumer.Config[F], queue: Queue[F, CommittableRecord[F]], fs2Config: FS2Config ) - object Config { - - /** Low-level constructor for - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]]. - * - * @param checkpointConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java CheckpointConfig]] - * @param coordinatorConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java CoordinatorConfig]] - * @param leaseManagementConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java LeaseManagementConfig]] - * @param lifecycleConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java LifecycleConfig]] - * @param metricsConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java MetricsConfig]] - * @param retrievalConfig - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RetrievalConfig.java RetrievalConfig]] - * @param queueSize - * Size of the underlying queue for the FS2 stream. If the queue fills - * up, backpressure on the processors will occur. Default 100 - * @param commitMaxChunk - * Max records to be received in the commitRecords [[fs2.Pipe Pipe]] - * before a commit is run. Default is 1000 - * @param commitMaxWait - * Max duration to wait in commitRecords [[fs2.Pipe Pipe]] before a - * commit is run. Default is 10 seconds - * @param commitMaxRetries - * Max number of retries for a commit operation - * @param commitRetryInterval - * Delay between retries of commits - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] - */ - def create[F[_]]( - checkpointConfig: CheckpointConfig, - coordinatorConfig: CoordinatorConfig, - leaseManagementConfig: LeaseManagementConfig, - lifecycleConfig: LifecycleConfig, - metricsConfig: MetricsConfig, - retrievalConfig: RetrievalConfig, - fs2Config: FS2Config, - processConfig: KCLConsumer.ProcessConfig = defaultProcessConfig, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )(implicit F: Async[F]): Resource[F, Config[F]] = for { - queue <- Queue - .bounded[F, CommittableRecord[F]](fs2Config.queueSize) - .toResource - underlying <- kinesis4cats.kcl.KCLConsumer.Config - .create( - checkpointConfig, - coordinatorConfig, - leaseManagementConfig, - lifecycleConfig, - metricsConfig, - retrievalConfig, - processConfig, - encoders - )(callback(queue)) - } yield Config(underlying, queue, fs2Config) - - /** Constructor for the - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] - * that leverages the - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ConfigsBuilder.java ConfigsBuilder]] - * from the KCL. This is a simpler entry-point for creating the - * configuration, and provides a transform function to add any custom - * configuration that was not covered by the default - * - * @param kinesisClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param dynamoClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/dynamodb/DynamoDbAsyncClient.html DynamoDbAsyncClient]] - * @param cloudWatchClient - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cloudwatch/CloudWatchClient.html CloudWatchClient]] - * @param streamTracker - * [[https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/StreamTracker.java StreamTracker]] - * to use, which defines the name of the stream(s) and the initial - * position within them - * @param appName - * Name of the application. Usually also the dynamo table name for - * checkpoints - * @param workerId - * Unique identifier for a single instance of this consumer. Default is a - * random UUID. - * @param processConfig - * [[kinesis4cats.kcl.KCLConsumer.ProcessConfig KCLConsumer.ProcessConfig]] - * @param tfn - * Function to update the - * [[kinesis4cats.kcl.KCLConsumer.Config KCLConsumer.Config]]. Useful for - * overriding defaults. - * @param F - * [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.kcl.RecordProcessor.LogEncoders RecordProcessor.LogEncoders]] - * for encoding structured logs - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kcl.fs2.KCLConsumerFS2.Config KCLConsumerFS2.Config]] - */ - def configsBuilder[F[_]]( - kinesisClient: KinesisAsyncClient, - dynamoClient: DynamoDbAsyncClient, - cloudWatchClient: CloudWatchAsyncClient, - streamTracker: StreamTracker, - appName: String, - fs2Config: KCLConsumerFS2.FS2Config = KCLConsumerFS2.FS2Config.default, - workerId: String = Utils.randomUUIDString, - processConfig: KCLConsumer.ProcessConfig = defaultProcessConfig, - encoders: RecordProcessor.LogEncoders = RecordProcessor.LogEncoders.show - )( - tfn: kinesis4cats.kcl.KCLConsumer.Config[ - F - ] => kinesis4cats.kcl.KCLConsumer.Config[F] = - (x: kinesis4cats.kcl.KCLConsumer.Config[F]) => x - )(implicit - F: Async[F] - ): Resource[F, Config[F]] = for { - queue <- Queue - .bounded[F, CommittableRecord[F]](fs2Config.queueSize) - .toResource - underlying <- kinesis4cats.kcl.KCLConsumer.Config - .configsBuilder( - kinesisClient, - dynamoClient, - cloudWatchClient, - streamTracker, - appName, - workerId, - processConfig, - encoders - )(callback(queue))(tfn) - } yield Config(underlying, queue, fs2Config) - } - /** Configuration for the FS2 implementation * * @param queueSize diff --git a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackCloudWatchClient.scala b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackCloudWatchClient.scala index ffca73b5..d8a45683 100644 --- a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackCloudWatchClient.scala +++ b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackCloudWatchClient.scala @@ -17,97 +17,70 @@ package kinesis4cats.client package localstack -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ +import org.typelevel.log4cats.StructuredLogger import org.typelevel.log4cats.slf4j.Slf4jLogger +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient import kinesis4cats.localstack.LocalstackConfig import kinesis4cats.localstack.aws.v2.AwsClients object LocalstackCloudWatchClient { - /** Builds a [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] that is - * compliant for Localstack usage. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.CloudWatchClient.LogEncoders LogEncoders]] - * @return - * F of [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] - */ - def client[F[_]]( - config: LocalstackConfig, - encoders: CloudWatchClient.LogEncoders - )(implicit - F: Async[F] - ): F[CloudWatchClient[F]] = - for { - underlying <- AwsClients.cloudwatchClient(config) - logger <- Slf4jLogger.create[F] - } yield new CloudWatchClient(underlying, logger, encoders) + final case class Builder[F[_]] private ( + encoders: CloudWatchClient.LogEncoders, + logger: StructuredLogger[F], + clientResource: Resource[F, CloudWatchAsyncClient] + )(implicit F: Async[F]) { + def withClient( + client: => CloudWatchAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + if (managed) Resource.fromAutoCloseable(F.delay(client)) + else Resource.pure(client) + ) - /** Builds a [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] that is - * compliant for Localstack usage. - * - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.CloudWatchClient.LogEncoders LogEncoders]] - * @return - * F of [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] - */ - def client[F[_]]( - prefix: Option[String] = None, - encoders: CloudWatchClient.LogEncoders = CloudWatchClient.LogEncoders.show - )(implicit F: Async[F]): F[CloudWatchClient[F]] = - for { - underlying <- AwsClients.cloudwatchClient(prefix) - logger <- Slf4jLogger.create[F] - } yield new CloudWatchClient(underlying, logger, encoders) + def withLogEncoders(encoders: CloudWatchClient.LogEncoders): Builder[F] = + copy( + encoders = encoders + ) - /** Builds a [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] that is - * compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.CloudWatchClient.LogEncoders LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] - */ - def clientResource[F[_]]( - config: LocalstackConfig, - encoders: CloudWatchClient.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, CloudWatchClient[F]] = - client[F](config, encoders).toResource + def withLogger(logger: StructuredLogger[F]): Builder[F] = copy( + logger = logger + ) - /** Builds a [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] that is - * compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param F - * F with an [[cats.effect.Async Async]] instance - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.CloudWatchClient CloudWatchClient]] - */ - def clientResource[F[_]]( - prefix: Option[String] = None, - encoders: CloudWatchClient.LogEncoders = CloudWatchClient.LogEncoders.show - )(implicit F: Async[F]): Resource[F, CloudWatchClient[F]] = - client[F](prefix, encoders).toResource + def build: Resource[F, CloudWatchClient[F]] = for { + underlying <- clientResource + client <- CloudWatchClient.Builder + .default[F] + .withClient(underlying, false) + .withLogEncoders(encoders) + .withLogger(logger) + .build + } yield client + } + + object Builder { + def default[F[_]]( + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig.load(prefix).map(default(_)) + + def default[F[_]]( + localstackConfig: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( + CloudWatchClient.LogEncoders.show, + Slf4jLogger.getLogger, + AwsClients.cloudwatchClientResource[F](localstackConfig) + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackDynamoClient.scala b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackDynamoClient.scala index 8dfeb939..4f5b883e 100644 --- a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackDynamoClient.scala +++ b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackDynamoClient.scala @@ -17,101 +17,69 @@ package kinesis4cats.client package localstack -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ +import org.typelevel.log4cats.StructuredLogger import org.typelevel.log4cats.slf4j.Slf4jLogger +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import kinesis4cats.localstack.LocalstackConfig import kinesis4cats.localstack.aws.v2.AwsClients object LocalstackDynamoClient { - /** Builds a [[kinesis4cats.client.DynamoClient DynamoClient]] that is - * compliant for Localstack usage. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.DynamoClient.LogEncoders LogEncoders]] - * @return - * F of [[kinesis4cats.client.DynamoClient DynamoClient]] - */ - def client[F[_]]( - config: LocalstackConfig, - encoders: DynamoClient.LogEncoders - )(implicit F: Async[F]): F[DynamoClient[F]] = - for { - underlying <- AwsClients.dynamoClient(config) - logger <- Slf4jLogger.create[F] - } yield new DynamoClient(underlying, logger, encoders) + final case class Builder[F[_]] private ( + encoders: DynamoClient.LogEncoders, + logger: StructuredLogger[F], + clientResource: Resource[F, DynamoDbAsyncClient] + )(implicit F: Async[F]) { + def withClient( + client: => DynamoDbAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + if (managed) Resource.fromAutoCloseable(F.delay(client)) + else Resource.pure(client) + ) - /** Builds a [[kinesis4cats.client.DynamoClient DynamoClient]] that is - * compliant for Localstack usage. - * - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.DynamoClient.LogEncoders LogEncoders]]. Defaults - * to show instances - * @return - * F of [[kinesis4cats.client.DynamoClient DynamoClient]] - */ - def client[F[_]]( - prefix: Option[String] = None, - encoders: DynamoClient.LogEncoders = DynamoClient.LogEncoders.show - )(implicit F: Async[F]): F[DynamoClient[F]] = - for { - underlying <- AwsClients.dynamoClient(prefix) - logger <- Slf4jLogger.create[F] - } yield new DynamoClient(underlying, logger, encoders) + def withLogEncoders(encoders: DynamoClient.LogEncoders): Builder[F] = copy( + encoders = encoders + ) - /** Builds a [[kinesis4cats.client.DynamoClient DynamoClient]] that is - * compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.DynamoClient.LogEncoders LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.DynamoClient DynamoClient]] - */ - def clientResource[F[_]]( - config: LocalstackConfig, - encoders: DynamoClient.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, DynamoClient[F]] = - client[F](config, encoders).toResource + def withLogger(logger: StructuredLogger[F]): Builder[F] = copy( + logger = logger + ) - /** Builds a [[kinesis4cats.client.DynamoClient DynamoClient]] that is - * compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param encoders - * [[kinesis4cats.client.DynamoClient.LogEncoders LogEncoders]]. Defaults - * to show instances - * @param F - * F with an [[cats.effect.Async Async]] instance - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.DynamoClient DynamoClient]] - */ - def clientResource[F[_]]( - prefix: Option[String] = None, - encoders: DynamoClient.LogEncoders = DynamoClient.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, DynamoClient[F]] = - client[F](prefix, encoders).toResource + def build: Resource[F, DynamoClient[F]] = for { + underlying <- clientResource + client <- DynamoClient.Builder + .default[F] + .withClient(underlying, false) + .withLogEncoders(encoders) + .withLogger(logger) + .build + } yield client + } + + object Builder { + def default[F[_]]( + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig.load(prefix).map(default(_)) + + def default[F[_]]( + localstackConfig: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( + DynamoClient.LogEncoders.show, + Slf4jLogger.getLogger, + AwsClients.dynamoClientResource[F](localstackConfig) + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackKinesisClient.scala b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackKinesisClient.scala index 73596a3b..ac2f88d3 100644 --- a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackKinesisClient.scala +++ b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/localstack/LocalstackKinesisClient.scala @@ -17,174 +17,87 @@ package kinesis4cats.client package localstack -import scala.concurrent.duration._ - -import cats.effect.std.Dispatcher -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ +import org.typelevel.log4cats.StructuredLogger import org.typelevel.log4cats.slf4j.Slf4jLogger +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.localstack.LocalstackConfig +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.localstack.aws.v2.AwsClients object LocalstackKinesisClient { - /** Builds a [[kinesis4cats.client.KinesisClient KinesisClient]] that is - * compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.KinesisClient.LogEncoders LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.KinesisClient KinesisClient]] - */ - def clientResource[F[_]]( - config: LocalstackConfig, - encoders: KinesisClient.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, KinesisClient[F]] = for { - underlying <- AwsClients.kinesisClient(config).toResource - logger <- Slf4jLogger.create[F].toResource - dispatcher <- Dispatcher.parallel[F] - } yield new KinesisClient(underlying, logger, dispatcher, encoders) - - /** Builds a [[kinesis4cats.client.KinesisClient KinesisClient]] that is - * compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param encoders - * [[kinesis4cats.client.KinesisClient.LogEncoders KinesisClient.LogEncoders]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.KinesisClient KinesisClient]] - */ - def clientResource[F[_]]( - prefix: Option[String] = None, - encoders: KinesisClient.LogEncoders = KinesisClient.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KinesisClient[F]] = - LocalstackConfig.resource(prefix).flatMap(clientResource[F](_, encoders)) + final case class Builder[F[_]] private ( + encoders: KinesisClient.LogEncoders, + logger: StructuredLogger[F], + streamsToCreate: List[TestStreamConfig[F]], + clientResource: Resource[F, KinesisAsyncClient] + )(implicit F: Async[F]) { + def withClient( + client: => KinesisAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + if (managed) Resource.fromAutoCloseable(F.delay(client)) + else Resource.pure(client) + ) - /** A resources that does the following: - * - * - Builds a [[kinesis4cats.client.KinesisClient KinesisClient]] that is - * compliant for Localstack usage. - * - Creates a stream with the desired name and shard count, and waits - * until the stream is active. - * - Destroys the stream when the [[cats.effect.Resource Resource]] is - * closed - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamName - * Stream name - * @param shardCount - * Shard count for stream - * @param describeRetries - * How many times to retry DescribeStreamSummary when checking the stream - * status - * @param describeRetryDuration - * How long to delay between retries of the DescribeStreamSummary call - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param LE - * [[kinesis4cats.client.KinesisClient.LogEncoders LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.KinesisClient KinesisClient]] - */ - def streamResource[F[_]]( - config: LocalstackConfig, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration, - encoders: KinesisClient.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, KinesisClient[F]] = for { - client <- clientResource(config, encoders) - result <- Resource.make( - AwsClients - .createStream( - client.client, - streamName, - shardCount, - describeRetries, - describeRetryDuration - ) - .as(client) - )(client => - AwsClients - .deleteStream( - client.client, - streamName, - describeRetries, - describeRetryDuration - ) + def withLogEncoders(encoders: KinesisClient.LogEncoders): Builder[F] = copy( + encoders = encoders ) - } yield result - /** A resources that does the following: - * - * - Builds a [[kinesis4cats.client.KinesisClient KinesisClient]] that is - * compliant for Localstack usage. - * - Creates a stream with the desired name and shard count, and waits - * until the stream is active. - * - Destroys the stream when the [[cats.effect.Resource Resource]] is - * closed - * - * @param streamName - * Stream name - * @param shardCount - * Shard count for stream - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param describeRetries - * How many times to retry DescribeStreamSummary when checking the stream - * status. Default to 5 - * @param describeRetryDuration - * How long to delay between retries of the DescribeStreamSummary call. - * Default to 500 ms - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.KinesisClient.LogEncoders LogEncoders]]. Default - * to show instances - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.KinesisClient KinesisClient]] - */ - def streamResource[F[_]]( - streamName: String, - shardCount: Int, - prefix: Option[String] = None, - describeRetries: Int = 5, - describeRetryDuration: FiniteDuration = 500.millis, - encoders: KinesisClient.LogEncoders = KinesisClient.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KinesisClient[F]] = for { - config <- LocalstackConfig.resource(prefix) - result <- streamResource( - config, - streamName, - shardCount, - describeRetries, - describeRetryDuration, - encoders + def withLogger(logger: StructuredLogger[F]): Builder[F] = copy( + logger = logger ) - } yield result + + def withStreamsToCreate( + streamsToCreate: List[TestStreamConfig[F]] + ): Builder[F] = + copy( + streamsToCreate = streamsToCreate + ) + + def build: Resource[F, KinesisClient[F]] = for { + underlying <- clientResource + client <- KinesisClient.Builder + .default[F] + .withClient(underlying, false) + .withLogEncoders(encoders) + .withLogger(logger) + .build + _ <- streamsToCreate.traverse_(config => managedStream(config, client)) + } yield client + } + + object Builder { + def default[F[_]]( + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig.load(prefix).map(default(_)) + + def default[F[_]]( + localstackConfig: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( + KinesisClient.LogEncoders.show, + Slf4jLogger.getLogger, + Nil, + AwsClients.kinesisClientResource[F](localstackConfig) + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } + + private[kinesis4cats] def managedStream[F[_]]( + config: TestStreamConfig[F], + client: KinesisClient[F] + )(implicit F: Async[F]): Resource[F, KinesisClient[F]] = Resource.make( + AwsClients.createStream(client.client, config).as(client) + )(c => AwsClients.deleteStream(c.client, config)) } diff --git a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala index f3f0b64c..3ab9aaa7 100644 --- a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala +++ b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala @@ -18,94 +18,131 @@ package kinesis4cats.client.producer package fs2 package localstack -import cats.Applicative +import _root_.fs2.concurrent.Channel import cats.effect._ +import cats.effect.syntax.all._ +import cats.syntax.all._ +import org.typelevel.log4cats.StructuredLogger +import org.typelevel.log4cats.slf4j.Slf4jLogger +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse +import kinesis4cats.client.KinesisClient +import kinesis4cats.client.localstack.LocalstackKinesisClient +import kinesis4cats.client.producer.localstack.LocalstackKinesisProducer import kinesis4cats.localstack.LocalstackConfig -import kinesis4cats.localstack.aws.v2.AwsClients +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer +import kinesis4cats.producer.Record +import kinesis4cats.producer.ShardMap +import kinesis4cats.producer.ShardMapCache import kinesis4cats.producer.fs2.FS2Producer object LocalstackFS2KinesisProducer { - /** Builds a [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - * that is compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param producerConfig - * [[kinesis4cats.producer.fs2.FS2Producer.Config FS2Producer.Config]] - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param callback - * Function that can be run after each of the put results from the - * underlying - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - */ - def resource[F[_]]( - producerConfig: FS2Producer.Config[F], - config: LocalstackConfig, - callback: (Producer.Res[PutRecordsResponse], Async[F]) => F[Unit], - encoders: KinesisProducer.LogEncoders - )(implicit F: Async[F]): Resource[F, FS2KinesisProducer[F]] = AwsClients - .kinesisClientResource[F](config) - .flatMap(underlying => - FS2KinesisProducer.instance[F]( - producerConfig, - underlying, - callback, - encoders - ) + final case class Builder[F[_]] private ( + clientResource: Resource[F, KinesisClient[F]], + localstackConfig: LocalstackConfig, + config: FS2Producer.Config[F], + logger: StructuredLogger[F], + encoders: KinesisProducer.LogEncoders, + streamsToCreate: List[TestStreamConfig[F]], + shardMapF: ( + KinesisClient[F], + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]], + callback: Producer.Res[PutRecordsResponse] => F[Unit] + )(implicit F: Async[F]) { + + def withLocalstackConfig(localstackConfig: LocalstackConfig): Builder[F] = + copy(localstackConfig = localstackConfig) + def withConfig(config: FS2Producer.Config[F]): Builder[F] = copy( + config = config + ) + def withClient( + client: => KinesisAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + KinesisClient.Builder.default.withClient(client, managed).build + ) + def withClient(client: KinesisClient[F]): Builder[F] = copy( + clientResource = Resource.pure(client) + ) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withLogEncoders(encoders: KinesisProducer.LogEncoders): Builder[F] = + copy(encoders = encoders) + def withStreamsToCreate(streamsToCreate: List[TestStreamConfig[F]]) = + copy(streamsToCreate = streamsToCreate) + def withShardMapF( + shardMapF: ( + KinesisClient[F], + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]] + ): Builder[F] = copy( + shardMapF = shardMapF ) - /** Builds a [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - * that is compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param streamName - * Name of stream for the producer to produce to - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param producerConfig - * String => - * [[kinesis4cats.producer.fs2.FS2Producer.Config FS2Producer.Config]] - * function that creates configuration given a stream name. Defaults to - * Producer.Config.default - * @param callback - * Function that can be run after each of the put results from the - * underlying. Defaults to F.unit. - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - */ - def resource[F[_]]( - streamName: String, - prefix: Option[String] = None, - producerConfig: (String, Applicative[F]) => FS2Producer.Config[F] = - (streamName: String, f: Applicative[F]) => - FS2Producer.Config.default[F](StreamNameOrArn.Name(streamName))(f), - callback: (Producer.Res[PutRecordsResponse], Async[F]) => F[Unit] = - (_: Producer.Res[PutRecordsResponse], f: Async[F]) => f.unit, - encoders: KinesisProducer.LogEncoders = KinesisProducer.LogEncoders.show - )(implicit F: Async[F]): Resource[F, FS2KinesisProducer[F]] = LocalstackConfig - .resource[F](prefix) - .flatMap( - resource[F]( - producerConfig(streamName, F), - _, - callback, - encoders + def build: Resource[F, FS2KinesisProducer[F]] = for { + client <- clientResource + underlying <- LocalstackKinesisProducer.Builder + .default[F]( + config.producerConfig.streamNameOrArn, + localstackConfig + ) + .withClient(client) + .withConfig(config.producerConfig) + .withLogEncoders(encoders) + .withLogger(logger) + .withStreamsToCreate(streamsToCreate) + .withShardMapF(shardMapF) + .build + channel <- Channel + .bounded[F, Record](config.queueSize) + .toResource + producer = new FS2KinesisProducer[F]( + logger, + config, + channel, + underlying + )( + callback ) - ) + _ <- producer.resource + } yield producer + } + + object Builder { + def default[F[_]]( + streamNameOrArn: StreamNameOrArn, + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig + .load(prefix) + .map(default(streamNameOrArn, _)) + + def default[F[_]]( + streamNameOrArn: StreamNameOrArn, + config: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( + LocalstackKinesisClient.Builder.default[F](config).build, + config, + FS2Producer.Config.default[F](streamNameOrArn), + Slf4jLogger.getLogger[F], + KinesisProducer.LogEncoders.show, + Nil, + (client: KinesisClient[F], snoa: StreamNameOrArn) => + KinesisProducer.getShardMap(client, snoa), + (_: Producer.Res[PutRecordsResponse]) => F.unit + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/localstack/LocalstackKinesisProducer.scala b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/localstack/LocalstackKinesisProducer.scala index 5631e2b1..39011c18 100644 --- a/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/localstack/LocalstackKinesisProducer.scala +++ b/kinesis-client-localstack/src/main/scala/kinesis4cats/client/producer/localstack/LocalstackKinesisProducer.scala @@ -16,16 +16,17 @@ package kinesis4cats.client.producer.localstack -import cats.Applicative import cats.effect._ -import cats.effect.syntax.all._ import cats.syntax.all._ +import org.typelevel.log4cats.StructuredLogger import org.typelevel.log4cats.slf4j.Slf4jLogger +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import kinesis4cats.client.KinesisClient +import kinesis4cats.client.localstack.LocalstackKinesisClient import kinesis4cats.client.producer.KinesisProducer import kinesis4cats.localstack.LocalstackConfig -import kinesis4cats.localstack.aws.v2.AwsClients +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.ShardMap @@ -33,100 +34,97 @@ import kinesis4cats.producer.ShardMapCache object LocalstackKinesisProducer { - /** Builds a [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - * that is compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param producerConfig - * [[kinesis4cats.producer.Producer.Config Producer.Config]] - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - */ - def resource[F[_]]( - producerConfig: Producer.Config[F], - config: LocalstackConfig, + final case class Builder[F[_]] private ( + clientResource: Resource[F, KinesisClient[F]], + localstackConfig: LocalstackConfig, + config: Producer.Config[F], + logger: StructuredLogger[F], + encoders: KinesisProducer.LogEncoders, + streamsToCreate: List[TestStreamConfig[F]], shardMapF: ( KinesisClient[F], - StreamNameOrArn, - Async[F] - ) => F[Either[ShardMapCache.Error, ShardMap]], - encoders: KinesisProducer.LogEncoders - )(implicit F: Async[F]): Resource[F, KinesisProducer[F]] = AwsClients - .kinesisClientResource[F](config) - .flatMap(_underlying => - for { - logger <- Slf4jLogger.create[F].toResource - underlying <- KinesisClient[F]( - _underlying, - encoders.kinesisClientLogEncoders - ) - shardMapCache <- ShardMapCache[F]( - producerConfig.shardMapCacheConfig, - shardMapF(underlying, producerConfig.streamNameOrArn, F), - Slf4jLogger.create[F].widen, + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]] + )(implicit F: Async[F]) { + + def withLocalstackConfig(localstackConfig: LocalstackConfig): Builder[F] = + copy(localstackConfig = localstackConfig) + def withConfig(config: Producer.Config[F]): Builder[F] = copy( + config = config + ) + def withClient( + client: => KinesisAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + KinesisClient.Builder.default.withClient(client, managed).build + ) + def withClient(client: KinesisClient[F]): Builder[F] = copy( + clientResource = Resource.pure(client) + ) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withLogEncoders(encoders: KinesisProducer.LogEncoders): Builder[F] = + copy(encoders = encoders) + def withStreamsToCreate(streamsToCreate: List[TestStreamConfig[F]]) = + copy(streamsToCreate = streamsToCreate) + def withShardMapF( + shardMapF: ( + KinesisClient[F], + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]] + ): Builder[F] = copy( + shardMapF = shardMapF + ) + + def build: Resource[F, KinesisProducer[F]] = for { + client <- clientResource + _ <- streamsToCreate.traverse_(x => + LocalstackKinesisClient.managedStream(x, client) + ) + shardMapCache <- ShardMapCache.Builder + .default[F](shardMapF(client, config.streamNameOrArn), logger) + .withLogEncoders( encoders.producerLogEncoders.shardMapLogEncoders ) - producer = new KinesisProducer[F]( - logger, - shardMapCache, - producerConfig, - underlying, - encoders.producerLogEncoders - ) - } yield producer + .build + } yield new KinesisProducer[F]( + logger, + shardMapCache, + config, + client, + encoders.producerLogEncoders ) + } - /** Builds a [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - * that is compliant for Localstack usage. Lifecycle is managed as a - * [[cats.effect.Resource Resource]]. - * - * @param streamName - * Name of stream for the producer to produce to - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param producerConfig - * String => [[kinesis4cats.producer.Producer.Config Producer.Config]] - * function that creates configuration given a stream name. Defaults to - * Producer.Config.default - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - */ - def resource[F[_]]( - streamName: String, - prefix: Option[String] = None, - producerConfig: (String, Applicative[F]) => Producer.Config[F] = - (streamName: String, f: Applicative[F]) => - Producer.Config.default[F](StreamNameOrArn.Name(streamName))(f), - shardMapF: ( - KinesisClient[F], - StreamNameOrArn, - Async[F] - ) => F[Either[ShardMapCache.Error, ShardMap]] = ( - client: KinesisClient[F], - streamNameOrArn: StreamNameOrArn, - f: Async[F] - ) => KinesisProducer.getShardMap(client, streamNameOrArn)(f), - encoders: KinesisProducer.LogEncoders = KinesisProducer.LogEncoders.show - )(implicit F: Async[F]): Resource[F, KinesisProducer[F]] = LocalstackConfig - .resource[F](prefix) - .flatMap( - resource[F]( - producerConfig(streamName, F), - _, - shardMapF, - encoders + object Builder { + def default[F[_]]( + streamNameOrArn: StreamNameOrArn, + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig + .load(prefix) + .map(default(streamNameOrArn, _)) + + def default[F[_]]( + streamNameOrArn: StreamNameOrArn, + config: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( + LocalstackKinesisClient.Builder.default[F](config).build, + config, + Producer.Config.default[F](streamNameOrArn), + Slf4jLogger.getLogger[F], + KinesisProducer.LogEncoders.show, + Nil, + (client: KinesisClient[F], snoa: StreamNameOrArn) => + KinesisProducer.getShardMap(client, snoa) ) - ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kinesis-client/src/main/scala/kinesis4cats/client/CloudWatchClient.scala b/kinesis-client/src/main/scala/kinesis4cats/client/CloudWatchClient.scala index bc54f93a..3bda8cef 100644 --- a/kinesis-client/src/main/scala/kinesis4cats/client/CloudWatchClient.scala +++ b/kinesis-client/src/main/scala/kinesis4cats/client/CloudWatchClient.scala @@ -20,7 +20,6 @@ package client import java.util.concurrent.CompletableFuture import cats.Show -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ import org.typelevel.log4cats.StructuredLogger @@ -101,29 +100,40 @@ class CloudWatchClient[F[_]] private[kinesis4cats] ( object CloudWatchClient { - /** Constructor for the CloudWatchClient, as a managed - * [[cats.effect.Resource Resource]] - * - * @param client - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/CloudWatchAsyncClient.html CloudWatchAsyncClient]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.CloudWatchClient.LogEncoders LogEncoders]]. - * Defaults to show instances - * @return - * [[cats.effect.Resource Resource]] containing a - * [[kinesis4cats.client.CloudWatchClient]] - */ - def apply[F[_]]( - client: CloudWatchAsyncClient, - encoders: CloudWatchClient.LogEncoders = CloudWatchClient.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, CloudWatchClient[F]] = for { - clientResource <- Resource.fromAutoCloseable(F.pure(client)) - logger <- Slf4jLogger.create[F].toResource - } yield new CloudWatchClient[F](clientResource, logger, encoders) + final case class Builder[F[_]] private ( + clientResource: Resource[F, CloudWatchAsyncClient], + encoders: LogEncoders, + logger: StructuredLogger[F] + )(implicit F: Async[F]) { + def withClient( + client: => CloudWatchAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + if (managed) Resource.fromAutoCloseable(F.delay(client)) + else Resource.pure(client) + ) + def withLogEncoders(encoders: LogEncoders): Builder[F] = + copy(encoders = encoders) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + + def build: Resource[F, CloudWatchClient[F]] = + clientResource.map(new CloudWatchClient[F](_, logger, encoders)) + } + + object Builder { + def default[F[_]](implicit F: Async[F]): Builder[F] = Builder[F]( + Resource.fromAutoCloseable( + F.delay(CloudWatchAsyncClient.builder().build()) + ), + LogEncoders.show, + Slf4jLogger.getLogger + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** Helper class containing required * [[kinesis4cats.logging.LogEncoder LogEncoders]] for the diff --git a/kinesis-client/src/main/scala/kinesis4cats/client/DynamoClient.scala b/kinesis-client/src/main/scala/kinesis4cats/client/DynamoClient.scala index e191d5ff..12d18821 100644 --- a/kinesis-client/src/main/scala/kinesis4cats/client/DynamoClient.scala +++ b/kinesis-client/src/main/scala/kinesis4cats/client/DynamoClient.scala @@ -24,7 +24,6 @@ import java.util.concurrent.CompletableFuture import cats.Eval import cats.Show -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ import org.typelevel.log4cats.StructuredLogger @@ -134,27 +133,40 @@ class DynamoClient[F[_]] private[kinesis4cats] ( object DynamoClient { - /** Constructor for the DynamoClient, as a managed - * [[cats.effect.Resource Resource]] - * - * @param client - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/DynamoDbAsyncClient.html DynamoDbAsyncClient]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.DynamoClient.LogEncoders LogEncoders]]. Default to - * show instances - * @return - * [[cats.effect.Resource Resource]] containing a - * [[kinesis4cats.client.DynamoClient]] - */ - def apply[F[_]]( - client: DynamoDbAsyncClient, - encoders: LogEncoders = LogEncoders.show - )(implicit F: Async[F]): Resource[F, DynamoClient[F]] = for { - clientResource <- Resource.fromAutoCloseable(F.pure(client)) - logger <- Slf4jLogger.create[F].toResource - } yield new DynamoClient[F](clientResource, logger, encoders) + final case class Builder[F[_]] private ( + clientResource: Resource[F, DynamoDbAsyncClient], + encoders: LogEncoders, + logger: StructuredLogger[F] + )(implicit F: Async[F]) { + def withClient( + client: => DynamoDbAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + if (managed) Resource.fromAutoCloseable(F.delay(client)) + else Resource.pure(client) + ) + def withLogEncoders(encoders: LogEncoders): Builder[F] = + copy(encoders = encoders) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + + def build: Resource[F, DynamoClient[F]] = + clientResource.map(new DynamoClient[F](_, logger, encoders)) + } + + object Builder { + def default[F[_]](implicit F: Async[F]): Builder[F] = Builder[F]( + Resource.fromAutoCloseable( + F.delay(DynamoDbAsyncClient.builder().build()) + ), + LogEncoders.show, + Slf4jLogger.getLogger + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** Helper class containing required * [[kinesis4cats.logging.LogEncoder LogEncoders]] for the diff --git a/kinesis-client/src/main/scala/kinesis4cats/client/KinesisClient.scala b/kinesis-client/src/main/scala/kinesis4cats/client/KinesisClient.scala index 76349cab..be72f11a 100644 --- a/kinesis-client/src/main/scala/kinesis4cats/client/KinesisClient.scala +++ b/kinesis-client/src/main/scala/kinesis4cats/client/KinesisClient.scala @@ -345,30 +345,40 @@ class KinesisClient[F[_]] private[kinesis4cats] ( object KinesisClient { - /** Constructor for the KinesisClient, as a managed - * [[cats.effect.Resource Resource]] - * - * @param client - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param encoders - * [[kinesis4cats.client.KinesisClient.LogEncoders LogEncoders]]. Default - * to show instances - * @return - * [[cats.effect.Resource Resource]] containing a - * [[kinesis4cats.client.KinesisClient]] - */ - def apply[F[_]]( - client: KinesisAsyncClient, - encoders: LogEncoders = LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KinesisClient[F]] = for { - clientResource <- Resource.fromAutoCloseable(F.pure(client)) - logger <- Slf4jLogger.create[F].toResource - dispatcher <- Dispatcher.parallel[F] - } yield new KinesisClient[F](clientResource, logger, dispatcher, encoders) + final case class Builder[F[_]] private ( + clientResource: Resource[F, KinesisAsyncClient], + encoders: LogEncoders, + logger: StructuredLogger[F] + )(implicit F: Async[F]) { + def withClient( + client: => KinesisAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + if (managed) Resource.fromAutoCloseable(F.delay(client)) + else Resource.pure(client) + ) + def withLogEncoders(encoders: LogEncoders): Builder[F] = + copy(encoders = encoders) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + + def build: Resource[F, KinesisClient[F]] = for { + client <- clientResource + dispatcher <- Dispatcher.parallel[F] + } yield new KinesisClient[F](client, logger, dispatcher, encoders) + } + + object Builder { + def default[F[_]](implicit F: Async[F]): Builder[F] = Builder[F]( + Resource.fromAutoCloseable(F.delay(KinesisAsyncClient.builder().build())), + LogEncoders.show, + Slf4jLogger.getLogger + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** Helper class containing required * [[kinesis4cats.logging.LogEncoder LogEncoders]] for the diff --git a/kinesis-client/src/main/scala/kinesis4cats/client/producer/KinesisProducer.scala b/kinesis-client/src/main/scala/kinesis4cats/client/producer/KinesisProducer.scala index e717a27a..bc8e5fd9 100644 --- a/kinesis-client/src/main/scala/kinesis4cats/client/producer/KinesisProducer.scala +++ b/kinesis-client/src/main/scala/kinesis4cats/client/producer/KinesisProducer.scala @@ -24,7 +24,6 @@ import java.time.Instant import cats.data.NonEmptyList import cats.effect.Resource import cats.effect._ -import cats.effect.syntax.all._ import cats.syntax.all._ import org.typelevel.log4cats.StructuredLogger import org.typelevel.log4cats.slf4j.Slf4jLogger @@ -107,6 +106,59 @@ final class KinesisProducer[F[_]] private[kinesis4cats] ( object KinesisProducer { + final case class Builder[F[_]] private ( + config: Producer.Config[F], + clientResource: Resource[F, KinesisClient[F]], + encoders: LogEncoders, + logger: StructuredLogger[F] + )(implicit F: Async[F]) { + def withConfig(config: Producer.Config[F]): Builder[F] = copy( + config = config + ) + def withClient( + client: => KinesisAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + KinesisClient.Builder.default.withClient(client, managed).build + ) + def withClient(client: KinesisClient[F]): Builder[F] = copy( + clientResource = Resource.pure(client) + ) + def withLogEncoders(encoders: LogEncoders): Builder[F] = + copy(encoders = encoders) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + + def build: Resource[F, KinesisProducer[F]] = for { + client <- clientResource + shardMapCache <- ShardMapCache.Builder + .default(getShardMap(client, config.streamNameOrArn), logger) + .withLogEncoders(encoders.producerLogEncoders.shardMapLogEncoders) + .build + } yield new KinesisProducer[F]( + logger, + shardMapCache, + config, + client, + encoders.producerLogEncoders + ) + } + + object Builder { + def default[F[_]]( + streamNameOrArn: models.StreamNameOrArn + )(implicit F: Async[F]): Builder[F] = Builder[F]( + Producer.Config.default(streamNameOrArn), + KinesisClient.Builder.default.build, + LogEncoders.show, + Slf4jLogger.getLogger + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } + final class LogEncoders( val kinesisClientLogEncoders: KinesisClient.LogEncoders, val producerLogEncoders: Producer.LogEncoders @@ -165,71 +217,4 @@ object KinesisProducer { ) ) ) - - /** Basic constructor for the - * [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - * - * @param config - * [[kinesis4cats.producer.Producer.Config Producer.Config]] - * @param _underlying - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * instance - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * Default to show instances - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - */ - def instance[F[_]]( - config: Producer.Config[F], - _underlying: KinesisAsyncClient, - encoders: LogEncoders = LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KinesisProducer[F]] = - KinesisClient[F](_underlying, encoders.kinesisClientLogEncoders).flatMap( - apply(config, _, encoders) - ) - - /** Basic constructor for the - * [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - * - * @param config - * [[kinesis4cats.producer.Producer.Config Producer.Config]] - * @param underlying - * [[kinesis4cats.client.KinesisClient KinesisClient]] instance - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.producer.Producer.LogEncoders Producer.LogEncoders]]. - * Default to show instances - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.KinesisProducer KinesisProducer]] - */ - def apply[F[_]]( - config: Producer.Config[F], - underlying: KinesisClient[F], - encoders: LogEncoders = LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KinesisProducer[F]] = for { - logger <- Slf4jLogger.create[F].toResource - shardMapCache <- ShardMapCache[F]( - config.shardMapCacheConfig, - getShardMap(underlying, config.streamNameOrArn), - Slf4jLogger.create[F].widen, - encoders.producerLogEncoders.shardMapLogEncoders - ) - producer = new KinesisProducer[F]( - logger, - shardMapCache, - config, - underlying, - encoders.producerLogEncoders - ) - } yield producer } diff --git a/kinesis-client/src/main/scala/kinesis4cats/client/producer/fs2/FS2KinesisProducer.scala b/kinesis-client/src/main/scala/kinesis4cats/client/producer/fs2/FS2KinesisProducer.scala index 96ed0bed..9d05314b 100644 --- a/kinesis-client/src/main/scala/kinesis4cats/client/producer/fs2/FS2KinesisProducer.scala +++ b/kinesis-client/src/main/scala/kinesis4cats/client/producer/fs2/FS2KinesisProducer.scala @@ -14,7 +14,8 @@ * limitations under the License. */ -package kinesis4cats.client +package kinesis4cats +package client package producer package fs2 @@ -51,85 +52,66 @@ final class FS2KinesisProducer[F[_]] private[kinesis4cats] ( override protected val channel: Channel[F, Record], override protected val underlying: KinesisProducer[F] )( - override protected val callback: ( - Producer.Res[PutRecordsResponse], - Async[F] - ) => F[Unit] + override protected val callback: Producer.Res[PutRecordsResponse] => F[Unit] )(implicit F: Async[F] ) extends FS2Producer[F, PutRecordsRequest, PutRecordsResponse] object FS2KinesisProducer { - - /** Basic constructor for the - * [[kinesis4cats.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - * - * @param config - * [[kinesis4cats.producer.fs2.FS2Producer.Config FS2Producer.Config]] - * @param _underlying - * [[https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/kinesis/KinesisAsyncClient.html KinesisAsyncClient]] - * instance - * @param callback - * Function that can be run after each of the put results from the - * underlying - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - */ - def instance[F[_]]( + final case class Builder[F[_]] private ( config: FS2Producer.Config[F], - _underlying: KinesisAsyncClient, - callback: (Producer.Res[PutRecordsResponse], Async[F]) => F[Unit] = - (_: Producer.Res[PutRecordsResponse], f: Async[F]) => f.unit, - encoders: KinesisProducer.LogEncoders = KinesisProducer.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, FS2KinesisProducer[F]] = - KinesisClient(_underlying, encoders.kinesisClientLogEncoders).flatMap( - apply(config, _, callback, encoders) + clientResource: Resource[F, KinesisClient[F]], + encoders: KinesisProducer.LogEncoders, + logger: StructuredLogger[F], + callback: Producer.Res[PutRecordsResponse] => F[Unit] + )(implicit F: Async[F]) { + def withConfig(config: FS2Producer.Config[F]): Builder[F] = copy( + config = config ) - - /** Basic constructor for the - * [[kinesis4cats.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - * - * @param config - * [[kinesis4cats.producer.fs2.FS2Producer.Config FS2Producer.Config]] - * @param underlying - * [[kinesis4cats.client.KinesisClient KinesisClient]] instance - * @param callback - * Function that can be run after each of the put results from the - * underlying - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - */ - def apply[F[_]]( - config: FS2Producer.Config[F], - underlying: KinesisClient[F], - callback: (Producer.Res[PutRecordsResponse], Async[F]) => F[Unit] = - (_: Producer.Res[PutRecordsResponse], f: Async[F]) => f.unit, - encoders: KinesisProducer.LogEncoders = KinesisProducer.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, FS2KinesisProducer[F]] = for { - logger <- Slf4jLogger.create[F].toResource - underlying <- KinesisProducer( - config.producerConfig, - underlying, - encoders + def withClient( + client: => KinesisAsyncClient, + managed: Boolean = true + ): Builder[F] = copy( + clientResource = + KinesisClient.Builder.default.withClient(client, managed).build ) - channel <- Channel.bounded[F, Record](config.queueSize).toResource - producer = new FS2KinesisProducer[F](logger, config, channel, underlying)( - callback + def withClient(client: KinesisClient[F]): Builder[F] = copy( + clientResource = Resource.pure(client) ) - _ <- producer.resource - } yield producer + def withLogEncoders(encoders: KinesisProducer.LogEncoders): Builder[F] = + copy(encoders = encoders) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + + def build: Resource[F, FS2KinesisProducer[F]] = for { + client <- clientResource + underlying <- KinesisProducer.Builder + .default[F](config.producerConfig.streamNameOrArn) + .withConfig(config.producerConfig) + .withLogEncoders(encoders) + .withLogger(logger) + .withClient(client) + .build + channel <- Channel.bounded[F, Record](config.queueSize).toResource + producer = new FS2KinesisProducer[F](logger, config, channel, underlying)( + callback + ) + _ <- producer.resource + } yield producer + } + + object Builder { + def default[F[_]]( + streamNameOrArn: models.StreamNameOrArn + )(implicit F: Async[F]): Builder[F] = Builder[F]( + FS2Producer.Config.default(streamNameOrArn), + KinesisClient.Builder.default.build, + KinesisProducer.LogEncoders.show, + Slf4jLogger.getLogger, + (_: Producer.Res[PutRecordsResponse]) => F.unit + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/kpl-ciris/src/main/scala/kinesis4cats/kpl/ciris/KPLCiris.scala b/kpl-ciris/src/main/scala/kinesis4cats/kpl/ciris/KPLCiris.scala index 003d517c..63998ad3 100644 --- a/kpl-ciris/src/main/scala/kinesis4cats/kpl/ciris/KPLCiris.scala +++ b/kpl-ciris/src/main/scala/kinesis4cats/kpl/ciris/KPLCiris.scala @@ -55,7 +55,7 @@ object KPLCiris { * [[https://cir.is/docs/configurations ConfigValue]] containing * [[https://github.com/awslabs/aws-glue-schema-registry/blob/master/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java GlueSchemaRegistryConfiguration]] */ - def readGlueConfig( + private[kinesis4cats] def readGlueConfig( prefix: Option[String] = None ): ConfigValue[Effect, GlueSchemaRegistryConfiguration] = for { @@ -157,13 +157,13 @@ object KPLCiris { * [[https://cir.is/docs/configurations ConfigValue]] containing * [[https://github.com/awslabs/amazon-kinesis-producer/blob/master/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java KinesisProducerConfiguration]] */ - def readKplConfig( + private[kinesis4cats] def readKplConfig( credentialsProvider: Option[AWSCredentialsProvider] = None, metricsCredentialsProvider: Option[AWSCredentialsProvider] = None, glueSchemaRegistryCredentialsProvider: Option[AwsCredentialsProvider] = None, prefix: Option[String] = None - ): ConfigValue[Effect, KinesisProducerConfiguration] = for { + ): ConfigValue[Effect, KPLProducer.Config] = for { additionalMetricsDimensions <- CirisReader .readOptional[List[AdditionalMetricsDimension]]( List("kpl", "additional", "metrics", "dimensions"), @@ -341,62 +341,80 @@ object KPLCiris { prefix ) .map(_.map(_.toMillis)) - } yield new KinesisProducerConfiguration() - .maybeTransform(credentialsProvider)(_.setCredentialsProvider(_)) - .maybeTransform(metricsCredentialsProvider)( - _.setMetricsCredentialsProvider(_) - ) - .maybeTransform(glueSchemaRegistryCredentialsProvider)( - _.setGlueSchemaRegistryCredentialsProvider(_) - ) - .maybeTransform(glueConfig)(_.setGlueSchemaRegistryConfiguration(_)) - .maybeTransform(caCertPath)(_.setCaCertPath(_)) - .maybeTransform(aggregationEnabled)(_.setAggregationEnabled(_)) - .maybeTransform(aggregationMaxCount)(_.setAggregationMaxCount(_)) - .maybeTransform(aggregationMaxSize)(_.setAggregationMaxSize(_)) - .maybeTransform(cloudwatchEndpoint)(_.setCloudwatchEndpoint(_)) - .maybeTransform(cloudwatchPort)(_.setCloudwatchPort(_)) - .maybeTransform(collectionMaxCount)(_.setCollectionMaxCount(_)) - .maybeTransform(collectionMaxSize)(_.setCollectionMaxSize(_)) - .maybeTransform(connectTimeout)(_.setConnectTimeout(_)) - .maybeTransform(credentialsRefreshDelay)(_.setCredentialsRefreshDelay(_)) - .maybeTransform(enableCoreDumps)(_.setEnableCoreDumps(_)) - .maybeTransform(failIfThrottled)(_.setFailIfThrottled(_)) - .maybeTransform(kinesisEndpoint)(_.setKinesisEndpoint(_)) - .maybeTransform(kinesisPort)(_.setKinesisPort(_)) - .maybeTransform(logLevel)(_.setLogLevel(_)) - .maybeTransform(maxConnections)(_.setMaxConnections(_)) - .maybeTransform(metricsGranularity)(_.setMetricsGranularity(_)) - .maybeTransform(metricsLevel)(_.setMetricsLevel(_)) - .maybeTransform(metricsNamespace)(_.setMetricsNamespace(_)) - .maybeTransform(metricsUploadDelay)(_.setMetricsUploadDelay(_)) - .maybeTransform(minConnections)(_.setMinConnections(_)) - .maybeTransform(nativeExecutable)(_.setNativeExecutable(_)) - .maybeTransform(rateLimit)(_.setRateLimit(_)) - .maybeTransform(recordMaxBufferedTime)(_.setRecordMaxBufferedTime(_)) - .maybeTransform(recordTtl)(_.setRecordTtl(_)) - .maybeTransform(region)(_.setRegion(_)) - .maybeTransform(requestTimeout)(_.setRequestTimeout(_)) - .maybeTransform(tempDirectory)(_.setTempDirectory(_)) - .maybeTransform(verifyCertificate)(_.setVerifyCertificate(_)) - .maybeTransform(proxyHost)(_.setProxyHost(_)) - .maybeTransform(proxyPort)(_.setProxyPort(_)) - .maybeTransform(proxyUserName)(_.setProxyUserName(_)) - .maybeTransform(proxyPassword)(_.setProxyPassword(_)) - .maybeTransform(threadingModel)(_.setThreadingModel(_)) - .maybeTransform(threadPoolSize)(_.setThreadPoolSize(_)) - .maybeTransform(userRecordTimeout)(_.setUserRecordTimeoutInMillis(_)) - .maybeRunUnsafe(additionalMetricsDimensions) { - case (conf, additionalDims) => - additionalDims.foreach(x => - conf.addAdditionalMetricsDimension( - x.key, - x.value, - x.granularity.value + gracefulShutdownFlushAttempts <- CirisReader + .readDefaulted[Int]( + List("kpl", "graceful", "shutdown", "flush", "attempts"), + 5, + prefix + ) + gracefulShutdownFlushInterval <- CirisReader + .readDefaulted[FiniteDuration]( + List("kpl", "graceful", "shutdown", "flush", "interval"), + 500.millis, + prefix + ) + } yield KPLProducer.Config( + new KinesisProducerConfiguration() + .maybeTransform(credentialsProvider)(_.setCredentialsProvider(_)) + .maybeTransform(metricsCredentialsProvider)( + _.setMetricsCredentialsProvider(_) + ) + .maybeTransform(glueSchemaRegistryCredentialsProvider)( + _.setGlueSchemaRegistryCredentialsProvider(_) + ) + .maybeTransform(glueConfig)(_.setGlueSchemaRegistryConfiguration(_)) + .maybeTransform(caCertPath)(_.setCaCertPath(_)) + .maybeTransform(aggregationEnabled)(_.setAggregationEnabled(_)) + .maybeTransform(aggregationMaxCount)(_.setAggregationMaxCount(_)) + .maybeTransform(aggregationMaxSize)(_.setAggregationMaxSize(_)) + .maybeTransform(cloudwatchEndpoint)(_.setCloudwatchEndpoint(_)) + .maybeTransform(cloudwatchPort)(_.setCloudwatchPort(_)) + .maybeTransform(collectionMaxCount)(_.setCollectionMaxCount(_)) + .maybeTransform(collectionMaxSize)(_.setCollectionMaxSize(_)) + .maybeTransform(connectTimeout)(_.setConnectTimeout(_)) + .maybeTransform(credentialsRefreshDelay)(_.setCredentialsRefreshDelay(_)) + .maybeTransform(enableCoreDumps)(_.setEnableCoreDumps(_)) + .maybeTransform(failIfThrottled)(_.setFailIfThrottled(_)) + .maybeTransform(kinesisEndpoint)(_.setKinesisEndpoint(_)) + .maybeTransform(kinesisPort)(_.setKinesisPort(_)) + .maybeTransform(logLevel)(_.setLogLevel(_)) + .maybeTransform(maxConnections)(_.setMaxConnections(_)) + .maybeTransform(metricsGranularity)(_.setMetricsGranularity(_)) + .maybeTransform(metricsLevel)(_.setMetricsLevel(_)) + .maybeTransform(metricsNamespace)(_.setMetricsNamespace(_)) + .maybeTransform(metricsUploadDelay)(_.setMetricsUploadDelay(_)) + .maybeTransform(minConnections)(_.setMinConnections(_)) + .maybeTransform(nativeExecutable)(_.setNativeExecutable(_)) + .maybeTransform(rateLimit)(_.setRateLimit(_)) + .maybeTransform(recordMaxBufferedTime)(_.setRecordMaxBufferedTime(_)) + .maybeTransform(recordTtl)(_.setRecordTtl(_)) + .maybeTransform(region)(_.setRegion(_)) + .maybeTransform(requestTimeout)(_.setRequestTimeout(_)) + .maybeTransform(tempDirectory)(_.setTempDirectory(_)) + .maybeTransform(verifyCertificate)(_.setVerifyCertificate(_)) + .maybeTransform(proxyHost)(_.setProxyHost(_)) + .maybeTransform(proxyPort)(_.setProxyPort(_)) + .maybeTransform(proxyUserName)(_.setProxyUserName(_)) + .maybeTransform(proxyPassword)(_.setProxyPassword(_)) + .maybeTransform(threadingModel)(_.setThreadingModel(_)) + .maybeTransform(threadPoolSize)(_.setThreadPoolSize(_)) + .maybeTransform(userRecordTimeout)(_.setUserRecordTimeoutInMillis(_)) + .maybeRunUnsafe(additionalMetricsDimensions) { + case (conf, additionalDims) => + additionalDims.foreach(x => + conf.addAdditionalMetricsDimension( + x.key, + x.value, + x.granularity.value + ) ) - ) - } + }, + KPLProducer.Config.GracefulShutdown( + gracefulShutdownFlushAttempts, + gracefulShutdownFlushInterval + ) + ) /** Reads environment variables and system properties to load * [[https://github.com/awslabs/amazon-kinesis-producer/blob/master/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java KinesisProducerConfiguration]] @@ -427,13 +445,13 @@ object KPLCiris { * [[cats.effect.Async Async]] containing * [[https://github.com/awslabs/amazon-kinesis-producer/blob/master/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java KinesisProducerConfiguration]] */ - def loadKplConfig[F[_]]( + private[kinesis4cats] def loadKplConfig[F[_]]( credentialsProvider: Option[AWSCredentialsProvider] = None, metricsCredentialsProvider: Option[AWSCredentialsProvider] = None, glueSchemaRegistryCredentialsProvider: Option[AwsCredentialsProvider] = None, prefix: Option[String] = None - )(implicit F: Async[F]): F[KinesisProducerConfiguration] = readKplConfig( + )(implicit F: Async[F]): F[KPLProducer.Config] = readKplConfig( credentialsProvider, metricsCredentialsProvider, glueSchemaRegistryCredentialsProvider, @@ -475,7 +493,7 @@ object KPLCiris { glueSchemaRegistryCredentialsProvider: Option[AwsCredentialsProvider] = None, prefix: Option[String] = None - )(implicit F: Async[F]): Resource[F, KinesisProducerConfiguration] = + )(implicit F: Async[F]): Resource[F, KPLProducer.Config] = readKplConfig( credentialsProvider, metricsCredentialsProvider, @@ -524,12 +542,16 @@ object KPLCiris { )(implicit F: Async[F] ): Resource[F, KPLProducer[F]] = for { - kplConfig <- kplConfigResource[F]( + config <- kplConfigResource[F]( credentialsProvider, metricsCredentialsProvider, glueSchemaRegistryCredentialsProvider, prefix ) - kpl <- KPLProducer[F](kplConfig, encoders = encoders) + kpl <- KPLProducer.Builder + .default[F] + .withConfig(config) + .withLogEncoders(encoders) + .build } yield kpl } diff --git a/kpl-ciris/src/main/scala/kinesis4cats/kpl/instances/ciris.scala b/kpl-ciris/src/main/scala/kinesis4cats/kpl/instances/ciris.scala index d54131fd..cafa901c 100644 --- a/kpl-ciris/src/main/scala/kinesis4cats/kpl/instances/ciris.scala +++ b/kpl-ciris/src/main/scala/kinesis4cats/kpl/instances/ciris.scala @@ -84,5 +84,4 @@ object ciris { ) ) } - } diff --git a/kpl-ciris/src/test/scala/kinesis4cats/kpl/ciris/KPLCirisSpec.scala b/kpl-ciris/src/test/scala/kinesis4cats/kpl/ciris/KPLCirisSpec.scala index 52d7399f..911371fe 100644 --- a/kpl-ciris/src/test/scala/kinesis4cats/kpl/ciris/KPLCirisSpec.scala +++ b/kpl-ciris/src/test/scala/kinesis4cats/kpl/ciris/KPLCirisSpec.scala @@ -26,6 +26,7 @@ import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants.CO import com.amazonaws.services.schemaregistry.utils._ import software.amazon.awssdk.services.glue.model.Compatibility +import kinesis4cats.kpl.KPLProducer import kinesis4cats.kpl.instances.eq._ import kinesis4cats.kpl.instances.show._ import kinesis4cats.syntax.id._ @@ -37,7 +38,7 @@ class KPLCirisSpec extends munit.CatsEffectSuite { "It should load the environment variables the same as system properties" ) { // format: off - val expected = new KinesisProducerConfiguration() + val expected = KPLProducer.Config(new KinesisProducerConfiguration() .setGlueSchemaRegistryConfiguration( new GlueSchemaRegistryConfiguration("us-east-1") .runUnsafe(COMPRESSION.valueOf(BuildInfo.kplGlueCompressionType))( @@ -112,7 +113,10 @@ class KPLCirisSpec extends munit.CatsEffectSuite { .safeTransform(BuildInfo.kplThreadPoolSize.toInt)(_.setThreadPoolSize(_)) .safeTransform(BuildInfo.kplUserRecordTimeout.asMillisUnsafe)( _.setUserRecordTimeoutInMillis(_) - ) + ), KPLProducer.Config.GracefulShutdown( + BuildInfo.kplGracefulShutdownFlushAttempts.toInt, + BuildInfo.kplGracefulShutdownFlushInterval.asFiniteDurationUnsafe + )) // format: on for { kplConfigEnv <- KPLCiris.loadKplConfig[IO](prefix = Some("env")) diff --git a/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/eq.scala b/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/eq.scala index 1d0e6d00..26b07fb9 100644 --- a/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/eq.scala +++ b/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/eq.scala @@ -21,6 +21,8 @@ import cats.syntax.all._ import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration +import kinesis4cats.kpl.KPLProducer + object eq { implicit val glueSchemaRegistryConfigurationEq : Eq[GlueSchemaRegistryConfiguration] = (x, y) => @@ -78,4 +80,8 @@ object eq { x.isVerifyCertificate() === y.isVerifyCertificate() && Option(x.getGlueSchemaRegistryConfiguration()) === Option(y.getGlueSchemaRegistryConfiguration()) + + implicit val kplProducerConfigEq: Eq[KPLProducer.Config] = (x, y) => + x.gracefulShutdown == y.gracefulShutdown && + x.kpl === y.kpl } diff --git a/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/show.scala b/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/show.scala index 8ec8d7ab..4b150916 100644 --- a/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/show.scala +++ b/kpl-ciris/src/test/scala/kinesis4cats/kpl/instances/show.scala @@ -21,6 +21,7 @@ import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration import kinesis4cats.ShowBuilder +import kinesis4cats.kpl.KPLProducer import kinesis4cats.logging.instances.show._ object show { @@ -87,4 +88,17 @@ object show { ) .build + implicit val gracefulShutdownShow: Show[KPLProducer.Config.GracefulShutdown] = + x => + ShowBuilder("GracefulShutdown") + .add("flushAttempts", x.flushAttempts) + .add("flushInterval", x.flushInterval) + .build + + implicit val kplProducerConfigShow: Show[KPLProducer.Config] = x => + ShowBuilder("Config") + .add("kpl", x.kpl) + .add("gracefulShutdown", x.gracefulShutdown) + .build + } diff --git a/kpl-localstack/src/main/scala/kinesis4cats/kpl/localstack/LocalstackKPLProducer.scala b/kpl-localstack/src/main/scala/kinesis4cats/kpl/localstack/LocalstackKPLProducer.scala index 0250bbad..65345a99 100644 --- a/kpl-localstack/src/main/scala/kinesis4cats/kpl/localstack/LocalstackKPLProducer.scala +++ b/kpl-localstack/src/main/scala/kinesis4cats/kpl/localstack/LocalstackKPLProducer.scala @@ -17,18 +17,63 @@ package kinesis4cats.kpl package localstack -import scala.concurrent.duration._ - import cats.effect.{Async, Resource} +import cats.syntax.all._ import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration +import org.typelevel.log4cats.StructuredLogger import kinesis4cats.localstack.LocalstackConfig +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.localstack.aws.v1.{AwsClients, AwsCreds} /** Helpers for constructing and leveraging the KPL with Localstack. */ object LocalstackKPLProducer { + final case class Builder[F[_]] private ( + kplBuilder: KPLProducer.Builder[F], + streamsToCreate: List[TestStreamConfig[F]], + localstackConfig: LocalstackConfig + )(implicit F: Async[F]) { + def withConfig(config: KPLProducer.Config): Builder[F] = + copy(kplBuilder = kplBuilder.withConfig(config)) + def withLogger(logger: StructuredLogger[F]) = + copy(kplBuilder = kplBuilder.withLogger(logger)) + def withLogEncoders(encoders: KPLProducer.LogEncoders): Builder[F] = + copy(kplBuilder = kplBuilder.withLogEncoders(encoders)) + def configure( + f: KinesisProducerConfiguration => KinesisProducerConfiguration + ) = copy(kplBuilder = kplBuilder.configure(f)) + def withStreamsToCreate( + streamsToCreate: List[TestStreamConfig[F]] + ): Builder[F] = copy(streamsToCreate = streamsToCreate) + + def build: Resource[F, KPLProducer[F]] = for { + _ <- AwsClients.kinesisStreamResource(localstackConfig, streamsToCreate) + producer <- kplBuilder.build + } yield producer + } + + object Builder { + def default[F[_]](localstackConfig: LocalstackConfig)(implicit + F: Async[F] + ): Builder[F] = + Builder( + KPLProducer.Builder + .default[F] + .configure(_ => kplConfig(localstackConfig)), + Nil, + localstackConfig + ) + + def default[F[_]](prefix: Option[String] = None)(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig.load[F](prefix).map(default(_)) + + @annotation.unused + def unapply[F[_]](builder: Builder[F]): Unit = () + } + /** [[https://github.com/awslabs/amazon-kinesis-producer/blob/master/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java KinesisProducerConfiguration]] * configuration compliant with Localstack * @@ -37,7 +82,9 @@ object LocalstackKPLProducer { * @return * [[https://github.com/awslabs/amazon-kinesis-producer/blob/master/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java KinesisProducerConfiguration]] */ - def kplConfig(config: LocalstackConfig): KinesisProducerConfiguration = + private def kplConfig( + config: LocalstackConfig + ): KinesisProducerConfiguration = new KinesisProducerConfiguration() .setVerifyCertificate(false) .setKinesisEndpoint(config.kinesisHost) @@ -50,138 +97,4 @@ object LocalstackKPLProducer { .setMetricsLevel("none") .setLogLevel("warning") .setRegion(config.region.name) - - /** Creates a [[kinesis4cats.kpl.KPLProducer KPLProducer]] that is compliant - * with Localstack - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kpl.KPLProducer.LogEncoders KPLProducer.LogEncoders]] - * @return - * [[kinesis4cats.kpl.KPLProducer KPLProducer]] as a - * [[cats.effect.Resource Resource]] - */ - def producer[F[_]]( - config: LocalstackConfig, - encoders: KPLProducer.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, KPLProducer[F]] = - KPLProducer[F](kplConfig(config), encoders = encoders) - - /** Creates a [[kinesis4cats.kpl.KPLProducer KPLProducer]] that is compliant - * with Localstack - * - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.kpl.KPLProducer.LogEncoders KPLProducer.LogEncoders]]. - * Default to show - * @return - * [[kinesis4cats.kpl.KPLProducer KPLProducer]] as a - * [[cats.effect.Resource Resource]] - */ - def producer[F[_]]( - prefix: Option[String] = None, - encoders: KPLProducer.LogEncoders = KPLProducer.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KPLProducer[F]] = - LocalstackConfig.resource[F](prefix).flatMap(producer(_, encoders)) - - /** A resources that does the following: - * - * - Builds a [[kinesis4cats.kpl.KPLProducer KPLProducer]] that is - * compliant for Localstack usage. - * - Creates a stream with the desired name and shard count, and waits - * until the stream is active. - * - Destroys the stream when the [[cats.effect.Resource Resource]] is - * closed - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamName - * Stream name - * @param shardCount - * Shard count for stream - * @param describeRetries - * How many times to retry DescribeStreamSummary when checking the stream - * status - * @param describeRetryDuration - * How long to delay between retries of the DescribeStreamSummary call - * @param F - * F with an [[cats.effect.Async Async]] instance - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.kpl.KPLProducer KPLProducer]] - */ - def producerWithStream[F[_]]( - config: LocalstackConfig, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration, - encoders: KPLProducer.LogEncoders - )(implicit - F: Async[F] - ): Resource[F, KPLProducer[F]] = AwsClients - .kinesisStreamResource[F]( - config, - streamName, - shardCount, - describeRetries, - describeRetryDuration - ) - .flatMap(_ => producer(config, encoders)) - - /** A resource that does the following: - * - * - Builds a [[kinesis4cats.kpl.KPLProducer KPLProducer]] that is - * compliant for Localstack usage. - * - Creates a stream with the desired name and shard count, and waits - * until the stream is active. - * - Destroys the stream when the [[cats.effect.Resource Resource]] is - * closed - * - * @param streamName - * Stream name - * @param shardCount - * Shard count for stream - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param describeRetries - * How many times to retry DescribeStreamSummary when checking the stream - * status. Default to 5 - * @param describeRetryDuration - * How long to delay between retries of the DescribeStreamSummary call. - * Default to 500 ms - * @param F - * F with an [[cats.effect.Async Async]] instance - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.kpl.KPLProducer KPLProducer]] - */ - def producerWithStream[F[_]]( - streamName: String, - shardCount: Int, - prefix: Option[String] = None, - describeRetries: Int = 5, - describeRetryDuration: FiniteDuration = 500.millis, - encoders: KPLProducer.LogEncoders = KPLProducer.LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KPLProducer[F]] = AwsClients - .kinesisStreamResource[F]( - streamName, - shardCount, - prefix, - describeRetries, - describeRetryDuration - ) - .flatMap(_ => producer(prefix, encoders)) } diff --git a/kpl-logging-circe/src/main/scala/kinesis4cats/kpl/logging/instances/circe.scala b/kpl-logging-circe/src/main/scala/kinesis4cats/kpl/logging/instances/circe.scala index e1f6777a..87a2574d 100644 --- a/kpl-logging-circe/src/main/scala/kinesis4cats/kpl/logging/instances/circe.scala +++ b/kpl-logging-circe/src/main/scala/kinesis4cats/kpl/logging/instances/circe.scala @@ -28,7 +28,7 @@ import kinesis4cats.logging.syntax.circe._ * encoding of log structures using [[https://circe.github.io/circe/ Circe]] */ object circe { - val kplProducer: KPLProducer.LogEncoders = { + val kplCirceEncoders: KPLProducer.LogEncoders = { implicit val attemptEncoder: Encoder[Attempt] = x => { val fields: Map[String, Json] = Map diff --git a/kpl/src/main/scala/kinesis4cats/kpl/KPLProducer.scala b/kpl/src/main/scala/kinesis4cats/kpl/KPLProducer.scala index f5cfebe5..625a5249 100644 --- a/kpl/src/main/scala/kinesis4cats/kpl/KPLProducer.scala +++ b/kpl/src/main/scala/kinesis4cats/kpl/KPLProducer.scala @@ -458,47 +458,68 @@ class KPLProducer[F[_]] private ( object KPLProducer { - /** Constructor for the [[kinesis4cats.kpl.KPLProducer KPLProducer]] - * - * @param config - * [[https://github.com/awslabs/amazon-kinesis-producer/blob/master/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java KinesisProducerConfiguration]] - * @param gracefulShutdownFlushAttempts - * How many times to execute flush() and wait for the KPL's buffer to clear - * before shutting down - * @param gracefulShutdownFlushInterval - * Duration between flush() attempts during the graceful shutdown - * @param encoders - * [[kinesis4cats.kpl.KPLProducer.LogEncoders KPLProducer.LogEncoders]]. - * Default to show instances - * @param F - * [[cats.effect.Async Async]] - * @return - * [[cats.effect.Resource Resource]] containing the - * [[kinesis4cats.kpl.KPLProducer KPLProducer]] - */ - def apply[F[_]]( - config: KinesisProducerConfiguration = new KinesisProducerConfiguration(), - gracefulShutdownFlushAttempts: Int = 5, - gracefulShutdownFlushInterval: FiniteDuration = 500.millis, - encoders: LogEncoders = LogEncoders.show - )(implicit - F: Async[F] - ): Resource[F, KPLProducer[F]] = - Resource.make[F, KPLProducer[F]]( + final case class Config( + kpl: KinesisProducerConfiguration, + gracefulShutdown: Config.GracefulShutdown + ) + + object Config { + val default = + Config( + new KinesisProducerConfiguration(), + Config.GracefulShutdown.default + ) + + final case class GracefulShutdown( + flushAttempts: Int, + flushInterval: FiniteDuration + ) + + object GracefulShutdown { + val default: GracefulShutdown = GracefulShutdown(5, 500.millis) + } + } + + final case class Builder[F[_]] private ( + config: Config, + logger: StructuredLogger[F], + encoders: LogEncoders + )(implicit F: Async[F]) { + def withConfig(config: Config): Builder[F] = copy(config = config) + def withLogger(logger: StructuredLogger[F]) = copy(logger = logger) + def withLogEncoders(encoders: LogEncoders): Builder[F] = + copy(encoders = encoders) + def configure( + f: KinesisProducerConfiguration => KinesisProducerConfiguration + ) = copy(config = config.copy(kpl = f(config.kpl))) + + def build: Resource[F, KPLProducer[F]] = Resource.make[F, KPLProducer[F]]( for { - client <- F.delay(new KinesisProducer(config)) - logger <- Slf4jLogger.create[F] + client <- F.delay(new KinesisProducer(config.kpl)) state <- Ref.of[F, State](State.Up) } yield new KPLProducer(client, logger, state, encoders) ) { x => for { _ <- x.state.set(State.ShuttingDown) _ <- x.gracefulShutdown( - gracefulShutdownFlushAttempts, - gracefulShutdownFlushInterval + config.gracefulShutdown.flushAttempts, + config.gracefulShutdown.flushInterval ) } yield () } + } + + object Builder { + + def default[F[_]](implicit F: Async[F]): Builder[F] = Builder[F]( + KPLProducer.Config.default, + Slf4jLogger.getLogger, + LogEncoders.show + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** Helper class containing required * [[kinesis4cats.logging.LogEncoder LogEncoders]] for the diff --git a/project/KPLCirisSpecVars.scala b/project/KPLCirisSpecVars.scala index 38fcb267..fbc39aee 100644 --- a/project/KPLCirisSpecVars.scala +++ b/project/KPLCirisSpecVars.scala @@ -119,6 +119,14 @@ object KPLCirisSpecVars { CirisUtil.propAndEnv( List("kpl", "native", "executable"), "foo.exe" + ), + CirisUtil.propAndEnv( + List("kpl", "graceful", "shutdown", "flush", "attempts"), + "6" + ), + CirisUtil.propAndEnv( + List("kpl", "graceful", "shutdown", "flush", "interval"), + "1 second" ) ) ) diff --git a/project/Kinesis4CatsPlugin.scala b/project/Kinesis4CatsPlugin.scala index 9ed1e87b..f9824c76 100644 --- a/project/Kinesis4CatsPlugin.scala +++ b/project/Kinesis4CatsPlugin.scala @@ -236,7 +236,7 @@ object Kinesis4CatsPlugin extends AutoPlugin { "-source:3.0-migration" ) else - Seq("-language:_", "-Wconf:src=src_managed/.*:silent") + Seq("-language:_", "-Wconf:src=src_managed/.*:silent", "-Xsource:3") }, scalacOptions -= "-Ykind-projector:underscores", ThisBuild / semanticdbEnabled := true, diff --git a/shared-localstack/src/main/scala/kinesis4cats/localstack/TestStreamConfig.scala b/shared-localstack/src/main/scala/kinesis4cats/localstack/TestStreamConfig.scala new file mode 100644 index 00000000..6716e724 --- /dev/null +++ b/shared-localstack/src/main/scala/kinesis4cats/localstack/TestStreamConfig.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2023-2023 etspaceman + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kinesis4cats.localstack + +import scala.concurrent.duration._ + +import cats.Applicative + +import kinesis4cats.compat.retry.RetryPolicies._ +import kinesis4cats.compat.retry.RetryPolicy + +final case class TestStreamConfig[F[_]]( + streamName: String, + shardCount: Int, + describeRetryPolicy: RetryPolicy[F] +) + +object TestStreamConfig { + def default[F[_]](streamName: String, shardCount: Int)(implicit + F: Applicative[F] + ): TestStreamConfig[F] = TestStreamConfig[F]( + streamName, + shardCount, + constantDelay(500.millis).join(limitRetries(5)) + ) +} diff --git a/shared/src/main/scala/kinesis4cats/producer/ShardMapCache.scala b/shared/src/main/scala/kinesis4cats/producer/ShardMapCache.scala index 0e164c42..059b4569 100644 --- a/shared/src/main/scala/kinesis4cats/producer/ShardMapCache.scala +++ b/shared/src/main/scala/kinesis4cats/producer/ShardMapCache.scala @@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets import java.time.Instant import cats.Show +import cats.effect.kernel.Resource import cats.effect.syntax.all._ import cats.effect.{Async, Ref} import cats.syntax.all._ @@ -112,34 +113,43 @@ private[kinesis4cats] class ShardMapCache[F[_]] private ( object ShardMapCache { - /** Construct a ShardMapCache - * - * @param config - * ShardMapCache - * @param shardMapF - * F that supplies a new [[kinesis4cats.producer.ShardMap ShardMap]] - * @param loggerF - * F of [[org.typelevel.log4cats.StructuredLogger StructuredLogger]] - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.producer.ShardMapCache.LogEncoders ShardMapCache.LogEncoders]]. - * Defaults to show instances - * @return - */ - def apply[F[_]]( + final case class Builder[F[_]] private ( config: Config, shardMapF: F[Either[Error, ShardMap]], - loggerF: F[StructuredLogger[F]], - encoders: ShardMapCache.LogEncoders = LogEncoders.show - )(implicit - F: Async[F] - ) = for { - logger <- loggerF.toResource - ref <- Ref.of[F, ShardMap](ShardMap.empty).toResource - service = new ShardMapCache[F](config, logger, ref, shardMapF, encoders) - _ <- service.start() - } yield service + logger: StructuredLogger[F], + encoders: LogEncoders + )(implicit F: Async[F]) { + def withConfig(config: Config): Builder[F] = copy(config = config) + def withShardMapF(shardMapF: F[Either[Error, ShardMap]]): Builder[F] = + copy(shardMapF = shardMapF) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withLogEncoders(encoders: LogEncoders): Builder[F] = + copy(encoders = encoders) + + def build: Resource[F, ShardMapCache[F]] = for { + ref <- Ref.of[F, ShardMap](ShardMap.empty).toResource + service = new ShardMapCache[F](config, logger, ref, shardMapF, encoders) + _ <- service.start() + } yield service + } + + object Builder { + def default[F[_]]( + shardMapF: F[Either[Error, ShardMap]], + logger: StructuredLogger[F] + )(implicit + F: Async[F] + ): Builder[F] = Builder[F]( + Config.default, + shardMapF, + logger, + LogEncoders.show + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } /** [[kinesis4cats.logging.LogEncoder LogEncoder]] instances for the * ShardMapCache diff --git a/shared/src/main/scala/kinesis4cats/producer/batching/ShardBatch.scala b/shared/src/main/scala/kinesis4cats/producer/batching/ShardBatch.scala index 624793a6..6e673bd8 100644 --- a/shared/src/main/scala/kinesis4cats/producer/batching/ShardBatch.scala +++ b/shared/src/main/scala/kinesis4cats/producer/batching/ShardBatch.scala @@ -36,7 +36,7 @@ import kinesis4cats.models.ShardId * @param config * [[kinesis4cats.producer.batching.Batcher.Config Batcher.Config]] */ -private[kinesis4cats] final case class ShardBatch private ( +private[kinesis4cats] final case class ShardBatch private[kinesis4cats] ( shardId: ShardId, records: NonEmptyList[Record], count: Int, diff --git a/shared/src/main/scala/kinesis4cats/producer/fs2/FS2Producer.scala b/shared/src/main/scala/kinesis4cats/producer/fs2/FS2Producer.scala index 56fc0fe5..8296e476 100644 --- a/shared/src/main/scala/kinesis4cats/producer/fs2/FS2Producer.scala +++ b/shared/src/main/scala/kinesis4cats/producer/fs2/FS2Producer.scala @@ -21,8 +21,6 @@ import scala.concurrent.duration._ import _root_.fs2.concurrent.Channel import cats.Applicative -import cats.data.Ior -import cats.data.NonEmptyList import cats.effect._ import cats.effect.syntax.all._ import cats.syntax.all._ @@ -56,8 +54,7 @@ abstract class FS2Producer[F[_], PutReq, PutRes](implicit /** A user defined function that can be run against the results of a request */ - protected def callback - : (Ior[Producer.Error, NonEmptyList[PutRes]], Async[F]) => F[Unit] + protected def callback: Producer.Res[PutRes] => F[Unit] protected def underlying: Producer[F, PutReq, PutRes] @@ -118,7 +115,7 @@ abstract class FS2Producer[F[_], PutReq, PutRes](implicit ) _ <- underlying .put(records) - .flatMap(callback(_, implicitly)) + .flatMap(callback) .void _ <- logger.debug(c.context)( "Finished processing batch" diff --git a/shared/src/test/scala/kinesis4cats/producer/ProducerSpec.scala b/shared/src/test/scala/kinesis4cats/producer/ProducerSpec.scala index d114f458..aa83e3d3 100644 --- a/shared/src/test/scala/kinesis4cats/producer/ProducerSpec.scala +++ b/shared/src/test/scala/kinesis4cats/producer/ProducerSpec.scala @@ -179,23 +179,24 @@ class MockProducer( object MockProducer { def apply(): Resource[IO, MockProducer] = for { logger <- Resource.pure(NoOpLogger[IO]) - shardMapCache <- ShardMapCache[IO]( - ShardMapCache.Config.default, - IO.pure( - Right( - ShardMap( - List( - ShardMapRecord( - ShardId("1"), - HashKeyRange(BigInt("1"), BigInt("100")) - ) - ), - Instant.now() + shardMapCache <- ShardMapCache.Builder + .default[IO]( + IO.pure( + Right( + ShardMap( + List( + ShardMapRecord( + ShardId("1"), + HashKeyRange(BigInt("1"), BigInt("100")) + ) + ), + Instant.now() + ) ) - ) - ), - IO.pure(logger) - ) + ), + logger + ) + .build defaultConfig = Producer.Config .default[IO](StreamNameOrArn.Name("foo")) .copy(retryPolicy = RetryPolicies.limitRetries[IO](5)) diff --git a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/localstack/LocalstackKinesisClient.scala b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/localstack/LocalstackKinesisClient.scala index 1641f93a..c9af9105 100644 --- a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/localstack/LocalstackKinesisClient.scala +++ b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/localstack/LocalstackKinesisClient.scala @@ -17,11 +17,8 @@ package kinesis4cats.smithy4s.client package localstack -import scala.concurrent.duration._ - import cats.effect.Async import cats.effect.Resource -import cats.effect.syntax.all._ import cats.syntax.all._ import com.amazonaws.kinesis._ import org.http4s.client.Client @@ -30,9 +27,9 @@ import org.typelevel.log4cats.noop.NoOpLogger import smithy4s.aws._ import smithy4s.aws.kernel.AwsRegion -import kinesis4cats.compat.retry.RetryPolicies._ import kinesis4cats.compat.retry._ import kinesis4cats.localstack.LocalstackConfig +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.smithy4s.client.KinesisClient import kinesis4cats.smithy4s.client.middleware._ @@ -42,278 +39,147 @@ import kinesis4cats.smithy4s.client.middleware._ */ object LocalstackKinesisClient { - def localstackHttp4sClient[F[_]]( - client: Client[F], - config: LocalstackConfig, - loggerF: Async[F] => F[StructuredLogger[F]], - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders - )(implicit F: Async[F]): F[Client[F]] = - loggerF(F).map(logger => - LocalstackProxy[F]( - config, - logger, - kinesisClientEncoders, - localstackConfigEncoders - )(client) + final class LogEncoders[F[_]]( + val kinesisClientEncoders: KinesisClient.LogEncoders[F], + val localstackConfigEncoders: LocalstackConfig.LogEncoders + ) + + object LogEncoders { + def show[F[_]]: LogEncoders[F] = new LogEncoders( + KinesisClient.LogEncoders.show[F], + LocalstackConfig.LogEncoders.show ) + } - /** Creates a [[cats.effect.Resource Resource]] of a KinesisClient that is - * compatible with Localstack - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]] - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * @param kinesisClientEncoders - * [[kinesis4cats.smithy4s.client.KinesisClient.LogEncoders KinesisClient.LogEncoders]] - * @param localstackConfigEncoders - * [[kinesis4cats.localstack.LocalstackConfig.LogEncoders LocalstackConfig.LogEncoders]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsEnvironment.scala AwsEnvironment]] - */ - def clientResource[F[_]]( + final case class Builder[F[_]] private ( client: Client[F], - region: F[AwsRegion], - config: LocalstackConfig, - loggerF: Async[F] => F[StructuredLogger[F]], - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders - )(implicit F: Async[F]): Resource[F, KinesisClient[F]] = for { - http4sClient <- localstackHttp4sClient( - client, - config, - loggerF, - kinesisClientEncoders, - localstackConfigEncoders - ).toResource - awsClient <- KinesisClient[F]( - http4sClient, - region, - loggerF, - (_: SimpleHttpClient[F], f: Async[F]) => - Resource.pure( - f.pure(AwsCredentials.Default("mock-key-id", "mock-secret-key", None)) + region: AwsRegion, + localstackConfig: LocalstackConfig, + logger: StructuredLogger[F], + encoders: LogEncoders[F], + logRequestsResponses: Boolean, + streamsToCreate: List[TestStreamConfig[F]] + )(implicit F: Async[F]) { + + def withLocalstackConfig(localstackConfig: LocalstackConfig): Builder[F] = + copy(localstackConfig = localstackConfig) + def withClient(client: Client[F]): Builder[F] = copy(client = client) + def withRegion(region: AwsRegion): Builder[F] = copy(region = region) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withLogEncoders(encoders: LogEncoders[F]): Builder[F] = + copy(encoders = encoders) + def withLogRequestsResponses(logRequestsResponses: Boolean): Builder[F] = + copy(logRequestsResponses = logRequestsResponses) + def enableLogging: Builder[F] = withLogRequestsResponses(true) + def disableLogging: Builder[F] = withLogRequestsResponses(false) + def withStreamsToCreate( + streamsToCreate: List[TestStreamConfig[F]] + ): Builder[F] = + copy(streamsToCreate = streamsToCreate) + + def build: Resource[F, KinesisClient[F]] = for { + client <- KinesisClient.Builder + .default[F]( + LocalstackProxy[F](localstackConfig, logger, encoders)(client), + region ) - ) - } yield awsClient + .withLogEncoders(encoders.kinesisClientEncoders) + .withLogger(logger) + .withLogRequestsResponses(logRequestsResponses) + .withCredentials(_ => + Resource.pure( + F.pure( + AwsCredentials.Default("mock-key-id", "mock-secret-key", None) + ) + ) + ) + .build + _ <- streamsToCreate.traverse_(config => managedStream(config, client)) + } yield client - /** Creates a [[cats.effect.Resource Resource]] of a KinesisClient that is - * compatible with Localstack - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]]. - * @param prefix - * Optional string prefix to apply when loading configuration. Default to - * None - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * Default is - * [[https://github.com/typelevel/log4cats/blob/main/noop/shared/src/main/scala/org/typelevel/log4cats/noop/NoOpLogger.scala NoOpLogger]] - * @param kinesisClientEncoders - * [[kinesis4cats.smithy4s.client.KinesisClient.LogEncoders KinesisClient.LogEncoders]] - * @param localstackConfigEncoders - * [[kinesis4cats.localstack.LocalstackConfig.LogEncoders LocalstackConfig.LogEncoders]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsEnvironment.scala AwsEnvironment]] - */ - def clientResource[F[_]]( - client: Client[F], - region: F[AwsRegion], - prefix: Option[String] = None, - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger[F](f)), - kinesisClientEncoders: KinesisClient.LogEncoders[F] = - KinesisClient.LogEncoders.show[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders = - LocalstackConfig.LogEncoders.show - )(implicit F: Async[F]): Resource[F, KinesisClient[F]] = LocalstackConfig - .resource(prefix) - .flatMap( - clientResource( + } + + object Builder { + def default[F[_]]( + client: Client[F], + region: AwsRegion, + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig + .load(prefix) + .map(default(client, region, _)) + + def default[F[_]]( + client: Client[F], + region: AwsRegion, + config: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( client, region, - _, - loggerF, - kinesisClientEncoders, - localstackConfigEncoders + config, + NoOpLogger[F], + LogEncoders.show, + true, + Nil ) - ) - - /** A resources that does the following: - * - * - Builds a [[kinesis4cats.smithy4s.client.KinesisClient KinesisClient]] - * that is compliant for Localstack usage. - * - Creates a stream with the desired name and shard count, and waits - * until the stream is active. - * - Destroys the stream when the [[cats.effect.Resource Resource]] is - * closed - * - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param streamName - * Stream name - * @param shardCount - * Shard count for stream - * @param describeRetries - * How many times to retry DescribeStreamSummary when checking the stream - * status - * @param describeRetryDuration - * How long to delay between retries of the DescribeStreamSummary call - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param kinesisClientEncoders - * [[kinesis4cats.smithy4s.client.KinesisClient.LogEncoders KinesisClient.LogEncoders]] - * @param localstackConfigEncoders - * [[kinesis4cats.localstack.LocalstackConfig.LogEncoders LocalstackConfig.LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.smithy4s.client.KinesisClient KinesisClient]] - */ - def streamResource[F[_]]( - http4sClient: Client[F], - region: F[AwsRegion], - config: LocalstackConfig, - streamName: String, - shardCount: Int, - describeRetries: Int, - describeRetryDuration: FiniteDuration, - loggerF: Async[F] => F[StructuredLogger[F]], - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders - )(implicit F: Async[F]): Resource[F, KinesisClient[F]] = { - val retryPolicy = constantDelay(describeRetryDuration).join( - limitRetries(describeRetries) - ) - clientResource[F]( - http4sClient, - region, - config, - loggerF, - kinesisClientEncoders, - localstackConfigEncoders - ).flatMap { case client: KinesisClient[F] => - Resource.make[F, KinesisClient[F]]( - for { - _ <- client.createStream( - StreamName(streamName), - Some(PositiveIntegerObject(shardCount)) - ) - _ <- retryingOnFailuresAndAllErrors( - retryPolicy, - (x: DescribeStreamSummaryOutput) => - F.pure( - x.streamDescriptionSummary.streamStatus == StreamStatus.ACTIVE - ), - noop[F, DescribeStreamSummaryOutput], - noop[F, Throwable] - )( - client.describeStreamSummary(Some(StreamName(streamName))) - ) - } yield client - )(client => - for { - _ <- client.deleteStream( - Some(StreamName(streamName)) - ) - _ <- retryingOnFailuresAndSomeErrors( - retryPolicy, - (x: Either[Throwable, DescribeStreamSummaryOutput]) => - F.pure( - x.swap.exists { - case _: ResourceNotFoundException => true - case _ => false - } - ), - (e: Throwable) => - e match { - case _: ResourceNotFoundException => F.pure(false) - case _ => F.pure(true) - }, - noop[F, Either[Throwable, DescribeStreamSummaryOutput]], - noop[F, Throwable] - )( - client - .describeStreamSummary(Some(StreamName(streamName))) - .attempt - ) - } yield () - ) - } + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () } - /** A resources that does the following: - * - * - Builds a [[kinesis4cats.smithy4s.client.KinesisClient KinesisClient]] - * that is compliant for Localstack usage. - * - Creates a stream with the desired name and shard count, and waits - * until the stream is active. - * - Destroys the stream when the [[cats.effect.Resource Resource]] is - * closed - * - * @param streamName - * Stream name - * @param shardCount - * Shard count for stream - * @param prefix - * Optional prefix for parsing configuration. Default to None - * @param describeRetries - * How many times to retry DescribeStreamSummary when checking the stream - * status. Default to 5 - * @param describeRetryDuration - * How long to delay between retries of the DescribeStreamSummary call. - * Default to 500 ms - * @param F - * F with an [[cats.effect.Async Async]] instance - * @param kinesisClientEncoders - * [[kinesis4cats.smithy4s.client.KinesisClient.LogEncoders KinesisClient.LogEncoders]] - * @param localstackConfigEncoders - * [[kinesis4cats.localstack.LocalstackConfig.LogEncoders LocalstackConfig.LogEncoders]] - * Defaults to show instances - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.smithy4s.client.KinesisClient KinesisClient]] - */ - def streamResource[F[_]]( - http4sClient: Client[F], - region: F[AwsRegion], - streamName: String, - shardCount: Int, - prefix: Option[String] = None, - describeRetries: Int = 5, - describeRetryDuration: FiniteDuration = 500.millis, - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger(f)), - kinesisClientEncoders: KinesisClient.LogEncoders[F] = - KinesisClient.LogEncoders.show[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders = - LocalstackConfig.LogEncoders.show - )(implicit F: Async[F]): Resource[F, KinesisClient[F]] = for { - config <- LocalstackConfig.resource(prefix) - result <- streamResource( - http4sClient, - region, - config, - streamName, - shardCount, - describeRetries, - describeRetryDuration, - loggerF, - kinesisClientEncoders, - localstackConfigEncoders + private[kinesis4cats] def managedStream[F[_]]( + config: TestStreamConfig[F], + client: KinesisClient[F] + )(implicit F: Async[F]): Resource[F, KinesisClient[F]] = + Resource.make[F, KinesisClient[F]]( + for { + _ <- client.createStream( + StreamName(config.streamName), + Some(PositiveIntegerObject(config.shardCount)) + ) + _ <- retryingOnFailuresAndAllErrors( + config.describeRetryPolicy, + (x: DescribeStreamSummaryOutput) => + F.pure( + x.streamDescriptionSummary.streamStatus == StreamStatus.ACTIVE + ), + noop[F, DescribeStreamSummaryOutput], + noop[F, Throwable] + )( + client.describeStreamSummary(Some(StreamName(config.streamName))) + ) + } yield client + )(client => + for { + _ <- client.deleteStream( + Some(StreamName(config.streamName)) + ) + _ <- retryingOnFailuresAndSomeErrors( + config.describeRetryPolicy, + (x: Either[Throwable, DescribeStreamSummaryOutput]) => + F.pure( + x.swap.exists { + case _: ResourceNotFoundException => true + case _ => false + } + ), + (e: Throwable) => + e match { + case _: ResourceNotFoundException => F.pure(false) + case _ => F.pure(true) + }, + noop[F, Either[Throwable, DescribeStreamSummaryOutput]], + noop[F, Throwable] + )( + client + .describeStreamSummary(Some(StreamName(config.streamName))) + .attempt + ) + } yield () ) - } yield result } diff --git a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/middleware/LocalstackProxy.scala b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/middleware/LocalstackProxy.scala index c1f3e2c5..73e1310b 100644 --- a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/middleware/LocalstackProxy.scala +++ b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/middleware/LocalstackProxy.scala @@ -27,6 +27,7 @@ import org.typelevel.log4cats.StructuredLogger import kinesis4cats.localstack.LocalstackConfig import kinesis4cats.logging.LogContext +import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient /** Middleware for [[https://http4s.org/v0.23/docs/client.html Clients]] that * proxies request against the configured Localstack environment @@ -55,11 +56,10 @@ object LocalstackProxy { config: LocalstackConfig, req: Request[F], logger: StructuredLogger[F], - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders + encoders: LocalstackKinesisClient.LogEncoders[F] )(implicit F: Async[F]): F[Request[F]] = { - import kinesisClientEncoders._ - import localstackConfigEncoders._ + import encoders.kinesisClientEncoders._ + import encoders.localstackConfigEncoders._ val newReq = req .withUri( req.uri.copy(authority = @@ -101,8 +101,7 @@ object LocalstackProxy { def apply[F[_]]( config: LocalstackConfig, logger: StructuredLogger[F], - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders + encoders: LocalstackKinesisClient.LogEncoders[F] )( client: Client[F] )(implicit @@ -113,8 +112,7 @@ object LocalstackProxy { config, req, logger, - kinesisClientEncoders, - localstackConfigEncoders + encoders ).toResource res <- client.run(proxied) } yield res @@ -142,10 +140,8 @@ object LocalstackProxy { def apply[F[_]]( logger: StructuredLogger[F], prefix: Option[String] = None, - kinesisClientEncoders: KinesisClient.LogEncoders[F] = - KinesisClient.LogEncoders.show[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders = - LocalstackConfig.LogEncoders.show + encoders: LocalstackKinesisClient.LogEncoders[F] = + LocalstackKinesisClient.LogEncoders.show[F] )( client: Client[F] )(implicit @@ -157,8 +153,7 @@ object LocalstackProxy { config, req, logger, - kinesisClientEncoders, - localstackConfigEncoders + encoders ).toResource res <- client.run(proxied) } yield res diff --git a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala index 2a882fda..3922ff88 100644 --- a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala +++ b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/localstack/LocalstackFS2KinesisProducer.scala @@ -20,9 +20,9 @@ package fs2 package localstack import _root_.fs2.concurrent.Channel -import cats.Applicative import cats.effect._ import cats.effect.syntax.all._ +import cats.syntax.all._ import com.amazonaws.kinesis.PutRecordsOutput import org.http4s.client.Client import org.typelevel.log4cats.StructuredLogger @@ -30,158 +30,134 @@ import org.typelevel.log4cats.noop.NoOpLogger import smithy4s.aws.kernel.AwsRegion import kinesis4cats.localstack.LocalstackConfig +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.Record +import kinesis4cats.producer.ShardMap import kinesis4cats.producer.ShardMapCache import kinesis4cats.producer.fs2.FS2Producer -import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient +import kinesis4cats.smithy4s.client.producer.localstack.LocalstackKinesisProducer /** Like KinesisProducer, but also includes the * [[kinesis4cats.smithy4s.client.middleware.LocalstackProxy LocalstackProxy]] * middleware, and leverages mock AWS credentials */ object LocalstackFS2KinesisProducer { - - /** Creates a [[cats.effect.Resource Resource]] of a - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - * that is compatible with Localstack - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]] - * @param producerConfig - * [[kinesis4cats.producer.Producer.Config Producer.Config]] - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * @param callback - * Function that can be run after each of the put results from the - * underlying - * @param F - * [[cats.effect.Async Async]] - * @return - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsEnvironment.scala AwsEnvironment]] - */ - def resource[F[_]]( + final case class Builder[F[_]] private ( client: Client[F], - region: F[AwsRegion], - producerConfig: FS2Producer.Config[F], - config: LocalstackConfig, - loggerF: Async[F] => F[StructuredLogger[F]], - callback: (Producer.Res[PutRecordsOutput], Async[F]) => F[Unit], - encoders: Producer.LogEncoders, - shardMapEncoders: ShardMapCache.LogEncoders, - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders - )(implicit F: Async[F]): Resource[F, FS2KinesisProducer[F]] = for { - logger <- loggerF(F).toResource - _underlying <- LocalstackKinesisClient - .clientResource[F]( - client, - region, - config, - loggerF, - kinesisClientEncoders, - localstackConfigEncoders - ) - shardMapCache <- ShardMapCache[F]( - producerConfig.producerConfig.shardMapCacheConfig, - KinesisProducer.getShardMap( - _underlying, - producerConfig.producerConfig.streamNameOrArn - ), - loggerF(F), - shardMapEncoders + region: AwsRegion, + localstackConfig: LocalstackConfig, + config: FS2Producer.Config[F], + logger: StructuredLogger[F], + encoders: LocalstackKinesisProducer.LogEncoders[F], + logRequestsResponses: Boolean, + streamsToCreate: List[TestStreamConfig[F]], + shardMapF: ( + KinesisClient[F], + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]], + callback: Producer.Res[PutRecordsOutput] => F[Unit] + )(implicit F: Async[F]) { + + def withLocalstackConfig(localstackConfig: LocalstackConfig): Builder[F] = + copy(localstackConfig = localstackConfig) + def withConfig(config: FS2Producer.Config[F]): Builder[F] = copy( + config = config ) - channel <- Channel - .bounded[F, Record](producerConfig.queueSize) - .toResource - underlying = new KinesisProducer[F]( - logger, - shardMapCache, - producerConfig.producerConfig, - _underlying, - encoders + def withClient(client: Client[F]): Builder[F] = copy(client = client) + def withRegion(region: AwsRegion): Builder[F] = copy(region = region) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withLogEncoders( + encoders: LocalstackKinesisProducer.LogEncoders[F] + ): Builder[F] = + copy(encoders = encoders) + def withLogRequestsResponses(logRequestsResponses: Boolean): Builder[F] = + copy(logRequestsResponses = logRequestsResponses) + def enableLogging: Builder[F] = withLogRequestsResponses(true) + def disableLogging: Builder[F] = withLogRequestsResponses(false) + def withStreamsToCreate(streamsToCreate: List[TestStreamConfig[F]]) = + copy(streamsToCreate = streamsToCreate) + def withShardMapF( + shardMapF: ( + KinesisClient[F], + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]] + ): Builder[F] = copy( + shardMapF = shardMapF ) - producer = new FS2KinesisProducer[F]( - logger, - producerConfig, - channel, - underlying - )( - callback + def withCallback( + callback: Producer.Res[PutRecordsOutput] => F[Unit] + ): Builder[F] = copy( + callback = callback ) - _ <- producer.resource - } yield producer - /** Creates a [[cats.effect.Resource Resource]] of a - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - * that is compatible with Localstack - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] - * @param streamName - * Name of stream that this producer will produce to - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]]. - * @param prefix - * Optional string prefix to apply when loading configuration. Default to - * None - * @param producerConfig - * String => [[kinesis4cats.producer.Producer.Config Producer.Config]] - * function that creates configuration given a stream name. Defaults to - * Producer.Config.default - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * Default is - * [[https://github.com/typelevel/log4cats/blob/main/noop/shared/src/main/scala/org/typelevel/log4cats/noop/NoOpLogger.scala NoOpLogger]] - * @param callback - * Function that can be run after each of the put results from the - * underlying. Default is F.unit - * @param F - * [[cats.effect.Async Async]] - * @return - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsEnvironment.scala AwsEnvironment]] - */ - def resource[F[_]]( - client: Client[F], - streamName: String, - region: F[AwsRegion], - prefix: Option[String] = None, - producerConfig: (String, Applicative[F]) => FS2Producer.Config[F] = - (streamName: String, f: Applicative[F]) => - FS2Producer.Config - .default[F](StreamNameOrArn.Name(streamName))(f), - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger[F](f)), - callback: (Producer.Res[PutRecordsOutput], Async[F]) => F[Unit] = - (_: Producer.Res[PutRecordsOutput], f: Async[F]) => f.unit, - encoders: Producer.LogEncoders = Producer.LogEncoders.show, - shardMapEncoders: ShardMapCache.LogEncoders = - ShardMapCache.LogEncoders.show, - kinesisClientEncoders: KinesisClient.LogEncoders[F] = - KinesisClient.LogEncoders.show[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders = - LocalstackConfig.LogEncoders.show - )(implicit F: Async[F]): Resource[F, FS2KinesisProducer[F]] = LocalstackConfig - .resource[F](prefix) - .flatMap( - resource[F]( + def build: Resource[F, FS2KinesisProducer[F]] = for { + underlying <- LocalstackKinesisProducer.Builder + .default[F]( + client, + region, + config.producerConfig.streamNameOrArn, + localstackConfig + ) + .withConfig(config.producerConfig) + .withLogEncoders(encoders) + .withLogger(logger) + .withLogRequestsResponses(logRequestsResponses) + .withStreamsToCreate(streamsToCreate) + .withShardMapF(shardMapF) + .build + channel <- Channel + .bounded[F, Record](config.queueSize) + .toResource + producer = new FS2KinesisProducer[F]( + logger, + config, + channel, + underlying + )( + callback + ) + _ <- producer.resource + } yield producer + } + + object Builder { + def default[F[_]]( + client: Client[F], + region: AwsRegion, + streamNameOrArn: StreamNameOrArn, + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig + .load(prefix) + .map(default(client, region, streamNameOrArn, _)) + + def default[F[_]]( + client: Client[F], + region: AwsRegion, + streamNameOrArn: StreamNameOrArn, + config: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( client, region, - producerConfig(streamName, F), - _, - loggerF, - callback, - encoders, - shardMapEncoders, - kinesisClientEncoders, - localstackConfigEncoders + config, + FS2Producer.Config.default[F](streamNameOrArn), + NoOpLogger[F], + LocalstackKinesisProducer.LogEncoders.show, + true, + Nil, + (client: KinesisClient[F], snoa: StreamNameOrArn) => + KinesisProducer.getShardMap(client, snoa), + (_: Producer.Res[PutRecordsOutput]) => F.unit ) - ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/localstack/LocalstackKinesisProducer.scala b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/localstack/LocalstackKinesisProducer.scala index a9b2d474..62cb87e5 100644 --- a/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/localstack/LocalstackKinesisProducer.scala +++ b/smithy4s-client-localstack/src/main/scala/kinesis4cats/smithy4s/client/producer/localstack/LocalstackKinesisProducer.scala @@ -18,15 +18,15 @@ package kinesis4cats.smithy4s.client package producer package localstack -import cats.Applicative import cats.effect._ -import cats.effect.syntax.all._ +import cats.syntax.all._ import org.http4s.client.Client import org.typelevel.log4cats.StructuredLogger import org.typelevel.log4cats.noop.NoOpLogger import smithy4s.aws.kernel.AwsRegion import kinesis4cats.localstack.LocalstackConfig +import kinesis4cats.localstack.TestStreamConfig import kinesis4cats.models.StreamNameOrArn import kinesis4cats.producer.Producer import kinesis4cats.producer.ShardMap @@ -39,135 +39,125 @@ import kinesis4cats.smithy4s.client.localstack.LocalstackKinesisClient */ object LocalstackKinesisProducer { - /** Creates a [[cats.effect.Resource Resource]] of a - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - * that is compatible with Localstack - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]] - * @param producerConfig - * [[kinesis4cats.producer.Producer.Config Producer.Config]] - * @param config - * [[kinesis4cats.localstack.LocalstackConfig LocalstackConfig]] - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * @param F - * [[cats.effect.Async Async]] - * @return - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsEnvironment.scala AwsEnvironment]] - */ - def resource[F[_]]( + final class LogEncoders[F[_]]( + val kinesisProducerEncoders: KinesisProducer.LogEncoders[F], + val localstackConfigEncoders: LocalstackConfig.LogEncoders + ) + + object LogEncoders { + def show[F[_]]: LogEncoders[F] = new LogEncoders( + KinesisProducer.LogEncoders.show[F], + LocalstackConfig.LogEncoders.show + ) + } + + final case class Builder[F[_]] private ( client: Client[F], - region: F[AwsRegion], - producerConfig: Producer.Config[F], - config: LocalstackConfig, - loggerF: Async[F] => F[StructuredLogger[F]], + region: AwsRegion, + localstackConfig: LocalstackConfig, + config: Producer.Config[F], + logger: StructuredLogger[F], + encoders: LogEncoders[F], + logRequestsResponses: Boolean, + streamsToCreate: List[TestStreamConfig[F]], shardMapF: ( KinesisClient[F], - StreamNameOrArn, - Async[F] - ) => F[Either[ShardMapCache.Error, ShardMap]], - encoders: Producer.LogEncoders, - shardMapEncoders: ShardMapCache.LogEncoders, - kinesisClientEncoders: KinesisClient.LogEncoders[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders - )(implicit F: Async[F]): Resource[F, KinesisProducer[F]] = for { - logger <- loggerF(F).toResource - underlying <- LocalstackKinesisClient - .clientResource[F]( - client, - region, - config, - loggerF, - kinesisClientEncoders, - localstackConfigEncoders - ) - shardMapCache <- ShardMapCache[F]( - producerConfig.shardMapCacheConfig, - shardMapF(underlying, producerConfig.streamNameOrArn, F), - loggerF(F), - shardMapEncoders + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]] + )(implicit F: Async[F]) { + + def withLocalstackConfig(localstackConfig: LocalstackConfig): Builder[F] = + copy(localstackConfig = localstackConfig) + def withConfig(config: Producer.Config[F]): Builder[F] = copy( + config = config + ) + def withClient(client: Client[F]): Builder[F] = copy(client = client) + def withRegion(region: AwsRegion): Builder[F] = copy(region = region) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withLogEncoders(encoders: LogEncoders[F]): Builder[F] = + copy(encoders = encoders) + def withLogRequestsResponses(logRequestsResponses: Boolean): Builder[F] = + copy(logRequestsResponses = logRequestsResponses) + def enableLogging: Builder[F] = withLogRequestsResponses(true) + def disableLogging: Builder[F] = withLogRequestsResponses(false) + def withStreamsToCreate(streamsToCreate: List[TestStreamConfig[F]]) = + copy(streamsToCreate = streamsToCreate) + def withShardMapF( + shardMapF: ( + KinesisClient[F], + StreamNameOrArn + ) => F[Either[ShardMapCache.Error, ShardMap]] + ): Builder[F] = copy( + shardMapF = shardMapF ) - producer = new KinesisProducer[F]( + + def build: Resource[F, KinesisProducer[F]] = for { + underlying <- LocalstackKinesisClient.Builder + .default[F]( + client, + region, + localstackConfig + ) + .withLogEncoders( + new LocalstackKinesisClient.LogEncoders( + encoders.kinesisProducerEncoders.kinesisClientLogEncoders, + encoders.localstackConfigEncoders + ) + ) + .withLogger(logger) + .withLogRequestsResponses(logRequestsResponses) + .withStreamsToCreate(streamsToCreate) + .build + shardMapCache <- ShardMapCache.Builder + .default[F](shardMapF(underlying, config.streamNameOrArn), logger) + .withLogEncoders( + encoders.kinesisProducerEncoders.producerLogEncoders.shardMapLogEncoders + ) + .build + } yield new KinesisProducer[F]( logger, shardMapCache, - producerConfig, + config, underlying, - encoders + encoders.kinesisProducerEncoders.producerLogEncoders ) - } yield producer + } - /** Creates a [[cats.effect.Resource Resource]] of a - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - * that is compatible with Localstack - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] - * @param streamName - * Name of stream that this producer will produce to - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]]. - * @param prefix - * Optional string prefix to apply when loading configuration. Default to - * None - * @param producerConfig - * String => [[kinesis4cats.producer.Producer.Config Producer.Config]] - * function that creates configuration given a stream name. Defaults to - * Producer.Config.default - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * Default is - * [[https://github.com/typelevel/log4cats/blob/main/noop/shared/src/main/scala/org/typelevel/log4cats/noop/NoOpLogger.scala NoOpLogger]] - * @param F - * [[cats.effect.Async Async]] - * @return - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsEnvironment.scala AwsEnvironment]] - */ - def resource[F[_]]( - client: Client[F], - streamName: String, - region: F[AwsRegion], - prefix: Option[String] = None, - producerConfig: (String, Applicative[F]) => Producer.Config[F] = - (streamName: String, f: Applicative[F]) => - Producer.Config - .default[F](StreamNameOrArn.Name(streamName))(f), - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger[F](f)), - shardMapF: ( - KinesisClient[F], - StreamNameOrArn, - Async[F] - ) => F[Either[ShardMapCache.Error, ShardMap]] = ( - client: KinesisClient[F], - streamNameOrArn: StreamNameOrArn, - f: Async[F] - ) => KinesisProducer.getShardMap(client, streamNameOrArn)(f), - encoders: Producer.LogEncoders = Producer.LogEncoders.show, - shardMapEncoders: ShardMapCache.LogEncoders = - ShardMapCache.LogEncoders.show, - kinesisClientEncoders: KinesisClient.LogEncoders[F] = - KinesisClient.LogEncoders.show[F], - localstackConfigEncoders: LocalstackConfig.LogEncoders = - LocalstackConfig.LogEncoders.show - )(implicit F: Async[F]): Resource[F, KinesisProducer[F]] = LocalstackConfig - .resource[F](prefix) - .flatMap( - resource[F]( + object Builder { + def default[F[_]]( + client: Client[F], + region: AwsRegion, + streamNameOrArn: StreamNameOrArn, + prefix: Option[String] = None + )(implicit + F: Async[F] + ): F[Builder[F]] = LocalstackConfig + .load(prefix) + .map(default(client, region, streamNameOrArn, _)) + + def default[F[_]]( + client: Client[F], + region: AwsRegion, + streamNameOrArn: StreamNameOrArn, + config: LocalstackConfig + )(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( client, region, - producerConfig(streamName, F), - _, - loggerF, - shardMapF, - encoders, - shardMapEncoders, - kinesisClientEncoders, - localstackConfigEncoders + config, + Producer.Config.default[F](streamNameOrArn), + NoOpLogger[F], + LogEncoders.show, + true, + Nil, + (client: KinesisClient[F], snoa: StreamNameOrArn) => + KinesisProducer.getShardMap(client, snoa) ) - ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } } diff --git a/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/KinesisClient.scala b/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/KinesisClient.scala index 906c0a1d..564b614b 100644 --- a/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/KinesisClient.scala +++ b/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/KinesisClient.scala @@ -20,7 +20,6 @@ package smithy4s.client import _root_.smithy4s.aws._ import _root_.smithy4s.aws.http4s._ import cats.Show -import cats.effect.syntax.all._ import cats.effect.{Async, Resource} import cats.syntax.all._ import com.amazonaws.kinesis._ @@ -48,28 +47,69 @@ import kinesis4cats.smithy4s.client.middleware.RequestResponseLogger */ object KinesisClient { - def awsEnv[F[_]]( + final case class Builder[F[_]] private ( client: Client[F], - region: F[AwsRegion], - credsF: ( - SimpleHttpClient[F], - Async[F] - ) => Resource[F, F[AwsCredentials]] = - (x: SimpleHttpClient[F], f: Async[F]) => - AwsCredentialsProvider.default[F](x)(f), - backendF: (Client[F], Async[F]) => SimpleHttpClient[F] = - (client: Client[F], f: Async[F]) => AwsHttp4sBackend[F](client)(f) - )(implicit F: Async[F]) = { - val backend = backendF(client, F) - credsF(backend, F).map(creds => - AwsEnvironment - .make[F]( + region: AwsRegion, + logger: StructuredLogger[F], + credentialsResourceF: SimpleHttpClient[F] => Resource[F, F[ + AwsCredentials + ]], + encoders: LogEncoders[F], + logRequestsResponses: Boolean + )(implicit F: Async[F]) { + + def withClient(client: Client[F]): Builder[F] = copy(client = client) + def withRegion(region: AwsRegion): Builder[F] = copy(region = region) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withCredentials( + credentialsResourceF: SimpleHttpClient[F] => Resource[F, F[ + AwsCredentials + ]] + ): Builder[F] = + copy(credentialsResourceF = credentialsResourceF) + def withLogEncoders(encoders: LogEncoders[F]): Builder[F] = + copy(encoders = encoders) + def withLogRequestsResponses(logRequestsResponses: Boolean): Builder[F] = + copy(logRequestsResponses = logRequestsResponses) + def enableLogging: Builder[F] = withLogRequestsResponses(true) + def disableLogging: Builder[F] = withLogRequestsResponses(false) + + def build: Resource[F, KinesisClient[F]] = { + val clnt = + if (logRequestsResponses) + RequestResponseLogger(logger, encoders)(client) + else client + val backend = + AwsHttp4sBackend[F](RequestResponseLogger(logger, encoders)(clnt)) + for { + credentials <- credentialsResourceF(backend) + environment = AwsEnvironment.make[F]( backend, - region, - creds, + F.pure(region), + credentials, F.realTime.map(_.toSeconds).map(Timestamp(_, 0)) ) - ) + awsClient <- AwsClient.simple(Kinesis.service, environment) + } yield awsClient + } + } + + object Builder { + def default[F[_]](client: Client[F], region: AwsRegion)(implicit + F: Async[F] + ): Builder[F] = + Builder[F]( + client, + region, + NoOpLogger[F], + backend => AwsCredentialsProvider.default(backend), + LogEncoders.show[F], + true + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () } /** Helper class containing required @@ -103,57 +143,4 @@ object KinesisClient { new KinesisClient.LogEncoders[F]() } } - - /** Create a KinesisClient [[cats.effect.Resource Resource]] - * - * @param client - * [[https://http4s.org/v0.23/docs/client.html Client]] implementation for - * the api calls - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]] - * in use - * @param loggerF - * [[cats.effect.Async Async]] => [[cats.effect.Async Async]] of - * [[https://github.com/typelevel/log4cats/blob/main/core/shared/src/main/scala/org/typelevel/log4cats/StructuredLogger.scala StructuredLogger]]. - * Default is - * [[https://github.com/typelevel/log4cats/blob/main/noop/shared/src/main/scala/org/typelevel/log4cats/noop/NoOpLogger.scala NoOpLogger]] - * @param credsF - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws/src/smithy4s/aws/SimpleHttpClient.scala SimpleHttpClient]] - * \=> - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsCredentials.scala AwsCredentials]]. - * Default to - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws/src/smithy4s/aws/AwsCredentialsProvider.scala AwsCredentialsProvider.default]] - * @param F - * [[cats.effect.Async Async]] - * @param LE - * [[kinesis4cats.smithy4s.client.KinesisClient.LogEncoders KinesisClient.LogEncoders]] - * @return - * [[cats.effect.Resource Resource]] of a Kinesis Client. - */ - def apply[F[_]]( - client: Client[F], - region: F[AwsRegion], - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger[F](f)), - credsF: ( - SimpleHttpClient[F], - Async[F] - ) => Resource[F, F[AwsCredentials]] = - (x: SimpleHttpClient[F], f: Async[F]) => - AwsCredentialsProvider.default[F](x)(f), - backendF: (Client[F], Async[F]) => SimpleHttpClient[F] = - (client: Client[F], f: Async[F]) => AwsHttp4sBackend[F](client)(f), - encoders: LogEncoders[F] = LogEncoders.show[F] - )(implicit - F: Async[F] - ): Resource[F, KinesisClient[F]] = for { - logger <- loggerF(F).toResource - env <- awsEnv( - RequestResponseLogger(logger, encoders)(client), - region, - credsF, - backendF - ) - awsClient <- AwsClient.simple(Kinesis.service, env) - } yield awsClient } diff --git a/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/KinesisProducer.scala b/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/KinesisProducer.scala index 1a4ebeb3..7b89eba5 100644 --- a/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/KinesisProducer.scala +++ b/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/KinesisProducer.scala @@ -22,7 +22,6 @@ import java.time.Instant import cats.data.NonEmptyList import cats.effect.Resource import cats.effect._ -import cats.effect.syntax.all._ import cats.syntax.all._ import com.amazonaws.kinesis._ import org.http4s.client.Client @@ -104,6 +103,76 @@ final class KinesisProducer[F[_]] private[kinesis4cats] ( object KinesisProducer { + final case class Builder[F[_]] private ( + config: Producer.Config[F], + client: Client[F], + region: AwsRegion, + logger: StructuredLogger[F], + credentialsResourceF: SimpleHttpClient[F] => Resource[F, F[ + AwsCredentials + ]], + encoders: LogEncoders[F], + logRequestsResponses: Boolean + )(implicit F: Async[F]) { + def withConfig(config: Producer.Config[F]): Builder[F] = + copy(config = config) + def withClient(client: Client[F]): Builder[F] = copy(client = client) + def withRegion(region: AwsRegion): Builder[F] = copy(region = region) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withCredentials( + credentialsResourceF: SimpleHttpClient[F] => Resource[F, F[ + AwsCredentials + ]] + ): Builder[F] = + copy(credentialsResourceF = credentialsResourceF) + def withLogEncoders(encoders: LogEncoders[F]): Builder[F] = + copy(encoders = encoders) + def withLogRequestsResponses(logRequestsResponses: Boolean): Builder[F] = + copy(logRequestsResponses = logRequestsResponses) + def enableLogging: Builder[F] = withLogRequestsResponses(true) + def disableLogging: Builder[F] = withLogRequestsResponses(false) + + def build: Resource[F, KinesisProducer[F]] = for { + client <- KinesisClient.Builder + .default[F](client, region) + .withLogger(logger) + .withCredentials(credentialsResourceF) + .withLogEncoders(encoders.kinesisClientLogEncoders) + .withLogRequestsResponses(logRequestsResponses) + .build + shardMapCache <- ShardMapCache.Builder + .default(getShardMap(client, config.streamNameOrArn), logger) + .withLogEncoders(encoders.producerLogEncoders.shardMapLogEncoders) + .build + } yield new KinesisProducer[F]( + logger, + shardMapCache, + config, + client, + encoders.producerLogEncoders + ) + } + + object Builder { + def default[F[_]]( + streamNameOrArn: models.StreamNameOrArn, + client: Client[F], + region: AwsRegion + )(implicit F: Async[F]): Builder[F] = Builder[F]( + Producer.Config.default(streamNameOrArn), + client, + region, + NoOpLogger[F], + backend => AwsCredentialsProvider.default(backend), + LogEncoders.show[F], + true + ) + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } + final class LogEncoders[F[_]]( val kinesisClientLogEncoders: KinesisClient.LogEncoders[F], val producerLogEncoders: Producer.LogEncoders @@ -159,73 +228,4 @@ object KinesisProducer { ) ) ) - - /** Basic constructor for the - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - * - * @param config - * [[kinesis4cats.producer.Producer.Config Producer.Config]] - * @param client - * [[org.http4s.client.Client Client]] instance - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]] - * @param loggerF - * [[cats.effect.Async Async]] => F of - * [[org.typelevel.log4cats.StructuredLogger StructuredLogger]]. Default - * uses [[org.typelevel.log4cats.noop.NoOpLogger NoOpLogger]] - * @param credsF - * ( - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws/src/smithy4s/aws/SimpleHttpClient.scala SimpleHttpClient]], - * [[cats.effect.Async Async]]) => F of - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsCredentials.scala AwsCredentials]] - * Default uses - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws/src/smithy4s/aws/AwsCredentialsProvider.scala AwsCredentialsProvider.default]] - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * Default to show instances - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - */ - def apply[F[_]]( - config: Producer.Config[F], - client: Client[F], - region: F[AwsRegion], - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger[F](f)), - credsF: ( - SimpleHttpClient[F], - Async[F] - ) => Resource[F, F[AwsCredentials]] = - (x: SimpleHttpClient[F], f: Async[F]) => - AwsCredentialsProvider.default[F](x)(f), - encoders: KinesisProducer.LogEncoders[F] = - KinesisProducer.LogEncoders.show[F] - )(implicit - F: Async[F] - ): Resource[F, KinesisProducer[F]] = for { - logger <- loggerF(F).toResource - underlying <- KinesisClient[F]( - client, - region, - loggerF, - credsF, - encoders = encoders.kinesisClientLogEncoders - ) - shardMapCache <- ShardMapCache[F]( - config.shardMapCacheConfig, - getShardMap(underlying, config.streamNameOrArn), - loggerF(F), - encoders.producerLogEncoders.shardMapLogEncoders - ) - producer = new KinesisProducer[F]( - logger, - shardMapCache, - config, - underlying, - encoders.producerLogEncoders - ) - } yield producer } diff --git a/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducer.scala b/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducer.scala index 71d14cac..5cf3364c 100644 --- a/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducer.scala +++ b/smithy4s-client/src/main/scala/kinesis4cats/smithy4s/client/producer/fs2/FS2KinesisProducer.scala @@ -31,6 +31,7 @@ import smithy4s.aws.SimpleHttpClient import smithy4s.aws.kernel.AwsCredentials import smithy4s.aws.kernel.AwsRegion +import kinesis4cats.models import kinesis4cats.producer._ import kinesis4cats.producer.fs2.FS2Producer @@ -57,80 +58,77 @@ final class FS2KinesisProducer[F[_]] private[kinesis4cats] ( override protected val channel: Channel[F, Record], override protected val underlying: KinesisProducer[F] )( - override protected val callback: ( - Producer.Res[PutRecordsOutput], - Async[F] - ) => F[Unit] + override protected val callback: Producer.Res[PutRecordsOutput] => F[Unit] )(implicit F: Async[F] ) extends FS2Producer[F, PutRecordsInput, PutRecordsOutput] object FS2KinesisProducer { - /** Basic constructor for the - * [[kinesis4cats.smithy4s.client.producer.fs2.FS2KinesisProducer FS2KinesisProducer]] - * - * @param config - * [[kinesis4cats.producer.fs2.FS2Producer.Config FS2Producer.Config]] - * @param client - * [[org.http4s.client.Client Client]] instance - * @param region - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsRegion.scala AwsRegion]] - * @param loggerF - * [[cats.effect.Async Async]] => F of - * [[org.typelevel.log4cats.StructuredLogger StructuredLogger]]. Default - * uses [[org.typelevel.log4cats.noop.NoOpLogger NoOpLogger]] - * @param credsF - * ( - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws/src/smithy4s/aws/SimpleHttpClient.scala SimpleHttpClient]], - * [[cats.effect.Async Async]]) => F of - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws-kernel/src/smithy4s/aws/AwsCredentials.scala AwsCredentials]] - * Default uses - * [[https://github.com/disneystreaming/smithy4s/blob/series/0.17/modules/aws/src/smithy4s/aws/AwsCredentialsProvider.scala AwsCredentialsProvider.default]] - * @param callback - * Function that can be run after each of the put results from the - * underlying - * @param F - * [[cats.effect.Async Async]] - * @param encoders - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer.LogEncoders KinesisProducer.LogEncoders]]. - * Default to show instances - * @return - * [[cats.effect.Resource Resource]] of - * [[kinesis4cats.smithy4s.client.producer.KinesisProducer KinesisProducer]] - */ - def apply[F[_]]( + final case class Builder[F[_]] private ( config: FS2Producer.Config[F], client: Client[F], - region: F[AwsRegion], - loggerF: Async[F] => F[StructuredLogger[F]] = (f: Async[F]) => - f.pure(NoOpLogger[F](f)), - credsF: ( - SimpleHttpClient[F], - Async[F] - ) => Resource[F, F[AwsCredentials]] = - (x: SimpleHttpClient[F], f: Async[F]) => - AwsCredentialsProvider.default[F](x)(f), - callback: (Producer.Res[PutRecordsOutput], Async[F]) => F[Unit] = - (_: Producer.Res[PutRecordsOutput], f: Async[F]) => f.unit, - encoders: KinesisProducer.LogEncoders[F] = - KinesisProducer.LogEncoders.show[F] - )(implicit - F: Async[F] - ): Resource[F, FS2KinesisProducer[F]] = for { - logger <- loggerF(F).toResource - underlying <- KinesisProducer( - config.producerConfig, + region: AwsRegion, + logger: StructuredLogger[F], + credentialsResourceF: SimpleHttpClient[F] => Resource[F, F[ + AwsCredentials + ]], + encoders: KinesisProducer.LogEncoders[F], + logRequestsResponses: Boolean, + callback: Producer.Res[PutRecordsOutput] => F[Unit] + )(implicit F: Async[F]) { + def withConfig(config: FS2Producer.Config[F]): Builder[F] = + copy(config = config) + def withClient(client: Client[F]): Builder[F] = copy(client = client) + def withRegion(region: AwsRegion): Builder[F] = copy(region = region) + def withLogger(logger: StructuredLogger[F]): Builder[F] = + copy(logger = logger) + def withCredentials( + credentialsResourceF: SimpleHttpClient[F] => Resource[F, F[ + AwsCredentials + ]] + ): Builder[F] = + copy(credentialsResourceF = credentialsResourceF) + def withLogEncoders(encoders: KinesisProducer.LogEncoders[F]): Builder[F] = + copy(encoders = encoders) + def withLogRequestsResponses(logRequestsResponses: Boolean): Builder[F] = + copy(logRequestsResponses = logRequestsResponses) + def enableLogging: Builder[F] = withLogRequestsResponses(true) + def disableLogging: Builder[F] = withLogRequestsResponses(false) + + def build: Resource[F, FS2KinesisProducer[F]] = for { + underlying <- KinesisProducer.Builder + .default[F](config.producerConfig.streamNameOrArn, client, region) + .withLogger(logger) + .withCredentials(credentialsResourceF) + .withLogEncoders(encoders) + .withLogRequestsResponses(logRequestsResponses) + .build + channel <- Channel.bounded[F, Record](config.queueSize).toResource + producer = new FS2KinesisProducer[F](logger, config, channel, underlying)( + callback + ) + _ <- producer.resource + } yield producer + } + + object Builder { + def default[F[_]]( + streamNameOrArn: models.StreamNameOrArn, + client: Client[F], + region: AwsRegion + )(implicit F: Async[F]): Builder[F] = Builder[F]( + FS2Producer.Config.default(streamNameOrArn), client, region, - loggerF, - credsF, - encoders + NoOpLogger[F], + backend => AwsCredentialsProvider.default(backend), + KinesisProducer.LogEncoders.show[F], + true, + (_: Producer.Res[PutRecordsOutput]) => F.unit ) - channel <- Channel.bounded[F, Record](config.queueSize).toResource - producer = new FS2KinesisProducer[F](logger, config, channel, underlying)( - callback - ) - _ <- producer.resource - } yield producer + + @annotation.unused + private def unapply[F[_]](builder: Builder[F]): Unit = () + } }