Merge remote-tracking branch 'origin/v1.5-main' into fix-overhaul-ep-…

…discovery

.brazil.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "org.jetbrains.kotlin:kotlin-stdlib:2.0.*": "KotlinStdlib-2.x",
+    "org.jetbrains.kotlin:kotlin-stdlib:2.*": "KotlinStdlib-2.x",
     "org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.*": "KotlinxCoroutinesCoreJvm-1.x",
 
     "com.squareup.okhttp3:okhttp-coroutines:5.*": "OkHttp3Coroutines-5.x",

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [1.4.0] - 01/15/2025
+
+### Features
+* [#1431](https://github.com/awslabs/aws-sdk-kotlin/issues/1431) ⚠️ **IMPORTANT**: Add `retryStrategy` configuration option for waiters
+
+### Fixes
+* [#1321](https://github.com/awslabs/aws-sdk-kotlin/issues/1321) Include more information when retry strategy halts early due to token bucket capacity errors
+
+### Miscellaneous
+* ⚠️ **IMPORTANT**: Upgrade to Kotlin 2.1.0
+
 ## [1.3.34] - 01/10/2025
 
 ## [1.3.33] - 01/10/2025

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDependency.kt

@@ -37,7 +37,7 @@ private fun getDefaultRuntimeVersion(): String {
 // publishing info
 const val RUNTIME_GROUP: String = "aws.smithy.kotlin"
 val RUNTIME_VERSION: String = System.getProperty("smithy.kotlin.codegen.clientRuntimeVersion", getDefaultRuntimeVersion())
-val KOTLIN_COMPILER_VERSION: String = System.getProperty("smithy.kotlin.codegen.kotlinCompilerVersion", "2.0.10")
+val KOTLIN_COMPILER_VERSION: String = System.getProperty("smithy.kotlin.codegen.kotlinCompilerVersion", "2.1.0")
 
 enum class SourceSet {
     CommonMain,

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt

@@ -82,7 +82,7 @@ object RuntimeTypes {
             val ContinueInterceptor = symbol("ContinueInterceptor")
             val DiscoveredEndpointErrorInterceptor = symbol("DiscoveredEndpointErrorInterceptor")
             val HttpInterceptor = symbol("HttpInterceptor")
-            val Md5ChecksumInterceptor = symbol("Md5ChecksumInterceptor")
+            val HttpChecksumRequiredInterceptor = symbol("HttpChecksumRequiredInterceptor")
             val FlexibleChecksumsRequestInterceptor = symbol("FlexibleChecksumsRequestInterceptor")
             val FlexibleChecksumsResponseInterceptor = symbol("FlexibleChecksumsResponseInterceptor")
             val ResponseLengthValidationInterceptor = symbol("ResponseLengthValidationInterceptor")
@@ -233,6 +233,9 @@ object RuntimeTypes {
         object Config : RuntimeTypePackage(KotlinDependency.SMITHY_CLIENT, "config") {
             val RequestCompressionConfig = symbol("RequestCompressionConfig")
             val CompressionClientConfig = symbol("CompressionClientConfig")
+            val HttpChecksumConfig = symbol("HttpChecksumConfig")
+            val RequestHttpChecksumConfig = symbol("RequestHttpChecksumConfig")
+            val ResponseHttpChecksumConfig = symbol("ResponseHttpChecksumConfig")
         }
 
         object Endpoints : RuntimeTypePackage(KotlinDependency.SMITHY_CLIENT, "endpoints") {
@@ -397,6 +400,7 @@ object RuntimeTypes {
             val TelemetryContextElement = symbol("TelemetryContextElement", "context")
             val TraceSpan = symbol("TraceSpan", "trace")
             val withSpan = symbol("withSpan", "trace")
+            val warn = symbol("warn", "logging")
         }
         object TelemetryDefaults : RuntimeTypePackage(KotlinDependency.TELEMETRY_DEFAULTS) {
             val Global = symbol("Global")
@@ -411,6 +415,7 @@ object RuntimeTypes {
 
         val CompletableDeferred = "kotlinx.coroutines.CompletableDeferred".toSymbol()
         val job = "kotlinx.coroutines.job".toSymbol()
+        val runBlocking = "kotlinx.coroutines.runBlocking".toSymbol()
 
         object Flow {
             // NOTE: smithy-kotlin core has an API dependency on this already

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/checksums/HttpChecksumRequiredIntegration.kt

@@ -0,0 +1,66 @@
+package software.amazon.smithy.kotlin.codegen.rendering.checksums
+
+import software.amazon.smithy.aws.traits.HttpChecksumTrait
+import software.amazon.smithy.kotlin.codegen.KotlinSettings
+import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
+import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
+import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.shapes.OperationShape
+import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait
+
+/**
+ * Handles the `httpChecksumRequired` trait.
+ * See: https://smithy.io/2.0/spec/http-bindings.html#httpchecksumrequired-trait
+ */
+class HttpChecksumRequiredIntegration : KotlinIntegration {
+    override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
+        model.isTraitApplied(HttpChecksumRequiredTrait::class.java)
+
+    override fun customizeMiddleware(
+        ctx: ProtocolGenerator.GenerationContext,
+        resolved: List<ProtocolMiddleware>,
+    ): List<ProtocolMiddleware> = resolved + httpChecksumRequiredDefaultAlgorithmMiddleware + httpChecksumRequiredMiddleware
+}
+
+/**
+ * Adds default checksum algorithm to the execution context
+ */
+private val httpChecksumRequiredDefaultAlgorithmMiddleware = object : ProtocolMiddleware {
+    override val name: String = "httpChecksumRequiredDefaultAlgorithmMiddleware"
+    override val order: Byte = -2 // Before S3 Express (possibly) changes the default (-1) and before calculating checksum (0)
+
+    override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean =
+        op.hasTrait<HttpChecksumRequiredTrait>() && !op.hasTrait<HttpChecksumTrait>()
+
+    override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) {
+        writer.write(
+            "op.context[#T.DefaultChecksumAlgorithm] = #S",
+            RuntimeTypes.HttpClient.Operation.HttpOperationContext,
+            "MD5",
+        )
+    }
+}
+
+/**
+ * Adds interceptor to calculate request checksums.
+ * The `httpChecksum` trait supersedes the `httpChecksumRequired` trait. If both are applied to an operation use `httpChecksum`.
+ *
+ * See: https://smithy.io/2.0/aws/aws-core.html#behavior-with-httpchecksumrequired
+ */
+private val httpChecksumRequiredMiddleware = object : ProtocolMiddleware {
+    override val name: String = "httpChecksumRequiredMiddleware"
+
+    override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean =
+        op.hasTrait<HttpChecksumRequiredTrait>() && !op.hasTrait<HttpChecksumTrait>()
+
+    override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) {
+        writer.write(
+            "op.interceptors.add(#T())",
+            RuntimeTypes.HttpClient.Interceptors.HttpChecksumRequiredInterceptor,
+        )
+    }
+}

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolClientGenerator.kt

@@ -4,7 +4,6 @@
  */
 package software.amazon.smithy.kotlin.codegen.rendering.protocol
 
-import software.amazon.smithy.aws.traits.HttpChecksumTrait
 import software.amazon.smithy.codegen.core.Symbol
 import software.amazon.smithy.kotlin.codegen.core.*
 import software.amazon.smithy.kotlin.codegen.integration.SectionId
@@ -22,7 +21,6 @@ import software.amazon.smithy.model.knowledge.OperationIndex
 import software.amazon.smithy.model.knowledge.TopDownIndex
 import software.amazon.smithy.model.shapes.OperationShape
 import software.amazon.smithy.model.traits.EndpointTrait
-import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait
 
 /**
  * Renders an implementation of a service interface for HTTP protocol
@@ -318,8 +316,6 @@ open class HttpProtocolClientGenerator(
             .forEach { middleware ->
                 middleware.render(ctx, op, writer)
             }
-
-        op.renderIsMd5ChecksumRequired(writer)
     }
 
     /**
@@ -336,27 +332,6 @@ open class HttpProtocolClientGenerator(
      */
     protected open fun renderAdditionalMethods(writer: KotlinWriter) { }
 
-    /**
-     * Render optionally installing Md5ChecksumMiddleware.
-     * The Md5 middleware will only be installed if the operation requires a checksum and the user has not opted-in to flexible checksums.
-     */
-    private fun OperationShape.renderIsMd5ChecksumRequired(writer: KotlinWriter) {
-        val httpChecksumTrait = getTrait<HttpChecksumTrait>()
-
-        // the checksum requirement can be modeled in either HttpChecksumTrait's `requestChecksumRequired` or the HttpChecksumRequired trait
-        if (!hasTrait<HttpChecksumRequiredTrait>() && httpChecksumTrait == null) {
-            return
-        }
-
-        if (hasTrait<HttpChecksumRequiredTrait>() || httpChecksumTrait?.isRequestChecksumRequired == true) {
-            val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.Md5ChecksumInterceptor
-            val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(inputShape))
-            writer.withBlock("op.interceptors.add(#T<#T> {", "})", interceptorSymbol, inputSymbol) {
-                writer.write("op.context.getOrNull(#T.ChecksumAlgorithm) == null", RuntimeTypes.HttpClient.Operation.HttpOperationContext)
-            }
-        }
-    }
-
     /**
      * render a utility function to populate an operation's ExecutionContext with defaults from service config, environment, etc
      */

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/waiters/WaiterGenerator.kt

@@ -17,7 +17,7 @@ import java.text.DecimalFormatSymbols
  * Renders the top-level retry strategy for a waiter.
  */
 private fun KotlinWriter.renderRetryStrategy(wi: WaiterInfo, asValName: String) {
-    withBlock("val #L = #T {", "}", asValName, RuntimeTypes.Core.Retries.StandardRetryStrategy) {
+    withBlock("val #L = retryStrategy ?: #T {", "}", asValName, RuntimeTypes.Core.Retries.StandardRetryStrategy) {
         write("maxAttempts = 20")
         write("tokenBucket = #T", RuntimeTypes.Core.Retries.Delay.InfiniteTokenBucket)
         withBlock("delayProvider {", "}") {
@@ -35,18 +35,21 @@ private fun KotlinWriter.renderRetryStrategy(wi: WaiterInfo, asValName: String)
 internal fun KotlinWriter.renderWaiter(wi: WaiterInfo) {
     write("")
     wi.waiter.documentation.ifPresent(::dokka)
-    val inputParameter = if (wi.input.hasAllOptionalMembers) {
-        format("request: #1T = #1T { }", wi.inputSymbol)
+
+    val requestType = if (wi.input.hasAllOptionalMembers) {
+        format("#1T = #1T { }", wi.inputSymbol)
     } else {
-        format("request: #T", wi.inputSymbol)
+        format("#T", wi.inputSymbol)
     }
+
     withBlock(
-        "#L suspend fun #T.#L(#L): #T<#T> {",
+        "#L suspend fun #T.#L(request: #L, retryStrategy: #T? = null): #T<#T> {",
         "}",
         wi.ctx.settings.api.visibility,
         wi.serviceSymbol,
         wi.methodName,
-        inputParameter,
+        requestType,
+        RuntimeTypes.Core.Retries.RetryStrategy,
         RuntimeTypes.Core.Retries.Outcome,
         wi.outputSymbol,
     ) {

codegen/smithy-kotlin-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration

@@ -13,3 +13,4 @@ software.amazon.smithy.kotlin.codegen.rendering.endpoints.SdkEndpointBuiltinInte
 software.amazon.smithy.kotlin.codegen.rendering.compression.RequestCompressionIntegration
 software.amazon.smithy.kotlin.codegen.rendering.auth.SigV4AsymmetricAuthSchemeIntegration
 software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestsIntegration
+software.amazon.smithy.kotlin.codegen.rendering.checksums.HttpChecksumRequiredIntegration

codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/waiters/ServiceWaitersGeneratorTest.kt

@@ -40,7 +40,7 @@ class ServiceWaitersGeneratorTest {
             /**
              * Wait until a foo exists with optional input
              */
-            public suspend fun TestClient.waitUntilFooOptionalExists(request: DescribeFooOptionalRequest = DescribeFooOptionalRequest { }): Outcome<DescribeFooOptionalResponse> {
+            public suspend fun TestClient.waitUntilFooOptionalExists(request: DescribeFooOptionalRequest = DescribeFooOptionalRequest { }, retryStrategy: RetryStrategy? = null): Outcome<DescribeFooOptionalResponse> {
         """.trimIndent()
         val methodFooter = """
                 val policy = AcceptorRetryPolicy(request, acceptors)
@@ -56,7 +56,7 @@ class ServiceWaitersGeneratorTest {
             /**
              * Wait until a foo exists with required input
              */
-            public suspend fun TestClient.waitUntilFooRequiredExists(request: DescribeFooRequiredRequest): Outcome<DescribeFooRequiredResponse> {
+            public suspend fun TestClient.waitUntilFooRequiredExists(request: DescribeFooRequiredRequest, retryStrategy: RetryStrategy? = null): Outcome<DescribeFooRequiredResponse> {
         """.trimIndent()
         listOf(
             generateService("simple-service-with-operation-waiter.smithy"),
@@ -102,7 +102,7 @@ class ServiceWaitersGeneratorTest {
     @Test
     fun testRetryStrategy() {
         val expected = """
-            val strategy = StandardRetryStrategy {
+            val strategy = retryStrategy ?: StandardRetryStrategy {
                 maxAttempts = 20
                 tokenBucket = InfiniteTokenBucket
                 delayProvider {

codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/waiters/WaiterGeneratorTest.kt

@@ -28,7 +28,7 @@ class WaiterGeneratorTest {
     @Test
     fun testDefaultDelays() {
         val expected = """
-            val strategy = StandardRetryStrategy {
+            val strategy = retryStrategy ?: StandardRetryStrategy {
                 maxAttempts = 20
                 tokenBucket = InfiniteTokenBucket
                 delayProvider {
@@ -45,7 +45,7 @@ class WaiterGeneratorTest {
     @Test
     fun testCustomDelays() {
         val expected = """
-            val strategy = StandardRetryStrategy {
+            val strategy = retryStrategy ?: StandardRetryStrategy {
                 maxAttempts = 20
                 tokenBucket = InfiniteTokenBucket
                 delayProvider {

gradle.properties

@@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false
 org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G
 
 # SDK
-sdkVersion=1.3.35-SNAPSHOT
+sdkVersion=1.4.1-SNAPSHOT
 
 # codegen
-codegenVersion=0.33.35-SNAPSHOT
+codegenVersion=0.34.1-SNAPSHOT

gradle/libs.versions.toml

@@ -1,20 +1,21 @@
 [versions]
-kotlin-version = "2.0.21"
+kotlin-version = "2.1.0"
 dokka-version = "1.9.10"
 
-aws-kotlin-repo-tools-version = "0.4.16"
+aws-kotlin-repo-tools-version = "0.4.17"
 
 # libs
 coroutines-version = "1.9.0"
 atomicfu-version = "0.25.0"
 okhttp-version = "5.0.0-alpha.14"
 okhttp4-version = "4.12.0"
 okio-version = "3.9.1"
-otel-version = "1.43.0"
+otel-version = "1.45.0"
 slf4j-version = "2.0.16"
 slf4j-v1x-version = "1.7.36"
-crt-kotlin-version = "0.8.10"
-micrometer-version = "1.13.6"
+crt-kotlin-version = "0.9.0"
+micrometer-version = "1.14.2"
+binary-compatibility-validator-version = "0.16.3"
 
 # codegen
 smithy-version = "1.53.0"
@@ -23,7 +24,7 @@ smithy-gradle-version = "0.9.0"
 # testing
 junit-version = "5.10.5"
 kotest-version = "5.9.1"
-kotlin-compile-testing-version = "1.6.0"
+kotlin-compile-testing-version = "0.7.0"
 kotlinx-benchmark-version = "0.4.12"
 kotlinx-serialization-version = "1.7.3"
 docker-java-version = "3.4.0"
@@ -80,7 +81,7 @@ smithy-smoke-test-traits = { module = "software.amazon.smithy:smithy-smoke-test-
 junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-version" }
 junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-version" }
 
-kotlin-compile-testing = {module = "com.github.tschuchortdev:kotlin-compile-testing", version.ref = "kotlin-compile-testing-version" }
+kotlin-compile-testing = {module = "dev.zacsweers.kctfork:core", version.ref = "kotlin-compile-testing-version" }
 kotest-assertions-core = { module = "io.kotest:kotest-assertions-core", version.ref = "kotest-version" }
 kotest-assertions-core-jvm = { module = "io.kotest:kotest-assertions-core-jvm", version.ref = "kotest-version" }
 kotlinx-benchmark-runtime = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-runtime", version.ref = "kotlinx-benchmark-version" }
@@ -104,7 +105,7 @@ dokka = { id = "org.jetbrains.dokka", version.ref = "dokka-version"}
 kotlin-jvm = {id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin-version" }
 kotlin-multiplatform = {id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin-version" }
 kotlinx-benchmark = { id = "org.jetbrains.kotlinx.benchmark", version.ref = "kotlinx-benchmark-version" }
-kotlinx-binary-compatibility-validator = { id = "org.jetbrains.kotlinx.binary-compatibility-validator", version = "0.13.2" }
+kotlinx-binary-compatibility-validator = { id = "org.jetbrains.kotlinx.binary-compatibility-validator", version.ref = "binary-compatibility-validator-version" }
 kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin-version"}
 aws-kotlin-repo-tools-kmp = { id = "aws.sdk.kotlin.gradle.kmp", version.ref = "aws-kotlin-repo-tools-version" }
 aws-kotlin-repo-tools-smithybuild = { id = "aws.sdk.kotlin.gradle.smithybuild", version.ref = "aws-kotlin-repo-tools-version" }

runtime/auth/aws-signing-tests/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/tests/SigningSuiteTestBaseJVM.kt

@@ -28,6 +28,7 @@ import io.ktor.util.*
 import io.ktor.utils.io.*
 import io.ktor.utils.io.core.*
 import kotlinx.coroutines.runBlocking
+import kotlinx.io.readByteArray
 import kotlinx.serialization.json.*
 import org.junit.jupiter.api.Assumptions.assumeTrue
 import org.junit.jupiter.api.Test
@@ -383,7 +384,7 @@ public actual abstract class SigningSuiteTestBase : HasSigner {
         }
 
         if (hasBody) {
-            val bytes = runBlocking { chan.readRemaining().readBytes() }
+            val bytes = runBlocking { chan.readRemaining().readByteArray() }
             builder.body = HttpBody.fromBytes(bytes)
         }