From 65600aa583d2f3db600de9ffb7eba289d01982fd Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Thu, 24 Aug 2023 16:42:03 -0400 Subject: [PATCH 01/18] Integrate ShellSentry on CI --- .github/workflows/ci.yml | 4 +- config/shell-sentry/config.json | 4 + scripts/shell-sentry.main.kts | 148 ++++++++++++++++++++++++++++++++ scripts/shell_sentry.sh | 4 + 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 config/shell-sentry/config.json create mode 100755 scripts/shell-sentry.main.kts create mode 100755 scripts/shell_sentry.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 303779c43..089fdc46c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,9 @@ jobs: path: build/skippy/** - name: Build and run tests - run: ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --continue --quiet --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt + env: + OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} + run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --continue --quiet --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt - name: Filter sarif if: always() && github.event_name == 'pull_request' diff --git a/config/shell-sentry/config.json b/config/shell-sentry/config.json new file mode 100644 index 000000000..8d2e0da63 --- /dev/null +++ b/config/shell-sentry/config.json @@ -0,0 +1,4 @@ +{ + "version": 2, + "gradle_enterprise_server": "https://gradle.com" +} diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts new file mode 100755 index 000000000..dc2c6d2b8 --- /dev/null +++ b/scripts/shell-sentry.main.kts @@ -0,0 +1,148 @@ +#!/usr/bin/env kotlin +@file:DependsOn("com.github.ajalt.clikt:clikt-jvm:4.2.0") +@file:DependsOn("com.slack.cli:kotlin-cli-util:2.2.0") +@file:DependsOn("com.squareup.retrofit2:retrofit:2.9.0") +@file:DependsOn("com.squareup.retrofit2:converter-moshi:2.9.0") +@file:DependsOn("com.squareup.okhttp3:okhttp:5.0.0-alpha.11") +@file:DependsOn("com.squareup.moshi:moshi:1.15.0") +@file:DependsOn("com.squareup.moshi:moshi-kotlin:1.15.0") + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass +import com.squareup.moshi.Moshi +import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory +import java.nio.file.Path +import kotlin.io.path.readText +import kotlinx.coroutines.runBlocking +import okhttp3.OkHttpClient +import retrofit2.Retrofit +import retrofit2.converter.moshi.MoshiConverterFactory +import retrofit2.create +import retrofit2.http.Body +import retrofit2.http.POST +import slack.cli.shellsentry.AnalysisResult +import slack.cli.shellsentry.NoStacktraceThrowable +import slack.cli.shellsentry.RetrySignal +import slack.cli.shellsentry.ShellSentry +import slack.cli.shellsentry.ShellSentryExtension + +class GuessedIssue(message: String) : NoStacktraceThrowable(message) + +class AiClient(private val accessToken: String) { + + private val moshi = Moshi.Builder().add(KotlinJsonAdapterFactory()).build() + private val client = + OkHttpClient.Builder() + .addInterceptor { chain -> + val request = + chain.request().newBuilder().addHeader("Authorization", "Bearer $accessToken").build() + chain.proceed(request) + } + .build() + private val api = + Retrofit.Builder() + .baseUrl("https://api.openai.com") + .client(client) + .addConverterFactory(MoshiConverterFactory.create(moshi)) + .build() + .create() + + suspend fun analyze(content: String): AnalysisResult? { + return try { + val response = api.analyze(content) + val rawJson = response.choices.first().message.content.trim() + val parsed = + moshi.adapter(ChoiceAnalysis::class.java).fromJson(rawJson) + ?: error("Could not parse: $rawJson") + AnalysisResult( + parsed.message, + parsed.explanation, + if (parsed.retry) RetrySignal.RetryImmediately else RetrySignal.Ack, + parsed.confidence + ) { message -> + GuessedIssue(message) + } + } catch (t: Throwable) { + t.printStackTrace() + null + } + } + + interface ChatGptApi { + @POST("/v1/chat/completions") + suspend fun completion(@Body request: CompletionRequest): CompletionResponse + + suspend fun analyze(content: String) = + completion( + CompletionRequest(messages = listOf(Message(content = "$ANALYSIS_PROMPT\n\n$content"))) + ) + + companion object { + val ANALYSIS_PROMPT = + """ + Given the following console output, please provide a diagnosis in a raw JSON object format: + + - "message": A broad single-line description of the error without specifying exact details, suitable for crash reporter grouping. + - "explanation": A detailed, multi-line message explaining the error and suggesting a solution. + - "retry": A boolean value (true/false) indicating whether a retry could potentially resolve the CI issue. + - "confidence": An integer value between 1-100 representing your confidence about the accuracy of your error identification. + """ + .trimIndent() + } + } + + @JsonClass(generateAdapter = false) + data class CompletionRequest(val model: String = "gpt-3.5-turbo", val messages: List) + + @JsonClass(generateAdapter = false) + data class Message(val role: String = "user", val content: String) + + @JsonClass(generateAdapter = false) + data class CompletionResponse( + val id: String, + @Json(name = "object") val objectType: String, + val created: Long, + val model: String, + val choices: List, + val usage: Usage + ) { + + @JsonClass(generateAdapter = false) + data class Choice( + val index: Int, + @Json(name = "finish_reason") val finishReason: String, + val message: Message, + ) + + @JsonClass(generateAdapter = false) + data class Usage( + @Json(name = "prompt_tokens") val promptTokens: Int, + @Json(name = "completion_tokens") val completionTokens: Int, + @Json(name = "total_tokens") val totalTokens: Int, + ) + } + + @JsonClass(generateAdapter = false) + data class ChoiceAnalysis( + val message: String, + val explanation: String, + val retry: Boolean, + val confidence: Int, + ) +} + +class AiExtension(private val aiClient: AiClient) : ShellSentryExtension { + override fun check( + command: String, + exitCode: Int, + isAfterRetry: Boolean, + consoleOutput: Path + ): AnalysisResult? { + return runBlocking { aiClient.analyze(consoleOutput.readText()) } + } +} + +val openAiKey: String? = System.getenv("OPEN_AI_KEY") +val extensions = if (openAiKey != null) listOf(AiExtension(AiClient(openAiKey))) else emptyList() + +ShellSentry.create(args, ::println).copy(extensions = extensions).exec() diff --git a/scripts/shell_sentry.sh b/scripts/shell_sentry.sh new file mode 100755 index 000000000..4d93b435b --- /dev/null +++ b/scripts/shell_sentry.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +./scripts/shell-sentry.main.kts --debug --verbose --config config/shell-sentry/config.json -- "$*" From c0fb659b5aee2d7b5176a3453a65120e17530abf Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Thu, 24 Aug 2023 16:48:12 -0400 Subject: [PATCH 02/18] Cap tokens --- scripts/shell-sentry.main.kts | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index dc2c6d2b8..89b2425ae 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -49,7 +49,19 @@ class AiClient(private val accessToken: String) { suspend fun analyze(content: String): AnalysisResult? { return try { - val response = api.analyze(content) + val contentTokens = content.split(" ").size + val truncatedContent = + if (contentTokens > ChatGptApi.remainingTokens) { + println("Truncating content due to token limit") + val tokensToRemove = contentTokens - ChatGptApi.remainingTokens + val split = content.split(" ") + // Split in reverse as we want to focus on the end of the content and travel up + split.asReversed().subList(tokensToRemove, split.size).reversed().joinToString(" ") + } else { + content + } + val prompt = "${ChatGptApi.ANALYSIS_PROMPT}\n\n$truncatedContent" + val response = api.analyze(prompt) val rawJson = response.choices.first().message.content.trim() val parsed = moshi.adapter(ChoiceAnalysis::class.java).fromJson(rawJson) @@ -72,12 +84,13 @@ class AiClient(private val accessToken: String) { @POST("/v1/chat/completions") suspend fun completion(@Body request: CompletionRequest): CompletionResponse - suspend fun analyze(content: String) = - completion( - CompletionRequest(messages = listOf(Message(content = "$ANALYSIS_PROMPT\n\n$content"))) - ) + suspend fun analyze(content: String): CompletionResponse { + return completion(CompletionRequest(messages = listOf(Message(content = content)))) + } companion object { + private const val MAX_TOKENS = 4096 + val ANALYSIS_PROMPT = """ Given the following console output, please provide a diagnosis in a raw JSON object format: @@ -88,6 +101,9 @@ class AiClient(private val accessToken: String) { - "confidence": An integer value between 1-100 representing your confidence about the accuracy of your error identification. """ .trimIndent() + + private val promptTokens = ANALYSIS_PROMPT.split(" ").size + val remainingTokens = MAX_TOKENS - promptTokens - 100 // 100 for buffer } } From 54b7f2f5f42ebb9a94d4512f2fe2793574e549be Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 25 Aug 2023 14:05:02 -0400 Subject: [PATCH 03/18] Update --- scripts/shell-sentry.main.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index 89b2425ae..c333ac845 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -1,6 +1,6 @@ #!/usr/bin/env kotlin @file:DependsOn("com.github.ajalt.clikt:clikt-jvm:4.2.0") -@file:DependsOn("com.slack.cli:kotlin-cli-util:2.2.0") +@file:DependsOn("com.slack.cli:kotlin-cli-util:2.2.1") @file:DependsOn("com.squareup.retrofit2:retrofit:2.9.0") @file:DependsOn("com.squareup.retrofit2:converter-moshi:2.9.0") @file:DependsOn("com.squareup.okhttp3:okhttp:5.0.0-alpha.11") From af6e4ed8a5dc23a7d435fd46681f694daf37d418 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 25 Aug 2023 14:33:10 -0400 Subject: [PATCH 04/18] Test failure --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d64d9c9d8..dc65f9579 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -121,7 +121,7 @@ androidx-paging-compose = "androidx.paging:paging-compose:3.2.0" androidx-compose-material-material3 = { module = "androidx.compose.material3:material3", version.ref = "compose-material3" } androidx-compose-material-material3-windowSizeClass = { module = "androidx.compose.material3:material3-window-size-class", version.ref = "compose-material3" } androidx-window = { module = "androidx.window:window", version = "1.2.0-beta01" } -androidx-splashscreen = "androidx.core:core-splashscreen:1.0.1" +androidx-splashscreen = "androidx.core:core-splashscreen:1.1.0-alpha01" androidx-profileinstaller = "androidx.profileinstaller:profileinstaller:1.3.1" androidx-test-ext-junit = "androidx.test.ext:junit:1.1.5" From 6d70a7d7e9e1cf4f6b9a5b9d26acddf442ca46f8 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 25 Aug 2023 17:13:31 -0400 Subject: [PATCH 05/18] What is happening here --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 089fdc46c..79e5e7911 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: - name: Build and run tests env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} - run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --continue --quiet --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt + run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt - name: Filter sarif if: always() && github.event_name == 'pull_request' From 4b318dd0fded4118f477601dec4500e0dcbb67c5 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 25 Aug 2023 18:16:42 -0400 Subject: [PATCH 06/18] WHAT IS UP --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79e5e7911..82d386790 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: path: build/skippy/** - name: Build and run tests + timeout-minutes: 25 env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt From 2cbd2a193ebe88d0fe712b0cb95ff37af979bedb Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 1 Sep 2023 17:18:38 -0400 Subject: [PATCH 07/18] Remove stacktrace --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82d386790..b4246e536 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: touch ~/.gradle/gradle.properties - name: Bootstrap - run: ./gradlew bootstrap --stacktrace + run: ./gradlew bootstrap - name: Get changed files if: always() && github.event_name == 'pull_request' @@ -92,7 +92,7 @@ jobs: timeout-minutes: 25 env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} - run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt + run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt - name: Filter sarif if: always() && github.event_name == 'pull_request' From e2f6bbd21f088ab5da06600d0449301d59d172f5 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 1 Sep 2023 17:18:47 -0400 Subject: [PATCH 08/18] Bump to 30 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4246e536..1b3a88fd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: path: build/skippy/** - name: Build and run tests - timeout-minutes: 25 + timeout-minutes: 30 env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt From 1a08d0452d4ce7f5f82f2112be4451f82b04a7b4 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 1 Sep 2023 17:53:02 -0400 Subject: [PATCH 09/18] Remove timeout --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b3a88fd1..ebaf8b429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,6 @@ jobs: path: build/skippy/** - name: Build and run tests - timeout-minutes: 30 env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt From 9f859b01493715b53c665ab3779cdc7e35cbc6f2 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 1 Sep 2023 21:25:31 -0400 Subject: [PATCH 10/18] Is this looping --- scripts/shell-sentry.main.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index c333ac845..e7a2de14b 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -154,6 +154,7 @@ class AiExtension(private val aiClient: AiClient) : ShellSentryExtension { isAfterRetry: Boolean, consoleOutput: Path ): AnalysisResult? { + println("AIExtension: Checking") return runBlocking { aiClient.analyze(consoleOutput.readText()) } } } From 92989a9a5a7f7be2b38211f637d5e75cfcf75ea8 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 1 Sep 2023 23:38:55 -0400 Subject: [PATCH 11/18] Move analyze out due to lack of jvm-defaults --- scripts/shell-sentry.main.kts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index e7a2de14b..10e710654 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -84,10 +84,6 @@ class AiClient(private val accessToken: String) { @POST("/v1/chat/completions") suspend fun completion(@Body request: CompletionRequest): CompletionResponse - suspend fun analyze(content: String): CompletionResponse { - return completion(CompletionRequest(messages = listOf(Message(content = content)))) - } - companion object { private const val MAX_TOKENS = 4096 @@ -107,6 +103,11 @@ class AiClient(private val accessToken: String) { } } + + private suspend fun ChatGptApi.analyze(content: String): CompletionResponse { + return completion(CompletionRequest(messages = listOf(Message(content = content)))) + } + @JsonClass(generateAdapter = false) data class CompletionRequest(val model: String = "gpt-3.5-turbo", val messages: List) From 7e3ceaa275f85697bee88c2116f9b3498f160dd8 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Fri, 1 Sep 2023 23:43:19 -0400 Subject: [PATCH 12/18] Better error messages --- scripts/shell-sentry.main.kts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index 10e710654..d39973568 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -15,6 +15,7 @@ import java.nio.file.Path import kotlin.io.path.readText import kotlinx.coroutines.runBlocking import okhttp3.OkHttpClient +import retrofit2.HttpException import retrofit2.Retrofit import retrofit2.converter.moshi.MoshiConverterFactory import retrofit2.create @@ -75,7 +76,15 @@ class AiClient(private val accessToken: String) { GuessedIssue(message) } } catch (t: Throwable) { - t.printStackTrace() + if (t is HttpException) { + System.err.println(""" + HTTP Error: ${t.code()} + Message: ${t.message()} + ${t.response()?.errorBody()?.string()} + """.trimIndent()) + } else { + System.err.println(t.stackTraceToString()) + } null } } From 36a737444f0ecb6117d7d8d4d45c7022fac2571d Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Sat, 2 Sep 2023 00:04:24 -0400 Subject: [PATCH 13/18] Add back limit --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebaf8b429..1b3a88fd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: path: build/skippy/** - name: Build and run tests + timeout-minutes: 30 env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt From 349c2812c4bc322b0336af3e86f0c812004c11cc Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Sat, 2 Sep 2023 00:05:58 -0400 Subject: [PATCH 14/18] Improve logging and token handling --- scripts/shell-sentry.main.kts | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index d39973568..d097cbbb0 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -50,18 +50,17 @@ class AiClient(private val accessToken: String) { suspend fun analyze(content: String): AnalysisResult? { return try { - val contentTokens = content.split(" ").size - val truncatedContent = - if (contentTokens > ChatGptApi.remainingTokens) { - println("Truncating content due to token limit") - val tokensToRemove = contentTokens - ChatGptApi.remainingTokens - val split = content.split(" ") - // Split in reverse as we want to focus on the end of the content and travel up - split.asReversed().subList(tokensToRemove, split.size).reversed().joinToString(" ") - } else { - content - } - val prompt = "${ChatGptApi.ANALYSIS_PROMPT}\n\n$truncatedContent" + val approxMaxChars = ChatGptApi.remainingTokens * ChatGptApi.AVG_TOKEN + val start = if (content.length > approxMaxChars) { + println("-- Truncating content due to token limit (max: ${ChatGptApi.remainingTokens}, content: ~${content.length / ChatGptApi.AVG_TOKEN}))") + println("-- Truncating to last $approxMaxChars chars (~${(approxMaxChars / content.length.toFloat()) * 100}%)}") + content.length - approxMaxChars + } else { + println("-- Not truncating content") + 0 + } + val analyzableContent = content.substring(start) + val prompt = "${ChatGptApi.ANALYSIS_PROMPT}\n\n$analyzableContent" val response = api.analyze(prompt) val rawJson = response.choices.first().message.content.trim() val parsed = @@ -77,11 +76,13 @@ class AiClient(private val accessToken: String) { } } catch (t: Throwable) { if (t is HttpException) { - System.err.println(""" + System.err.println( + """ HTTP Error: ${t.code()} Message: ${t.message()} ${t.response()?.errorBody()?.string()} - """.trimIndent()) + """.trimIndent() + ) } else { System.err.println(t.stackTraceToString()) } @@ -95,6 +96,7 @@ class AiClient(private val accessToken: String) { companion object { private const val MAX_TOKENS = 4096 + const val AVG_TOKEN = 4 val ANALYSIS_PROMPT = """ @@ -107,15 +109,17 @@ class AiClient(private val accessToken: String) { """ .trimIndent() - private val promptTokens = ANALYSIS_PROMPT.split(" ").size - val remainingTokens = MAX_TOKENS - promptTokens - 100 // 100 for buffer + private val promptTokens = ANALYSIS_PROMPT.length / AVG_TOKEN + + // Because these are all sorta fuzzy guesses, we want to leave some buffer + private const val TOKEN_BUFFER = 700 + val remainingTokens = MAX_TOKENS - promptTokens - TOKEN_BUFFER } } - private suspend fun ChatGptApi.analyze(content: String): CompletionResponse { - return completion(CompletionRequest(messages = listOf(Message(content = content)))) - } + private suspend fun ChatGptApi.analyze(content: String) = + completion(CompletionRequest(messages = listOf(Message(content = content)))) @JsonClass(generateAdapter = false) data class CompletionRequest(val model: String = "gpt-3.5-turbo", val messages: List) @@ -164,7 +168,7 @@ class AiExtension(private val aiClient: AiClient) : ShellSentryExtension { isAfterRetry: Boolean, consoleOutput: Path ): AnalysisResult? { - println("AIExtension: Checking") + println("-- AIExtension: Checking") return runBlocking { aiClient.analyze(consoleOutput.readText()) } } } From 041b7aa2ffa3d01d00b91e355548ea4f995dfd29 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Sat, 2 Sep 2023 00:06:43 -0400 Subject: [PATCH 15/18] TODO --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b3a88fd1..eeace1ddb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,7 @@ jobs: timeout-minutes: 30 env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} + # TODO the check task undoes what Skippy provides, better to split it run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt - name: Filter sarif From a06d4d2d4c8970ef084d09dde13c0a77c4d92ffb Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Sat, 2 Sep 2023 00:35:51 -0400 Subject: [PATCH 16/18] Print terminal info --- scripts/shell-sentry.main.kts | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index d097cbbb0..0b20e1d7a 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -1,12 +1,18 @@ #!/usr/bin/env kotlin @file:DependsOn("com.github.ajalt.clikt:clikt-jvm:4.2.0") +@file:DependsOn("com.github.ajalt.mordant:mordant:2.1.0") @file:DependsOn("com.slack.cli:kotlin-cli-util:2.2.1") @file:DependsOn("com.squareup.retrofit2:retrofit:2.9.0") @file:DependsOn("com.squareup.retrofit2:converter-moshi:2.9.0") @file:DependsOn("com.squareup.okhttp3:okhttp:5.0.0-alpha.11") @file:DependsOn("com.squareup.moshi:moshi:1.15.0") @file:DependsOn("com.squareup.moshi:moshi-kotlin:1.15.0") +// // To silence this stupid log https://www.slf4j.org/codes.html#StaticLoggerBinder +@file:DependsOn("org.slf4j:slf4j-nop:2.0.7") +import com.github.ajalt.mordant.rendering.Whitespace +import com.github.ajalt.mordant.table.table +import com.github.ajalt.mordant.terminal.Terminal import com.squareup.moshi.Json import com.squareup.moshi.JsonClass import com.squareup.moshi.Moshi @@ -26,6 +32,8 @@ import slack.cli.shellsentry.NoStacktraceThrowable import slack.cli.shellsentry.RetrySignal import slack.cli.shellsentry.ShellSentry import slack.cli.shellsentry.ShellSentryExtension +import kotlin.io.path.createTempFile +import kotlin.io.path.writeText class GuessedIssue(message: String) : NoStacktraceThrowable(message) @@ -60,6 +68,10 @@ class AiClient(private val accessToken: String) { 0 } val analyzableContent = content.substring(start) + createTempFile().apply { + writeText(analyzableContent) + println("Analyzable content written to file://${toAbsolutePath()}") + } val prompt = "${ChatGptApi.ANALYSIS_PROMPT}\n\n$analyzableContent" val response = api.analyze(prompt) val rawJson = response.choices.first().message.content.trim() @@ -169,7 +181,29 @@ class AiExtension(private val aiClient: AiClient) : ShellSentryExtension { consoleOutput: Path ): AnalysisResult? { println("-- AIExtension: Checking") - return runBlocking { aiClient.analyze(consoleOutput.readText()) } + val result = runBlocking { aiClient.analyze(consoleOutput.readText()) } ?: return null + + println("\n") + val t = Terminal() + t.println(table { + captionTop("Analysis Result") + body { row("Message", result.message) } + body { + row("Explanation", result.explanation) { + this.whitespace = Whitespace.PRE_WRAP + } + } + body { + row( + "Retry?", + "${result.retrySignal != RetrySignal.Ack && result.retrySignal != RetrySignal.Unknown}" + ) + } + body { row("Confidence", "${result.confidence}%") } + }) + println("\n") + + return result } } From dcc968a63b5f91dbcbe0e306f902571e1cfcf1e1 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Sun, 1 Oct 2023 15:10:15 -0400 Subject: [PATCH 17/18] Update version --- scripts/shell-sentry.main.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/shell-sentry.main.kts b/scripts/shell-sentry.main.kts index 0b20e1d7a..0f0f14dd1 100755 --- a/scripts/shell-sentry.main.kts +++ b/scripts/shell-sentry.main.kts @@ -1,7 +1,7 @@ #!/usr/bin/env kotlin @file:DependsOn("com.github.ajalt.clikt:clikt-jvm:4.2.0") @file:DependsOn("com.github.ajalt.mordant:mordant:2.1.0") -@file:DependsOn("com.slack.cli:kotlin-cli-util:2.2.1") +@file:DependsOn("com.slack.cli:kotlin-cli-util:2.3.0-alpha01") @file:DependsOn("com.squareup.retrofit2:retrofit:2.9.0") @file:DependsOn("com.squareup.retrofit2:converter-moshi:2.9.0") @file:DependsOn("com.squareup.okhttp3:okhttp:5.0.0-alpha.11") From ed01e86f4d7414374512c9bd7b98f1104fbc4782 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Thu, 9 Nov 2023 22:45:20 -0500 Subject: [PATCH 18/18] Split checks --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84c0e407b..24201b208 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: env: OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} # TODO the check task undoes what Skippy provides, better to split it - run: ./scripts/shell_sentry.sh ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt + run: ./scripts/shell_sentry.sh ./gradlew ${{ env.APP_TARGET }} checkSortDependencies spotlessCheck globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt - name: Filter sarif if: always() && github.event_name == 'pull_request'