Skip to content

Commit

Permalink
Distinguishes error codes in flakeguard (#16244)
Browse files Browse the repository at this point in the history 
* Distinguishes error codes in flakeguard

* Add slack IDs

* Adds branch name to flakeguard report

* Better aggregation
  • Loading branch information
@kalverra
kalverra authored Feb 5, 2025
1 parent 5e9d9f2 commit 94076ad
Showing 1 changed file with 76 additions and 9 deletions.
85 changes: 76 additions & 9 deletions .github/workflows/flakeguard.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ jobs:
- name: Install flakeguard
if: ${{ inputs.runAllTests == false }}
shell: bash
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@32d2999ffd11edff77f5251f9a1510803b8652ae # [email protected]
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@051b483f1e9e99b0219d1ccea4d8f7bf3be203b7 # [email protected]

- name: Find new or updated test packages
if: ${{ inputs.runAllTests == false && env.RUN_CUSTOM_TEST_PACKAGES == '' }}
Expand Down Expand Up @@ -267,6 +267,8 @@ jobs:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.get-tests.outputs.matrix) }}
outputs:
flakeguard_error: ${{ steps.run-tests.outputs.flakeguard_error }}
env:
DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
steps:
Expand Down Expand Up @@ -332,15 +334,27 @@ jobs:

- name: Install flakeguard
shell: bash
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@32d2999ffd11edff77f5251f9a1510803b8652ae # [email protected]
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@051b483f1e9e99b0219d1ccea4d8f7bf3be203b7 # [email protected]

- name: Run tests with flakeguard
shell: bash
id: run-tests
env:
GH_INPUTS_PROJECT_PATH: ${{ inputs.projectPath }}
GH_INPUTS_MAX_PASS_RATIO: ${{ inputs.maxPassRatio }}
CL_DATABASE_URL: ${{ env.DB_URL }}
run: flakeguard run --project-path=$GH_INPUTS_PROJECT_PATH --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --max-pass-ratio=$GH_INPUTS_MAX_PASS_RATIO --race=${{ env.RUN_WITH_RACE }} --shuffle=${{ env.RUN_WITH_SHUFFLE }} --shuffle-seed=${{ env.SHUFFLE_SEED }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json --omit-test-outputs-on-success=${{ env.OMIT_TEST_OUTPUTS_ON_SUCCESS }}
run: |
flakeguard run --project-path=$GH_INPUTS_PROJECT_PATH --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --max-pass-ratio=$GH_INPUTS_MAX_PASS_RATIO --race=${{ env.RUN_WITH_RACE }} --shuffle=${{ env.RUN_WITH_SHUFFLE }} --shuffle-seed=${{ env.SHUFFLE_SEED }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json --omit-test-outputs-on-success=${{ env.OMIT_TEST_OUTPUTS_ON_SUCCESS }}
# Output the status of the flakeguard run to files so that the next step can aggregate them and act accordingly
EXIT_CODE=$?
echo "$EXIT_CODE" > status_${GITHUB_JOB}.txt
if [ $EXIT_CODE -e 1 ]; then
echo "Found flaky tests"
elif [ $EXIT_CODE -e 2 ]; then
echo "ERROR: Flakeguard encountered an error while running tests"
echo "flakeguard_error=true" >> $GITHUB_OUTPUT
fi
exit $EXIT_CODE
- name: Upload test result as artifact
if: always()
Expand All @@ -363,6 +377,15 @@ jobs:
with:
persist-credentials: false
ref: ${{ env.GIT_HEAD_REF }}

- name: Check For Flakeguard Run Errors
id: check-errors
run: |
if ${{ needs.run-tests.outputs.flakeguard_error == true }}; then
echo "ERROR: Flakeguard encountered an error while running tests"
echo "ERROR: Flakeguard encountered an error while running tests" >> $GITHUB_STEP_SUMMARY
exit 1
fi
- name: Setup Go
uses: ./.github/actions/setup-go
with:
Expand All @@ -388,7 +411,7 @@ jobs:

- name: Install flakeguard
shell: bash
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@32d2999ffd11edff77f5251f9a1510803b8652ae # [email protected]
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@051b483f1e9e99b0219d1ccea4d8f7bf3be203b7 # [email protected]

- name: Aggregate Flakeguard Results
id: results
Expand All @@ -397,14 +420,14 @@ jobs:
GH_INPUTS_REPO_URL: ${{ inputs.repoUrl }}
GH_INPUTS_MAX_PASS_RATIO: ${{ inputs.maxPassRatio }}
run: |
set -e # Exit immediately if a command exits with a non-zero status.
# Create test results folder if it doesn't exist
mkdir -p ci_test_results
# Fix flakeguard binary path
PATH=$PATH:$(go env GOPATH)/bin
export PATH

current_branch_name="${{ github.head_ref || github.ref_name }} "

# Aggregate Flakeguard test results
flakeguard aggregate-results \
Expand All @@ -414,13 +437,20 @@ jobs:
--codeowners-path "${{ github.workspace }}/.github/CODEOWNERS" \
--max-pass-ratio "$GH_INPUTS_MAX_PASS_RATIO" \
--repo-url "$GH_INPUTS_REPO_URL" \
--branch-name "$current_branch_name" \
--base-sha "${{ needs.get-tests.outputs.git_base_sha }}" \
--head-sha "${{ needs.get-tests.outputs.git_head_sha }}" \
--github-workflow-name "${{ github.workflow }}" \
--github-workflow-run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
--splunk-url "${{ secrets.FLAKEGUARD_SPLUNK_ENDPOINT }}" \
--splunk-token "${{ secrets.FLAKEGUARD_SPLUNK_HEC }}" \
--splunk-event "${{ github.event_name }}"
EXIT_CODE=$?
if [ $EXIT_CODE -e 2 ]; then
echo "ERROR: Flakeguard encountered an error while aggregating results"
echo "ERROR: Flakeguard encountered an error while aggregating results" >> $GITHUB_STEP_SUMMARY
exit $EXIT_CODE
fi

# Print out the summary file
echo -e "\nFlakeguard Summary:"
Expand Down Expand Up @@ -457,15 +487,14 @@ jobs:
- name: Generate Flakeguard Reports
shell: bash
if: success() || failure()
id: generate-report
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
GH_INPUTS_MAX_PASS_RATIO: ${{ inputs.maxPassRatio }}
GH_EVENT_NAME: ${{ github.event_name }}
GH_EVENT_PULL_REQUEST_BASE_REF: ${{ github.event.pull_request.base.ref }}
GH_EVENT_PULL_REQUEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
set -e # Exit immediately if a command exits with a non-zero status.
# Fix flakeguard binary path
PATH=$PATH:$(go env GOPATH)/bin
export PATH
Expand Down Expand Up @@ -501,6 +530,12 @@ jobs:
--action-run-id "${{ github.run_id }}" \
--max-pass-ratio "$GH_INPUTS_MAX_PASS_RATIO"
fi
EXIT_CODE=$?
if [ $EXIT_CODE -e 2 ]; then
echo "ERROR: Flakeguard encountered an error while generating reports"
echo "ERROR: Flakeguard encountered an error while generating reports" >> $GITHUB_STEP_SUMMARY
exit $EXIT_CODE
fi

- name: Add Github Summary
if: (success() || failure())
Expand Down Expand Up @@ -624,4 +659,36 @@ jobs:
]
}
]
}
}
- name: Send Slack message for Flakeguard Errors
if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && (steps.check-errors.conclusion == 'failure' || steps.results.conclusion == 'failure' || steps.generate-report.conclusion == 'failure') }}
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
with:
channel-id: ${{ inputs.slackNotificationAfterTestsChannelId }}
payload: |
{
"attachments": [
{
"color": "#C62828",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Flakeguard Encountered an Unrecoverable Error :x:"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "<@U01Q4N37KFG> <@U04DYU1KLGJ> ${{ format('<{0}/{1}/actions/runs/{2}|See details>', github.server_url, github.repository, github.run_id) }} and diagnose the issue."
}
}
]
}
]
}

0 comments on commit 94076ad

Please sign in to comment.