diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/ci.yml similarity index 53% rename from .github/workflows/unit-tests.yml rename to .github/workflows/ci.yml index f91c5bcf5..61b45deb2 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/ci.yml @@ -1,34 +1,51 @@ -name: Unit Tests +name: Continuous Integration on: workflow_dispatch: null push: pull_request: jobs: + docker-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag linode/cli:$(date +%s) --build-arg="github_token=$GITHUB_TOKEN" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + lint: + runs-on: ubuntu-latest + steps: + - name: checkout repo + uses: actions/checkout@v4 + + - name: setup python 3 + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: install dependencies + run: make install + + - name: run linter + run: make lint + unit-tests-on-ubuntu: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.9','3.10','3.11', '3.12' ] + python-version: [ "3.9","3.10","3.11", "3.12", "3.13" ] steps: - name: Clone Repository uses: actions/checkout@v4 - - name: Update system packages - run: sudo apt-get update -y - - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install Python wheel - run: pip install wheel boto3 - - - name: Update cert - run: pip install certifi -U - - - name: Install deps - run: pip install .[dev] + - name: Install Python dependencies + run: pip install -U certifi - name: Install Package run: make install @@ -47,16 +64,10 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.x' - - - name: Install Python wheel - run: pip install wheel boto3 - - - name: Update cert - run: pip install certifi -U + python-version: "3.x" - - name: Install deps - run: pip install .[dev] + - name: Install Python dependencies + run: pip install -U certifi - name: Install Package shell: pwsh diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..e2ccc10da --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,37 @@ +name: "CodeQL Advanced" + +on: + push: + branches: [ "dev", "main" ] + pull_request: + branches: [ "dev", "main" ] + schedule: + - cron: "0 13 * * 5" + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + permissions: + security-events: write + + strategy: + fail-fast: false + matrix: + include: + - language: python + build-mode: none + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 000000000..bf9f46d87 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,19 @@ +name: 'Dependency review' +on: + pull_request: + branches: [ "dev", "main", "proj/*" ] + +permissions: + contents: read + pull-requests: write + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout repository' + uses: actions/checkout@v4 + - name: 'Dependency Review' + uses: actions/dependency-review-action@v4 + with: + comment-summary-in-pr: on-failure diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml deleted file mode 100644 index 5de34ac4a..000000000 --- a/.github/workflows/docker-build.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Docker Image Build CI - -on: - push: - branches: [ "dev", "main" ] - pull_request: - branches: [ "dev", "main" ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Build the Docker image - run: docker build . --file Dockerfile --tag linode/cli:$(date +%s) --build-arg="github_token=$GITHUB_TOKEN" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/e2e-suite-windows.yml b/.github/workflows/e2e-suite-windows.yml index 0c347f748..c3e030a6c 100644 --- a/.github/workflows/e2e-suite-windows.yml +++ b/.github/workflows/e2e-suite-windows.yml @@ -73,7 +73,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" testint + - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" test-int env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN_2 }} diff --git a/.github/workflows/e2e-suite.yml b/.github/workflows/e2e-suite.yml index e58b1fde5..7c73e3209 100644 --- a/.github/workflows/e2e-suite.yml +++ b/.github/workflows/e2e-suite.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: inputs: use_minimal_test_account: - description: 'Use minimal test account' + description: 'Indicate whether to use a minimal test account with limited resources for testing. Defaults to "false"' required: false default: 'false' - module: - description: "The module from 'test/integration' to the target to be tested, e.g. 'cli, domains, events, etc'" + test_suite: + description: "Specify test suite to run from the 'tests/integration' directory. Examples: 'cli', 'domains', 'events', etc. If not provided, all suites are executed" required: false run_long_tests: - description: "Select True to run long tests, e.g. database, rebuild, etc" + description: "Select 'True' to include long-running tests (e.g., database provisioning, server rebuilds). Defaults to 'False'" required: false type: choice options: @@ -19,21 +19,21 @@ on: - "False" default: "False" sha: - description: 'The hash value of the commit.' + description: 'Specify commit hash to test. This value is mandatory to ensure the tests run against a specific commit' required: true default: '' pull_request_number: - description: 'The number of the PR. Ensure sha value is provided' + description: 'Specify pull request number associated with the commit. Optional, but recommended when providing a commit hash (sha)' required: false openapi_spec_url: - description: 'URL of the OpenAPI spec to use for the tests' + description: 'Specify URL of the OpenAPI specification file to use for testing. Useful for validating tests against a specific API version or custom specification' required: false default: '' python-version: - description: 'Specify Python version to use' + description: 'Specify the Python version to use for running tests. Leave empty to use the default Python version configured in the environment' required: false run-eol-python-version: - description: 'Run EOL python version?' + description: 'Indicates whether to run tests using an End-of-Life (EOL) Python version. Defaults to "false". Choose "true" to include tests for deprecated Python versions' required: false default: 'false' type: choice @@ -124,24 +124,18 @@ jobs: run: | timestamp=$(date +'%Y%m%d%H%M') report_filename="${timestamp}_cli_test_report.xml" - make testint TEST_ARGS="--junitxml=${report_filename}" MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" + make test-int TEST_ARGS="--junitxml=${report_filename}" TEST_SUITE="${{ inputs.test_suite }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" env: LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }} - - name: Upload test results + - name: Upload Test Report as Artifact if: always() - run: | - filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$') - python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \ - --branch_name "${GITHUB_REF#refs/*/}" \ - --gha_run_id "$GITHUB_RUN_ID" \ - --gha_run_number "$GITHUB_RUN_NUMBER" \ - --xmlfile "${filename}" - sync - python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}" - env: - LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} - LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + uses: actions/upload-artifact@v4 + with: + name: test-report-file + if-no-files-found: ignore + path: '*.xml' + retention-days: 1 - name: Update PR Check Run uses: actions/github-script@v7 @@ -237,6 +231,51 @@ jobs: env: LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }} + process-upload-report: + runs-on: ubuntu-latest + needs: [integration_tests] + if: always() && github.repository == 'linode/linode-cli' # Run even if integration tests fail and only on main repository + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Download test report + uses: actions/download-artifact@v4 + with: + name: test-report-file + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: pip3 install requests wheel boto3==1.35.99 + + - name: Set release version env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + + - name: Add variables and upload test results + if: always() + run: | + filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$') + python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \ + --branch_name "${GITHUB_REF#refs/*/}" \ + --gha_run_id "$GITHUB_RUN_ID" \ + --gha_run_number "$GITHUB_RUN_NUMBER" \ + --xmlfile "${filename}" + sync + python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}" + env: + LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} + LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + + notify-slack: runs-on: ubuntu-latest needs: [integration_tests] diff --git a/.github/workflows/nightly-smoke-tests.yml b/.github/workflows/nightly-smoke-tests.yml index 10cbad35d..372de28c4 100644 --- a/.github/workflows/nightly-smoke-tests.yml +++ b/.github/workflows/nightly-smoke-tests.yml @@ -40,7 +40,7 @@ jobs: - name: Run smoke tests id: smoke_tests run: | - make smoketest + make test-smoke env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml deleted file mode 100644 index 46f3414aa..000000000 --- a/.github/workflows/publish-pypi.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: release -on: - workflow_dispatch: null - release: - types: [ published ] -jobs: - pypi-release: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Update system packages - run: sudo apt-get update -y - - - name: Install make - run: sudo apt-get install -y build-essential - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Python deps - run: pip install wheel - - - name: Install package requirements - run: make requirements - - - name: Build the package - run: make build - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LINODE_CLI_VERSION: ${{ github.event.release.tag_name }} - - - name: Publish the release artifacts to PyPI - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # pin@release/v1.12.3 - with: - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml deleted file mode 100644 index 3c14f367a..000000000 --- a/.github/workflows/pull-request.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Pull Request Actions -on: - pull_request: null - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: checkout repo - uses: actions/checkout@v4 - - - name: setup python 3 - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: install boto3 - run: pip3 install boto3 - - name: install dependencies - run: pip install .[obj,dev] - - - name: run linter - run: make lint diff --git a/.github/workflows/release-notify-slack.yml b/.github/workflows/release-notify-slack.yml deleted file mode 100644 index 9c8019534..000000000 --- a/.github/workflows/release-notify-slack.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Notify Dev DX Channel on Release -on: - release: - types: [published] - workflow_dispatch: null - -jobs: - notify: - if: github.repository == 'linode/linode-cli' - runs-on: ubuntu-latest - steps: - - name: Notify Slack - Main Message - id: main_message - uses: slackapi/slack-github-action@v2.0.0 - with: - method: chat.postMessage - token: ${{ secrets.SLACK_BOT_TOKEN }} - payload: | - channel: ${{ secrets.CLI_SLACK_CHANNEL_ID }} - blocks: - - type: section - text: - type: mrkdwn - text: "*New Release Published: _linode-cli_ <${{ github.event.release.html_url }}|${{ github.event.release.tag_name }}> is now live!* :tada:" diff --git a/.github/workflows/publish-oci.yml b/.github/workflows/release.yml similarity index 56% rename from .github/workflows/publish-oci.yml rename to .github/workflows/release.yml index 7e86b4d6b..980393de9 100644 --- a/.github/workflows/publish-oci.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,28 @@ -name: OCI Image Publish +name: Release on: workflow_dispatch: null release: types: [ published ] jobs: + notify: + needs: pypi-release + if: github.repository == 'linode/linode-cli' + runs-on: ubuntu-latest + steps: + - name: Notify Slack - Main Message + id: main_message + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: ${{ secrets.CLI_SLACK_CHANNEL_ID }} + blocks: + - type: section + text: + type: mrkdwn + text: "*New Release Published: _linode-cli_ <${{ github.event.release.html_url }}|${{ github.event.release.tag_name }}> is now live!* :tada:" + oci_publish: name: Build and publish the OCI image runs-on: ubuntu-latest @@ -58,3 +77,31 @@ jobs: build-args: | linode_cli_version=${{ steps.cli_version.outputs.result }} github_token=${{ secrets.GITHUB_TOKEN }} + + pypi-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python deps + run: pip install wheel + + - name: Install package requirements + run: make requirements + + - name: Build the package + run: make build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LINODE_CLI_VERSION: ${{ github.event.release.tag_name }} + + - name: Publish the release artifacts to PyPI + uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # pin@release/v1.12.3 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/remote-release-trigger.yml b/.github/workflows/remote-release-trigger.yml index b345df278..6806ed093 100644 --- a/.github/workflows/remote-release-trigger.yml +++ b/.github/workflows/remote-release-trigger.yml @@ -66,7 +66,7 @@ jobs: commit_sha: ${{ steps.calculate_head_sha.outputs.commit_sha }} - name: Release - uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 # pin@v2.0.9 + uses: softprops/action-gh-release@7b4da11513bf3f43f9999e90eabced41ab8bb048 # pin@v2.2.0 with: target_commitish: 'main' token: ${{ steps.generate_token.outputs.token }} diff --git a/Makefile b/Makefile index 3f9c6682b..27eb93550 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,6 @@ # Makefile for more convenient building of the Linode CLI and its baked content # -# Test-related arguments -MODULE := -TEST_CASE_COMMAND := -TEST_ARGS := - -ifdef TEST_CASE -TEST_CASE_COMMAND = -k $(TEST_CASE) -endif - SPEC_VERSION ?= latest ifndef SPEC override SPEC = $(shell ./resolve_spec_url ${SPEC_VERSION}) @@ -44,7 +35,7 @@ build: clean create-version bake .PHONY: requirements requirements: - pip3 install --upgrade .[dev,obj] + pip3 install --upgrade ".[dev,obj]" .PHONY: lint lint: build @@ -66,8 +57,8 @@ clean: rm -f data-* rm -rf dist linode_cli.egg-info build -.PHONY: testunit -testunit: +.PHONY: test-unit +test-unit: @mkdir -p /tmp/linode/.config @orig_xdg_config_home=$${XDG_CONFIG_HOME:-}; \ export LINODE_CLI_TEST_MODE=1 XDG_CONFIG_HOME=/tmp/linode/.config; \ @@ -76,9 +67,14 @@ testunit: export XDG_CONFIG_HOME=$$orig_xdg_config_home; \ exit $$exit_code -.PHONY: testint -testint: - pytest tests/integration/${MODULE} ${TEST_CASE_COMMAND} ${TEST_ARGS} +# Integration Test Arguments +# TEST_SUITE: Optional, specify a test suite (e.g. domains), Default to run everything if not set +# TEST_CASE: Optional, specify a test case (e.g. 'test_create_a_domain') +# TEST_ARGS: Optional, additional arguments for pytest (e.g. '-v' for verbose mode) + +.PHONY: test-int +test-int: + pytest tests/integration/$(TEST_SUITE) $(if $(TEST_CASE),-k $(TEST_CASE)) $(TEST_ARGS) .PHONY: testall testall: @@ -86,7 +82,7 @@ testall: # Alias for unit; integration tests should be explicit .PHONY: test -test: testunit +test: test-unit .PHONY: black black: @@ -103,6 +99,6 @@ autoflake: .PHONY: format format: black isort autoflake -@PHONEY: smoketest -smoketest: +@PHONEY: test-smoke +test-smoke: pytest -m smoke tests/integration diff --git a/linodecli/configuration/config.py b/linodecli/configuration/config.py index ea55da732..de2dca86f 100644 --- a/linodecli/configuration/config.py +++ b/linodecli/configuration/config.py @@ -397,16 +397,21 @@ def configure( print(f"\nConfiguring {username}\n") # Configuring Defaults - regions = [ - r["id"] for r in _do_get_request(self.base_url, "/regions")["data"] - ] - types = [ - t["id"] - for t in _do_get_request(self.base_url, "/linode/types")["data"] - ] - images = [ - i["id"] for i in _do_get_request(self.base_url, "/images")["data"] - ] + regions = sorted( + [ + r["id"] + for r in _do_get_request(self.base_url, "/regions")["data"] + ] + ) + types = sorted( + [ + t["id"] + for t in _do_get_request(self.base_url, "/linode/types")["data"] + ] + ) + images = sorted( + [i["id"] for i in _do_get_request(self.base_url, "/images")["data"]] + ) is_full_access = _check_full_access(self.base_url, token) @@ -423,9 +428,9 @@ def configure( ) if "data" in users: - auth_users = [ - u["username"] for u in users["data"] if "ssh_keys" in u - ] + auth_users = sorted( + [u["username"] for u in users["data"] if "ssh_keys" in u] + ) # get the preferred things config["region"] = _default_thing_input( diff --git a/linodecli/plugins/obj/__init__.py b/linodecli/plugins/obj/__init__.py index 9e613ff7a..9fd981b65 100644 --- a/linodecli/plugins/obj/__init__.py +++ b/linodecli/plugins/obj/__init__.py @@ -56,6 +56,7 @@ try: import boto3 + from botocore.config import Config from botocore.exceptions import ClientError HAS_BOTO = True @@ -443,6 +444,16 @@ def _get_boto_client(cluster, access_key, secret_key): aws_secret_access_key=secret_key, region_name=cluster, endpoint_url=BASE_URL_TEMPLATE.format(cluster), + config=Config( + # This addresses an incompatibility between boto3 1.36.x and + # some third-party S3-compatible storage platforms. + # In the future we may want to consider manually computing the + # CRC32 hash of a file before uploading it. + # + # See: https://github.com/boto/boto3/issues/4398#issuecomment-2619946229 + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), ) # set this for later use diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a0f283314..5e5ab8a96 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -186,7 +186,7 @@ def _generate_test_file( @pytest.fixture def generate_test_files( - generate_test_file: Callable[[Optional[str], Optional[str]], Path] + generate_test_file: Callable[[Optional[str], Optional[str]], Path], ): """ Return a function that can generate files with random text. diff --git a/tests/integration/image/test_plugin_image_upload.py b/tests/integration/image/test_plugin_image_upload.py index c81621104..3174986b5 100644 --- a/tests/integration/image/test_plugin_image_upload.py +++ b/tests/integration/image/test_plugin_image_upload.py @@ -19,8 +19,8 @@ # A minimal gzipped image that will be accepted by the API TEST_IMAGE_CONTENT = ( - b"\x1F\x8B\x08\x08\xBD\x5C\x91\x60\x00\x03\x74\x65\x73\x74\x2E\x69" - b"\x6D\x67\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x1f\x8b\x08\x08\xbd\x5c\x91\x60\x00\x03\x74\x65\x73\x74\x2e\x69" + b"\x6d\x67\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" )