diff --git a/.github/workflows/run_testsuite.yml b/.github/workflows/run_testsuite.yml deleted file mode 100644 index 37fc08525..000000000 --- a/.github/workflows/run_testsuite.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Run Testsuite (manual) # Unfortunately, we cannot modify the name: https://github.community/t/github-actions-dynamic-name-of-the-workflow-with-workflow-dispatch/150327 -on: - workflow_dispatch: - inputs: - testsuites: - description: 'Virtual Environment' - default: 'openfoam-adapter-release' - required: true - type: choice - options: - - openfoam-adapter-release - - openfoam-adapter-pr - - fenics-test - build_args: - description: 'Build arguments' - default: 'OPENFOAM_EXECUTABLE:openfoam2212,PRECICE_TAG:latest,OPENFOAM_ADAPTER_REF:master' - required: true - type: choice - options: - - 'OPENFOAM_EXECUTABLE:openfoam2212,PRECICE_TAG:latest,OPENFOAM_ADAPTER_REF:master' - - 'OPENFOAM_EXECUTABLE:openfoam2112,PRECICE_TAG:latest,OPENFOAM_ADAPTER_REF:master' - - 'OPENFOAM_EXECUTABLE:openfoam2212,PRECICE_TAG:develop,OPENFOAM_ADAPTER_REF:develop' - tutorial_branch: - description: 'Branch to take the tutorials from' - default: 'master' - required: true - type: choice - options: - - 'develop' - - 'master' - systests_branch: - description: 'Branch to take the systest from' - default: 'add-systemtests' - required: true - type: choice - options: - - 'develop' - - 'master' - - 'add-systemtests' -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Report log - run: | - echo "Initiated by: ${{ github.actor }}" - echo "Running systemtests --build_args=${{github.event.inputs.build_args}} --suites=${{github.event.inputs.testsuites}}" - echo "Tutorials branch: ${{ github.event.inputs.tutorial_branch }}" - echo "Systemtests branch: ${{ github.event.inputs.systests_branch }}" - - name: Check out Tutorials - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.tutorial_branch }} - path: 'tutorials' - - name: Check out Tutorials for systest - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.systests_branch }} - path: 'systest' - - name: Log directory - run: | - ls -al - pwd - - name: Copy folders and files from systest into tutorials - run: | - cp -R systest/tools/* tutorials/tools - cd systest - find . -type f -name "metadata.yaml" -exec cp --parents {} ../tutorials \; - cd .. - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: install python depencies - run: | - python -m pip install -r tutorials/tools/tests/requirements.txt - - name: Run tests - run: | - cd tutorials/tools/tests - python systemtests.py --build_args=${{github.event.inputs.build_args}} --suites=${{github.event.inputs.testsuites}} - cd ../../ - - name: Archive run files - uses: actions/upload-artifact@v2 - with: - name: runs - path: | - tutorials/runs/* \ No newline at end of file diff --git a/.github/workflows/run_testsuite_manual.yml b/.github/workflows/run_testsuite_manual.yml new file mode 100644 index 000000000..54781d30b --- /dev/null +++ b/.github/workflows/run_testsuite_manual.yml @@ -0,0 +1,36 @@ +name: Run Testsuite (manual) # Unfortunately, we cannot modify the name: https://github.community/t/github-actions-dynamic-name-of-the-workflow-with-workflow-dispatch/150327 +on: + workflow_dispatch: + inputs: + suites: + description: 'Comma seperated testsuites to execute' + required: true + type: string + build_args: + description: 'Build arguments, if not specified defaults will be taken' + required: false + type: string + systests_branch: + description: 'Branch to take the systest from' + default: 'develop' + required: true + type: string + loglevel: + description: 'loglevel used for the systemtests' + default: 'INFO' + required: true + type: choice + options: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' +jobs: + run_testsuite_manual: + uses: ./.github/workflows/run_testsuite_workflow.yml + with: + suites: ${{ inputs.suites }} + build_args: ${{ inputs.build_args }} + systests_branch: ${{ inputs.systests_branch }} + loglevel: ${{ inputs.loglevel }} \ No newline at end of file diff --git a/.github/workflows/run_testsuite_workflow.yml b/.github/workflows/run_testsuite_workflow.yml new file mode 100644 index 000000000..2e2781270 --- /dev/null +++ b/.github/workflows/run_testsuite_workflow.yml @@ -0,0 +1,63 @@ +name: Run Testsuite Workflow +on: + workflow_call: + inputs: + suites: + description: 'Comma seperated testsuites to execute' + required: true + type: string + build_args: + description: 'Build arguments' + required: false + type: string + systests_branch: + description: 'Branch to take the systest from' + default: 'develop' + required: true + type: string + loglevel: + description: 'loglevel used for the systemtests' + default: 'INFO' + required: false + type: string +jobs: + run_testsuite: + runs-on: ubuntu-latest + steps: + - name: Report log + run: | + echo "Initiated by: ${{ github.actor }}" + echo "Running systemtests --build_args=${{github.event.inputs.build_args}} --suites=${{github.event.inputs.suites}}" + echo "Systemtests branch: ${{ github.event.inputs.systests_branch }}" + - name: Check out Tutorials for systest + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.systests_branch }} + lfs: true + fetch-depth: 0 + - name: Log directory + run: | + ls -al + pwd + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: install python depencies + run: | + python -m pip install -r ./tools/tests/requirements.txt + - name: Expose GitHub Runtime (needed for accessing the gha cache) + uses: crazy-max/ghaction-github-runtime@v3 + - name: Run tests + run: | + cd ./tools/tests + python systemtests.py --build_args=${{github.event.inputs.build_args}} --suites=${{github.event.inputs.suites}} --log-level=${{github.event.inputs.loglevel}} + cd ../../ + - name: Archive run files + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: runs + path: | + runs/* \ No newline at end of file diff --git a/tools/tests/README.md b/tools/tests/README.md index 93c7b44b4..64a2c120d 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -19,6 +19,22 @@ python3 systemtests.py --suites=openfoam-adapter-release, To discover all tests, use `python print_test_suites.py`. To be able to fill in the right case tuple into the `tests.yaml`, you can use the `python3 print_case_combinations.py` script. +## Adding new tests + +### Adding tutorials + +In order for the systemtests to pick up the tutorial we need to define a `metadata.yaml` in the folder of the tutorial. There are a few `metadata.yaml` already present to get inspiration from. You can also have a look at the implementation details but normally the currently available ones should be easy to adopt. You can check your metadata parsing by `python print_metadata.py` and `python print_case_combinations.py` + +### Adding Testsuites + +To add a testsuite just open the `tests.yaml` file and use the output of `python print_case_combinations.py` to add the right case combinations you want to test. Note that you can specify a `reference_result` which is not yet present. The `generate_reference_data.py` will pick that up and create it for you. +Note that its important to carefully check the paths of the `reference_result` in order to not have typos in there. Also note that same cases in different testsuites should use the same `reference_result`. + +### Generate reference results + +Since we need data to compare against, you need to run `python generate_reference_data.py`. This process might take a while. +Please include the generated reference results in the pull request as they are strongly connected to the new testsuites. + ## Implementation details Each tutorial contains automation scripts (mainly `run.sh` and `clean.sh`), as well as metadata (`metadata.yaml`). The metadata file describes the available cases, how to run them, as well as their dependencies. A central `tests.yaml` file in this directory defines test suites, which execute different combinations of cases. The Python script `systemtests.py` executes the tests, allowing to filter for specific components or test suites. diff --git a/tools/tests/component-templates/calculix-adapter.yaml b/tools/tests/component-templates/calculix-adapter.yaml index 038a088b7..373c6e0f1 100644 --- a/tools/tests/component-templates/calculix-adapter.yaml +++ b/tools/tests/component-templates/calculix-adapter.yaml @@ -5,6 +5,10 @@ build: - {{key}}={{value}} {% endfor %} target: calculix_adapter + cache_from: + - type=gha + cache_to: + - type=gha,mode=min,scope=calculix_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/component-templates/fenics-adapter.yaml b/tools/tests/component-templates/fenics-adapter.yaml index b83a23707..52177d719 100644 --- a/tools/tests/component-templates/fenics-adapter.yaml +++ b/tools/tests/component-templates/fenics-adapter.yaml @@ -5,6 +5,10 @@ build: - {{key}}={{value}} {% endfor %} target: fenics_adapter + cache_from: + - type=gha + cache_to: + - type=gha,mode=min,scope=fenics_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/component-templates/nutils-adapter.yaml b/tools/tests/component-templates/nutils-adapter.yaml index 9963028ab..ee98f1ec1 100644 --- a/tools/tests/component-templates/nutils-adapter.yaml +++ b/tools/tests/component-templates/nutils-adapter.yaml @@ -5,6 +5,10 @@ build: - {{key}}={{value}} {% endfor %} target: nutils_adapter + cache_from: + - type=gha + cache_to: + - type=gha,mode=min,scope=nutils_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/component-templates/openfoam-adapter.yaml b/tools/tests/component-templates/openfoam-adapter.yaml index 4657b6e23..e3821247a 100644 --- a/tools/tests/component-templates/openfoam-adapter.yaml +++ b/tools/tests/component-templates/openfoam-adapter.yaml @@ -5,6 +5,10 @@ build: - {{key}}={{value}} {% endfor %} target: openfoam_adapter + cache_from: + - type=gha + cache_to: + - type=gha,mode=min,scope=openfoam_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 21d35dfea..4e86de2e7 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -7,3 +7,7 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} + - "-rtol 3e-7" + +# Currently its really hard to estimate the impact of compiling and executing in a different platform (like github actions) +# 3e-7 might not be the thightest we can afford and we want to have but its an okayish guestimation for now. \ No newline at end of file diff --git a/tools/tests/docker-compose.template.yaml b/tools/tests/docker-compose.template.yaml index ff6f7d279..2df2c144f 100644 --- a/tools/tests/docker-compose.template.yaml +++ b/tools/tests/docker-compose.template.yaml @@ -4,9 +4,16 @@ services: build: context: {{ dockerfile_context }} target: base_image + cache_from: + - type=gha + cache_to: + - type=gha,mode=min,scope=prepare + args: + {% for key, value in build_arguments.items() %} + - {{key}}={{value}} + {% endfor %} volumes: - {{ run_directory }}:/runs - command: > /bin/bash -c "id && cd '/runs/{{ tutorial_folder }}' && @@ -18,4 +25,4 @@ services: {% for service in services %} {{ service }}: {{ services[service] |indent(4) }} - {% endfor %} \ No newline at end of file + {% endfor %} diff --git a/tools/tests/dockerfiles/ubuntu_2204/Dockerfile b/tools/tests/dockerfiles/ubuntu_2204/Dockerfile index 3f34db82e..e31e911a1 100644 --- a/tools/tests/dockerfiles/ubuntu_2204/Dockerfile +++ b/tools/tests/dockerfiles/ubuntu_2204/Dockerfile @@ -2,8 +2,13 @@ FROM ubuntu:22.04 as base_image USER root SHELL ["/bin/bash", "-c"] ENV DEBIAN_FRONTEND=noninteractive - -RUN useradd -ms /bin/bash precice +# We set a sensical value, but still have the possibilty to influence this via the build time arguments. +# When the dockerfile is built using the systemtests.py we set the PRECICE_UID and PRECICE_GID to the user executing the systemtests. +# This ensures no file ownership problems down the line and is the most easy fix, as we normally built the containers locally +# If not built via the systemtests.py its either possible to specify manually but 1000 would be the default anyway. +ARG PRECICE_UID=1000 +ARG PRECICE_GID=1000 +RUN groupadd -g ${PRECICE_GID} precice && useradd -u ${PRECICE_UID} -g ${PRECICE_GID} -ms /bin/bash precice ENV PATH="${PATH}:/home/precice/.local/bin" ENV LD_LIBRARY_PATH="/home/precice/.local/lib:${LD_LIBRARY_PATH}" ENV CPATH="/home/precice/.local/include:$CPATH" diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 33056c7bc..b8dae1d6a 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -16,6 +16,7 @@ import unicodedata import re import logging +import os GLOBAL_TIMEOUT = 360 @@ -194,6 +195,7 @@ def __get_docker_compose_file(self): 'tutorial_folder': self.tutorial_folder, 'tutorial': self.tutorial.path.name, 'services': rendered_services, + 'build_arguments': self.params_to_use, 'dockerfile_context': self.dockerfile_context, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, } @@ -217,7 +219,7 @@ def _get_git_ref(self, repository: Path, abbrev_ref=False) -> Optional[str]: try: result = subprocess.run([ "git", - "-C", repository.resolve(), + "-C", os.fspath(repository.resolve()), "rev-parse", "--abbrev-ref" if abbrev_ref else "HEAD"], stdout=subprocess.PIPE, @@ -227,13 +229,28 @@ def _get_git_ref(self, repository: Path, abbrev_ref=False) -> Optional[str]: except Exception as e: raise RuntimeError(f"An error occurred while getting the current Git ref: {e}") from e + def _fetch_ref(self, repository: Path, ref: str): + try: + result = subprocess.run([ + "git", + "-C", os.fspath(repository.resolve()), + "fetch", + "origin", + f"{ref}:{ref}" + ], check=True, timeout=60) + if result.returncode != 0: + raise RuntimeError(f"git command returned code {result.returncode}") + + except Exception as e: + raise RuntimeError(f"An error occurred while fetching origin '{ref}': {e}") + def _checkout_ref_in_subfolder(self, repository: Path, subfolder: Path, ref: str): try: result = subprocess.run([ "git", - "-C", repository.resolve(), + "-C", os.fspath(repository.resolve()), "checkout", ref, - "--", subfolder.resolve() + "--", os.fspath(subfolder.resolve()) ], check=True, timeout=60) if result.returncode != 0: raise RuntimeError(f"git command returned code {result.returncode}") @@ -251,6 +268,7 @@ def __copy_tutorial_into_directory(self, run_directory: Path): ref_requested = self.params_to_use.get("TUTORIALS_REF") if ref_requested: logging.debug(f"Checking out tutorials {ref_requested} before copying") + self._fetch_ref(PRECICE_TUTORIAL_DIR, ref_requested) self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, ref_requested) self.tutorial_folder = slugify(f'{self.tutorial.path.name}_{self.case_combination.cases}_{current_time_string}') @@ -461,10 +479,9 @@ def __prepare_for_run(self, run_directory: Path): self.__copy_tutorial_into_directory(run_directory) self.__copy_tools(run_directory) self.__put_gitignore(run_directory) - uid, gid = self.__get_uid_gid() - self.env["UID"] = uid - self.env["GID"] = gid - self.__write_env_file() + host_uid, host_gid = self.__get_uid_gid() + self.params_to_use['PRECICE_UID'] = host_uid + self.params_to_use['PRECICE_GID'] = host_gid def run(self, run_directory: Path): """