diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..a6d9b4c6 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,5 @@ +# For more information, see: +# https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view + +# Black code formatting of repository. +50ff8f57feaff4471ce82274f34baafc3be9782f diff --git a/.gitignore b/.gitignore index ba13c090..e547f78a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ output/* .idea **/pulled-images.txt +**/.python-version diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..584e2082 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +ci: + autofix_prs: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.9 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.4.2 + hooks: + - id: black-jupyter + args: ["--skip-string-normalization"] + language_version: python3.11 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..9b74e70d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# CHANGELOG + +The harmony-regression-tests repository does not follow semantic +versioning. Rather than a static releases, this repository contains of a number +of regression tests that are each semi-independent. This CHANGELOG file should be used +to document pull requests to this repository. + + +## 2024-08-05 ([#86](https://github.com/nasa/harmony-regression-tests/pull/86)) + +Adds this file to capture changes to the repository. + +Adds pre-commit.ci behavior to the repository. This setup ensures consistent code style, catches common errors, and maintains file hygiene across the project. + +Updates the base image for all regression tests to `mambaorg/micromamba:1.5.8-jammy` diff --git a/README.md b/README.md index ad6e26b7..1928b243 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ UAT and Prod. This is the preferred method of verifying no regressons have occurred, when the services have been modified. Alternatively, each test can be run locally in a browser against SIT, UAT, PROD -or localhost (harmony-in-a-box). This is a good choice for test development and +or localhost (Harmony-In-A-Box). This is a good choice for test development and verifying service changes do not cause regression failures. Generally you run locally in the browser against a single service regression test. @@ -53,15 +53,18 @@ created Zarr store.* to run after the run_notebooks command. e.g. `./run_notebooks.sh hga n2z` would run the `harmony GDAL adapter` and `NetCDF-to-Zarr` regression tests.* -1. *`HARMONY_HOST_URL` is the harmony base url for your target environment. e.g. `SIT` would be `https://harmony.sit.earthdata.nasa.gov`* +1. *`HARMONY_HOST_URL` is the harmony base url for your target + environment. e.g. `SIT` would be `https://harmony.sit.earthdata.nasa.gov`* -1. The `run_notebooks.sh` script cannot be used to test against Harmony-in-a-Box, - i.e. `HARMONY_HOST_URL=http://localhost:3000`, due to Docker-in-Docker issues. - To test against a local Harmony instance, the notebook should be run - manually on a Jupyter notebook server (e.g., in a browser). +1. *The `run_notebooks.sh` script cannot be used to test against + Harmony-in-a-Box, i.e. `HARMONY_HOST_URL=http://localhost:3000`, due to + Docker-in-Docker issues. To test against a local Harmony instance, the + notebook should be run manually on a Jupyter notebook server (e.g., in a + browser).* + +For more information on running a local Harmony instance, see the [Harmony +README](https://github.com/nasa/harmony/blob/main/README.md). -For more information on running a local Harmony instance, see: -. ### Test in a Browser: @@ -248,3 +251,36 @@ if the new image is named `ghcr.io/nasa/regression-tests-foo`, then we would add The `run_notebooks.sh` file can be used as described above to run the test suite. Notebooks are expected to exit with a non-zero exit code on failure when run from `papermill`. + +## pre-commit hooks: + +This repository uses [pre-commit](https://pre-commit.com/) to enable pre-commit +checking the repository for some coding standard best practices. These include: + +* Removing trailing whitespaces. +* Removing blank lines at the end of a file. +* Ensure JSON files have valid formats. +* [ruff](https://github.com/astral-sh/ruff) Python linting checks. +* [black](https://black.readthedocs.io/en/stable/index.html) Python code + formatting checks. + +To enable these checks: + +```bash +# Install pre-commit Python package: +pip install pre-commit + +# Install the git hook scripts: +pre-commit install +``` + + +If you have installed the hooks locally, when you commit your changes the hook +will validate your changes before actually committing to your repository. If +there are failures you will have to opportunity to fix them and add them to +your commit. + +[pre-commit.ci](pre-commit.ci) is configured such that these same hooks will be +automatically run for every pull request. Because of this, it is highly +recommended that you also do this locally, since failures will prevent your PR +from being merged. diff --git a/test/Dockerfile b/test/Dockerfile index bdd6072a..d12023e7 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,6 +1,4 @@ -# Uses focal image because newer jammy image does not work with old docker versions -# see related information: https://github.com/adoptium/containers/issues/215#issuecomment-1142046045 -FROM mambaorg/micromamba:1.4.2-focal +FROM mambaorg/micromamba:1.5.8-jammy USER root ARG sub_dir diff --git a/test/geoloco/Geoloco_Regression.ipynb b/test/geoloco/Geoloco_Regression.ipynb index b2c9a584..0b563138 100644 --- a/test/geoloco/Geoloco_Regression.ipynb +++ b/test/geoloco/Geoloco_Regression.ipynb @@ -33,9 +33,16 @@ "\n", "from harmony import Client, Collection, Environment, Request\n", "\n", - "from utilities import (submit_and_download, get_dim_sizes, get_sds_data,\n", - " remove_results_files, print_error, print_success,\n", - " compare_dimensions, compare_data)" + "from utilities import (\n", + " submit_and_download,\n", + " get_dim_sizes,\n", + " get_sds_data,\n", + " remove_results_files,\n", + " print_error,\n", + " print_success,\n", + " compare_dimensions,\n", + " compare_data,\n", + ")" ] }, { @@ -89,10 +96,12 @@ "metadata": {}, "outputs": [], "source": [ - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", @@ -124,32 +133,42 @@ "metadata": {}, "outputs": [], "source": [ - "mod021km_non_production_info = {'collection' : Collection(id='C1256826282-LAADSCDUAT'),\n", - " 'granule_id' : 'G1259320275-LAADSCDUAT',\n", - " 'variable' : ['EV_250_Aggr1km_RefSB'],\n", - " 'downscale_size' : [0.01802, 0.01802]}\n", - "\n", - "mod35l2_non_production_info = {'collection' : Collection(id='C1257437479-LAADSCDUAT'),\n", - " 'granule_id' : 'G1261599141-LAADSCDUAT',\n", - " 'variable' : ['Cloud_Mask'],\n", - " 'downscale_size' : [0.01802, 0.01802]}\n", - "\n", - "mod08d3_non_production_info = {'collection' : Collection(id='C1257773477-LAADSCDUAT'),\n", - " 'granule_id' : 'G1259320277-LAADSCDUAT',\n", - " 'variable' : ['Aerosol_Optical_Depth_Land_Ocean_Mean'],\n", - " 'downscale_size' : [2, 2]}\n", + "mod021km_non_production_info = {\n", + " 'collection': Collection(id='C1256826282-LAADSCDUAT'),\n", + " 'granule_id': 'G1259320275-LAADSCDUAT',\n", + " 'variable': ['EV_250_Aggr1km_RefSB'],\n", + " 'downscale_size': [0.01802, 0.01802],\n", + "}\n", + "\n", + "mod35l2_non_production_info = {\n", + " 'collection': Collection(id='C1257437479-LAADSCDUAT'),\n", + " 'granule_id': 'G1261599141-LAADSCDUAT',\n", + " 'variable': ['Cloud_Mask'],\n", + " 'downscale_size': [0.01802, 0.01802],\n", + "}\n", + "\n", + "mod08d3_non_production_info = {\n", + " 'collection': Collection(id='C1257773477-LAADSCDUAT'),\n", + " 'granule_id': 'G1259320277-LAADSCDUAT',\n", + " 'variable': ['Aerosol_Optical_Depth_Land_Ocean_Mean'],\n", + " 'downscale_size': [2, 2],\n", + "}\n", "\n", "geo_proj4_string = '+a=6378137.0 +b=6356752.3142451793 +no_defs +proj=latlong'\n", "\n", "resampling_string = 'NN'\n", "\n", - "file_indicators = {'MOD021KM': 'EV_250_Aggr1km_RefSB_1.hdf',\n", - " 'MOD35_L2': 'Cloud_Mask_1.hdf',\n", - " 'MOD08_D3': 'Aerosol_Optical_Depth_Land_Ocean_Mean.hdf'}\n", - "\n", - "reference_data = {'MOD021KM': 'reference_data/MOD021KM.A2023001.0020.061.psrpcs_001701802061.EV_250_Aggr1km_RefSB_1.hdf',\n", - " 'MOD35_L2': 'reference_data/MOD35_L2.A2023001.0020.061.psrpcs_001701881013.Cloud_Mask_1.hdf',\n", - " 'MOD08_D3': 'reference_data/MOD08_D3.A2023001.061.psrpcs_001701881265.Aerosol_Optical_Depth_Land_Ocean_Mean.hdf'}" + "file_indicators = {\n", + " 'MOD021KM': 'EV_250_Aggr1km_RefSB_1.hdf',\n", + " 'MOD35_L2': 'Cloud_Mask_1.hdf',\n", + " 'MOD08_D3': 'Aerosol_Optical_Depth_Land_Ocean_Mean.hdf',\n", + "}\n", + "\n", + "reference_data = {\n", + " 'MOD021KM': 'reference_data/MOD021KM.A2023001.0020.061.psrpcs_001701802061.EV_250_Aggr1km_RefSB_1.hdf',\n", + " 'MOD35_L2': 'reference_data/MOD35_L2.A2023001.0020.061.psrpcs_001701881013.Cloud_Mask_1.hdf',\n", + " 'MOD08_D3': 'reference_data/MOD08_D3.A2023001.061.psrpcs_001701881265.Aerosol_Optical_Depth_Land_Ocean_Mean.hdf',\n", + "}" ] }, { @@ -167,28 +186,34 @@ "metadata": {}, "outputs": [], "source": [ - "mod021km_geoloco_env = {Environment.LOCAL: mod021km_non_production_info,\n", - " Environment.UAT: mod021km_non_production_info,\n", - " Environment.SIT: mod021km_non_production_info}\n", - "mod35l2_geoloco_env = {Environment.LOCAL: mod35l2_non_production_info,\n", - " Environment.UAT: mod35l2_non_production_info,\n", - " Environment.SIT: mod35l2_non_production_info}\n", - "mod08d3_geoloco_env = {Environment.LOCAL: mod08d3_non_production_info,\n", - " Environment.UAT: mod08d3_non_production_info,\n", - " Environment.SIT: mod08d3_non_production_info}\n", + "mod021km_geoloco_env = {\n", + " Environment.LOCAL: mod021km_non_production_info,\n", + " Environment.UAT: mod021km_non_production_info,\n", + " Environment.SIT: mod021km_non_production_info,\n", + "}\n", + "mod35l2_geoloco_env = {\n", + " Environment.LOCAL: mod35l2_non_production_info,\n", + " Environment.UAT: mod35l2_non_production_info,\n", + " Environment.SIT: mod35l2_non_production_info,\n", + "}\n", + "mod08d3_geoloco_env = {\n", + " Environment.LOCAL: mod08d3_non_production_info,\n", + " Environment.UAT: mod08d3_non_production_info,\n", + " Environment.SIT: mod08d3_non_production_info,\n", + "}\n", "\n", "if harmony_environment in mod021km_geoloco_env:\n", - " mod021km_geoloco_info = mod021km_geoloco_env[harmony_environment]\n", + " mod021km_geoloco_info = mod021km_geoloco_env[harmony_environment]\n", "else:\n", " mod021km_geoloco_info = None\n", "\n", "if harmony_environment in mod35l2_geoloco_env:\n", - " mod35l2_geoloco_info = mod35l2_geoloco_env[harmony_environment]\n", + " mod35l2_geoloco_info = mod35l2_geoloco_env[harmony_environment]\n", "else:\n", " mod35l2_geoloco_info = None\n", "\n", "if harmony_environment in mod08d3_geoloco_env:\n", - " mod08d3_geoloco_info = mod08d3_geoloco_env[harmony_environment]\n", + " mod08d3_geoloco_info = mod08d3_geoloco_env[harmony_environment]\n", "else:\n", " mod08d3_geoloco_info = None" ] @@ -210,39 +235,53 @@ "metadata": {}, "outputs": [], "source": [ - "if (mod021km_geoloco_info is not None and \n", - " mod35l2_geoloco_info is not None and\n", - " mod08d3_geoloco_info is not None):\n", - "\n", - " mod021km_request = Request(collection=mod021km_geoloco_info['collection'],\n", - " granule_id=mod021km_geoloco_info['granule_id'],\n", - " variables=mod021km_geoloco_info['variable'],\n", - " scale_size=mod021km_geoloco_info['downscale_size'],\n", - " crs=geo_proj4_string,\n", - " interpolation=resampling_string)\n", - "\n", - " mod35l2_request = Request(collection=mod35l2_geoloco_info['collection'],\n", - " granule_id=mod35l2_geoloco_info['granule_id'],\n", - " variables=mod35l2_geoloco_info['variable'],\n", - " scale_size=mod35l2_geoloco_info['downscale_size'],\n", - " crs=geo_proj4_string,\n", - " interpolation=resampling_string)\n", - "\n", - " mod08d3_request = Request(collection=mod08d3_geoloco_info['collection'],\n", - " granule_id=mod08d3_geoloco_info['granule_id'],\n", - " variables=mod08d3_geoloco_info['variable'],\n", - " scale_size=mod08d3_geoloco_info['downscale_size'],\n", - " crs=geo_proj4_string,\n", - " interpolation=resampling_string)\n", - "\n", - " mod021km_compare_file = submit_and_download(harmony_client, mod021km_request, file_indicators['MOD021KM'])\n", - " mod35l2_compare_file = submit_and_download(harmony_client, mod35l2_request, file_indicators['MOD35_L2'])\n", - " mod08d3_compare_file = submit_and_download(harmony_client, mod08d3_request, file_indicators['MOD08_D3'])\n", + "if (\n", + " mod021km_geoloco_info is not None\n", + " and mod35l2_geoloco_info is not None\n", + " and mod08d3_geoloco_info is not None\n", + "):\n", + "\n", + " mod021km_request = Request(\n", + " collection=mod021km_geoloco_info['collection'],\n", + " granule_id=mod021km_geoloco_info['granule_id'],\n", + " variables=mod021km_geoloco_info['variable'],\n", + " scale_size=mod021km_geoloco_info['downscale_size'],\n", + " crs=geo_proj4_string,\n", + " interpolation=resampling_string,\n", + " )\n", + "\n", + " mod35l2_request = Request(\n", + " collection=mod35l2_geoloco_info['collection'],\n", + " granule_id=mod35l2_geoloco_info['granule_id'],\n", + " variables=mod35l2_geoloco_info['variable'],\n", + " scale_size=mod35l2_geoloco_info['downscale_size'],\n", + " crs=geo_proj4_string,\n", + " interpolation=resampling_string,\n", + " )\n", + "\n", + " mod08d3_request = Request(\n", + " collection=mod08d3_geoloco_info['collection'],\n", + " granule_id=mod08d3_geoloco_info['granule_id'],\n", + " variables=mod08d3_geoloco_info['variable'],\n", + " scale_size=mod08d3_geoloco_info['downscale_size'],\n", + " crs=geo_proj4_string,\n", + " interpolation=resampling_string,\n", + " )\n", + "\n", + " mod021km_compare_file = submit_and_download(\n", + " harmony_client, mod021km_request, file_indicators['MOD021KM']\n", + " )\n", + " mod35l2_compare_file = submit_and_download(\n", + " harmony_client, mod35l2_request, file_indicators['MOD35_L2']\n", + " )\n", + " mod08d3_compare_file = submit_and_download(\n", + " harmony_client, mod08d3_request, file_indicators['MOD08_D3']\n", + " )\n", "\n", " mod021km_test = True\n", " mod35l2_test = True\n", " mod08d3_test = True\n", - " \n", + "\n", " if compare_dimensions(reference_data['MOD021KM'], mod021km_compare_file):\n", " if not compare_data(reference_data['MOD021KM'], mod021km_compare_file):\n", " print_error('MOD021KM data mismatch.')\n", @@ -250,7 +289,7 @@ " else:\n", " print_error('MOD021KM data dimension mismatch.')\n", " mod021km_test = False\n", - " \n", + "\n", " if compare_dimensions(reference_data['MOD35_L2'], mod35l2_compare_file):\n", " if not compare_data(reference_data['MOD35_L2'], mod35l2_compare_file):\n", " print_error('MOD35_L2 data mismatch.')\n", @@ -258,7 +297,7 @@ " else:\n", " print_error('MOD35_L2 data dimension mismatch.')\n", " mod35l2_test = False\n", - " \n", + "\n", " if compare_dimensions(reference_data['MOD08_D3'], mod08d3_compare_file):\n", " if not compare_data(reference_data['MOD08_D3'], mod08d3_compare_file):\n", " print_error('MOD08_D3 data mismatch.')\n", @@ -275,9 +314,11 @@ " print_success('Geoloco Reprojection/Resampling/Regridding requests.')\n", " else:\n", " raise Exception('Geoloco test suite failed')\n", - " \n", + "\n", "else:\n", - " print(f'Geoloco is not configured for this environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'Geoloco is not configured for this environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] } ], diff --git a/test/geoloco/utilities.py b/test/geoloco/utilities.py index b80ab7c3..5e41fdf8 100644 --- a/test/geoloco/utilities.py +++ b/test/geoloco/utilities.py @@ -1,6 +1,7 @@ """" Common utility functions used by the geoloco regression tests. """ + import os from pyhdf.SD import SD, SDC import numpy @@ -9,11 +10,12 @@ from harmony.harmony import ProcessingFailedException -def submit_and_download(harmony_client: Client, request: Request, - file_indicator: str) -> str: - """ Submit a Harmony request via a `harmony-py` client. Wait for the - Harmony job to finish, then download the results to the specified file - path. +def submit_and_download( + harmony_client: Client, request: Request, file_indicator: str +) -> str: + """Submit a Harmony request via a `harmony-py` client. Wait for the + Harmony job to finish, then download the results to the specified file + path. """ downloaded_filenames = [] @@ -22,10 +24,10 @@ def submit_and_download(harmony_client: Client, request: Request, try: job_id = harmony_client.submit(request) - for filename in [file_future.result() - for file_future - in harmony_client.download_all(job_id, - overwrite=True)]: + for filename in [ + file_future.result() + for file_future in harmony_client.download_all(job_id, overwrite=True) + ]: print(f'Downloaded: {filename}') downloaded_filenames.extend([filename]) @@ -35,7 +37,6 @@ def submit_and_download(harmony_client: Client, request: Request, output_filename = filename print(f'Saved output to: {output_filename}') - except ProcessingFailedException as exception: print_error('Harmony request failed to complete successfully.') raise exception @@ -78,8 +79,8 @@ def get_sds_data(file: str): def remove_results_files() -> None: - """ Remove all HDF-4 files downloaded during the Geoloco - regression tests. + """Remove all HDF-4 files downloaded during the Geoloco + regression tests. """ directory_files = os.listdir() diff --git a/test/geoloco/version.txt b/test/geoloco/version.txt index afaf360d..7dea76ed 100644 --- a/test/geoloco/version.txt +++ b/test/geoloco/version.txt @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.0.1 diff --git a/test/harmony-regression/HarmonyRegression.ipynb b/test/harmony-regression/HarmonyRegression.ipynb index e3a9573c..8b0c6afd 100644 --- a/test/harmony-regression/HarmonyRegression.ipynb +++ b/test/harmony-regression/HarmonyRegression.ipynb @@ -33,12 +33,23 @@ }, "outputs": [], "source": [ - "#harmony_host_url = 'https://harmony.sit.earthdata.nasa.gov'\n", + "# harmony_host_url = 'https://harmony.sit.earthdata.nasa.gov'\n", "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'\n", - "#harmony_host_url = 'https://harmony.earthdata.nasa.gov'\n", + "# harmony_host_url = 'https://harmony.earthdata.nasa.gov'\n", "\n", "# Import libraries used throughout the notebook\n", - "from notebook_helpers import get, post, show, get_data_urls, show_async, show_async_condensed, show_shape, print_async_status, check_bbox_subset, check_stac\n", + "from notebook_helpers import (\n", + " get,\n", + " post,\n", + " show,\n", + " get_data_urls,\n", + " show_async,\n", + " show_async_condensed,\n", + " show_shape,\n", + " print_async_status,\n", + " check_bbox_subset,\n", + " check_stac,\n", + ")\n", "import json\n", "import intake\n", "import re" @@ -104,10 +115,12 @@ "source": [ "from harmony import Client, Collection, Environment, Request\n", "\n", - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", @@ -134,22 +147,20 @@ "outputs": [], "source": [ "if is_not_prod:\n", - " #By default, this reformats to tiff\n", + " # By default, this reformats to tiff\n", " params = {\n", " 'subset': [\n", " 'lon(-20:90)',\n", " 'lat(0:60)',\n", - " 'time(\"2020-01-15T00:00:00Z\":\"2020-01-15T01:00:00Z\")'\n", + " 'time(\"2020-01-15T00:00:00Z\":\"2020-01-15T01:00:00Z\")',\n", " ]\n", " }\n", "\n", " response = get(\n", " coverages_root.format(\n", - " root=harmony_host_url,\n", - " collection=l3_collection,\n", - " variable='all'\n", + " root=harmony_host_url, collection=l3_collection, variable='all'\n", " ),\n", - " params=params\n", + " params=params,\n", " )\n", "\n", " show(response, example_vars)" @@ -164,11 +175,9 @@ "if is_not_prod:\n", " response = get(\n", " coverages_root.format(\n", - " root=harmony_host_url,\n", - " collection=l3_collection,\n", - " variable='green_var'\n", + " root=harmony_host_url, collection=l3_collection, variable='green_var'\n", " ),\n", - " params=params\n", + " params=params,\n", " )\n", "\n", " show(response, color_index=1)" @@ -192,16 +201,13 @@ "outputs": [], "source": [ "if is_not_prod:\n", - " #By default, this reformats to tiff\n", + " # By default, this reformats to tiff\n", " response = get(\n", - " edr_root.format(\n", - " root=harmony_host_url,\n", - " collection=l3_collection\n", - " ),\n", + " edr_root.format(root=harmony_host_url, collection=l3_collection),\n", " params={\n", " 'bbox': '-20,0,90,60',\n", - " 'datetime': '2020-01-15T00:00:00Z/2020-01-15T01:00:00Z'\n", - " }\n", + " 'datetime': '2020-01-15T00:00:00Z/2020-01-15T01:00:00Z',\n", + " },\n", " )\n", "\n", " show(response, example_vars)" @@ -215,15 +221,12 @@ "source": [ "if is_not_prod:\n", " response = get(\n", - " edr_root.format(\n", - " root=harmony_host_url,\n", - " collection=l3_collection\n", - " ),\n", + " edr_root.format(root=harmony_host_url, collection=l3_collection),\n", " params={\n", " 'bbox': '-20,0,90,60',\n", " 'datetime': '2020-01-15T00:00:00Z/2020-01-15T01:00:00Z',\n", - " 'parameter-name': 'green_var'\n", - " }\n", + " 'parameter-name': 'green_var',\n", + " },\n", " )\n", "\n", " show(response, color_index=1)" @@ -249,17 +252,15 @@ "if is_not_prod:\n", " response = get(\n", " coverages_root.format(\n", - " root=harmony_host_url,\n", - " collection=l3_collection,\n", - " variable='all'\n", + " root=harmony_host_url, collection=l3_collection, variable='all'\n", " ),\n", " params={\n", " 'subset': [\n", " 'lon(-20:90)',\n", " 'lat(0:60)',\n", - " 'time(\"2020-01-01T00:00:00Z\":\"2020-01-05T01:00:00Z\")'\n", + " 'time(\"2020-01-01T00:00:00Z\":\"2020-01-05T01:00:00Z\")',\n", " ]\n", - " }\n", + " },\n", " )\n", " show_async_condensed(response)" ] @@ -283,34 +284,40 @@ "outputs": [], "source": [ "if is_not_prod:\n", - " #Add 3 requests\n", + " # Add 3 requests\n", " response1 = get(\n", - " coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'),\n", - " params={'format': 'image/tiff', 'maxResults': '20'}\n", + " coverages_root.format(\n", + " root=harmony_host_url, collection=l3_collection, variable='all'\n", + " ),\n", + " params={'format': 'image/tiff', 'maxResults': '20'},\n", " )\n", "\n", " response2 = get(\n", - " coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'),\n", - " params={'format': 'image/tiff', 'maxResults': '20'}\n", + " coverages_root.format(\n", + " root=harmony_host_url, collection=l3_collection, variable='all'\n", + " ),\n", + " params={'format': 'image/tiff', 'maxResults': '20'},\n", " )\n", "\n", " response3 = get(\n", - " coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'),\n", - " params={'format': 'image/tiff', 'maxResults': '20'}\n", + " coverages_root.format(\n", + " root=harmony_host_url, collection=l3_collection, variable='all'\n", + " ),\n", + " params={'format': 'image/tiff', 'maxResults': '20'},\n", " )\n", "\n", - " #List the requests\n", + " # List the requests\n", "\n", " jobs_root = '{root}/jobs'\n", " my_jobs = jobs_root.format(root=harmony_host_url)\n", - " response = get(my_jobs,params={'page': '1','limit': '10'})\n", + " response = get(my_jobs, params={'page': '1', 'limit': '10'})\n", " body = response.json()\n", "\n", " for job in body['jobs']:\n", " print_async_status(job)\n", "\n", - " #Cancel one\n", - " my_jobs_cancel_root=my_jobs+'/{job_id}/cancel'\n", + " # Cancel one\n", + " my_jobs_cancel_root = my_jobs + '/{job_id}/cancel'\n", " response = post(my_jobs_cancel_root.format(job_id=response3.json()['jobID']))\n", "\n", " print_async_status(response.json())\n", @@ -342,15 +349,13 @@ "source": [ "zarr_response = get(\n", " coverages_root.format(\n", - " root=harmony_host_url,\n", - " collection=l3_zarr_collection,\n", - " variable='all'\n", + " root=harmony_host_url, collection=l3_zarr_collection, variable='all'\n", " ),\n", " params={\n", " 'subset': 'time(\"2020-01-15T00:00:00Z\":\"2020-01-16T01:00:00Z\")',\n", - " 'maxResults': '3'\n", + " 'maxResults': '3',\n", " },\n", - " headers = {'accept': 'application/x-zarr'}\n", + " headers={'accept': 'application/x-zarr'},\n", ")\n", "\n", "show_async(zarr_response)" @@ -376,8 +381,7 @@ "stac_root = '{root}/stac/{jobID}/{item}'\n", "\n", "stac_cat = intake.open_stac_catalog(\n", - " stac_root.format(root=harmony_host_url, jobID=job, item=''),\n", - " name='Harmony output'\n", + " stac_root.format(root=harmony_host_url, jobID=job, item=''), name='Harmony output'\n", ")\n", "\n", "display(list(stac_cat))" @@ -390,7 +394,11 @@ "outputs": [], "source": [ "for i in range(len(list(stac_cat))):\n", - " display(intake.open_stac_item(stac_root.format(root=harmony_host_url,jobID=job,item=i)))\n", + " display(\n", + " intake.open_stac_item(\n", + " stac_root.format(root=harmony_host_url, jobID=job, item=i)\n", + " )\n", + " )\n", "\n", "entries = []\n", "for id, entry in stac_cat.search('type').items():\n", @@ -414,7 +422,7 @@ "metadata": {}, "outputs": [], "source": [ - "#da.plot.imshow()" + "# da.plot.imshow()" ] }, { @@ -460,18 +468,16 @@ "source": [ "response = get(\n", " coverages_root.format(\n", - " root=harmony_host_url,\n", - " collection=l2ss_collection,\n", - " variable='all'\n", + " root=harmony_host_url, collection=l2ss_collection, variable='all'\n", " ),\n", " params={\n", " 'maxResults': '1',\n", " 'subset': [\n", - " 'lon(-160:160)',\n", - " 'lat(-80:80)',\n", - " 'time(\"2012-03-03T12:17:00\":\"2012-03-03T12:18:00\")'\n", - " ]\n", - " }\n", + " 'lon(-160:160)',\n", + " 'lat(-80:80)',\n", + " 'time(\"2012-03-03T12:17:00\":\"2012-03-03T12:18:00\")',\n", + " ],\n", + " },\n", ")\n", "\n", "if is_not_prod:\n", @@ -497,9 +503,7 @@ "source": [ "response = get(\n", " coverages_root.format(\n", - " root=harmony_host_url,\n", - " collection=l2ss_collection,\n", - " variable='all'\n", + " root=harmony_host_url, collection=l2ss_collection, variable='all'\n", " ),\n", " params={\n", " 'maxResults': '3',\n", @@ -507,9 +511,9 @@ " 'subset': [\n", " 'lon(-160:160)',\n", " 'lat(-80:80)',\n", - " 'time(\"2012-03-03T00:00:00Z\":\"2012-03-03T02:59:59Z\")'\n", - " ]\n", - " }\n", + " 'time(\"2012-03-03T00:00:00Z\":\"2012-03-03T02:59:59Z\")',\n", + " ],\n", + " },\n", ")\n", "\n", "if is_not_prod:\n", @@ -535,8 +539,9 @@ "metadata": {}, "outputs": [], "source": [ - "download_service_request = Request(collection=Collection(download_collection),\n", - " max_results=1)\n", + "download_service_request = Request(\n", + " collection=Collection(download_collection), max_results=1\n", + ")\n", "\n", "download_service_job_id = harmony_client.submit(download_service_request)\n", "download_service_json = harmony_client.result_json(download_service_job_id)\n", diff --git a/test/harmony-regression/notebook_helpers/__init__.py b/test/harmony-regression/notebook_helpers/__init__.py index 456961f4..b764d626 100644 --- a/test/harmony-regression/notebook_helpers/__init__.py +++ b/test/harmony-regression/notebook_helpers/__init__.py @@ -1,4 +1,3 @@ -from contextlib import contextmanager import http.client as http_client import logging from datetime import datetime @@ -21,356 +20,398 @@ import requests from cachecontrol import CacheController, CacheControlAdapter + def _build_session(): - """Builds a requests session that caches responses where possible, making redirects faster. + """Builds a requests session that caches responses where possible, making redirects faster. + + Returns: + requests.Session -- A shared session to use for the notebook + """ + result = requests.session() - Returns: - requests.Session -- A shared session to use for the notebook - """ - result = requests.session() + # Set up caching. Particularly obey and cache 307 redirects to avoid duplicate expensive calls when we already + # have a result + cache_adapter = CacheControlAdapter() + cache_adapter.controller = CacheController( + cache=cache_adapter.cache, status_codes=(200, 203, 300, 301, 307) + ) - # Set up caching. Particularly obey and cache 307 redirects to avoid duplicate expensive calls when we already - # have a result - cache_adapter = CacheControlAdapter() - cache_adapter.controller = CacheController(cache=cache_adapter.cache, status_codes=(200, 203, 300, 301, 307)) + result.mount('http://', cache_adapter) + result.mount('https://', cache_adapter) + return result - result.mount('http://', cache_adapter) - result.mount('https://', cache_adapter) - return result # Session accessible by callers session = _build_session() -def debug_http(): - """Adds debugging output to HTTP requests to show redirects, headers, etc - """ - http_client.HTTPConnection.debuglevel = 1 - logging.basicConfig() - logging.getLogger().setLevel(logging.DEBUG) - requests_log = logging.getLogger("requests.packages.urllib3") - requests_log.setLevel(logging.DEBUG) - requests_log.propagate = True - -def request(*args, **kwargs): - """Thin wrapper around requests.Request, logging URL sent and Content-Type received - - See https://requests.readthedocs.io/en/master/api/#requests.Request for args - Returns: - requests.Response -- The response to the request - """ - req = requests.Request(*args, **kwargs) - prepped = session.prepare_request(req) - - print('%s %s' % (prepped.method, prepped.path_url)) - response = session.send(prepped) - #print('Received %s' % (response.headers.get('Content-Type', 'unknown content',))) - return response - -def get(*args, **kwargs): - """Performs a GET request using the request wrapper +def debug_http(): + """Adds debugging output to HTTP requests to show redirects, headers, etc""" + http_client.HTTPConnection.debuglevel = 1 + logging.basicConfig() + logging.getLogger().setLevel(logging.DEBUG) + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.DEBUG) + requests_log.propagate = True - See https://requests.readthedocs.io/en/master/api/#requests.Request for args - Returns: - requests.Response -- The response to the request - """ - return request('GET', *args, **kwargs) +def request(*args, **kwargs): + """Thin wrapper around requests.Request, logging URL sent and Content-Type received -def post(*args, **kwargs): - """Performs a POST request using the request wrapper + See https://requests.readthedocs.io/en/master/api/#requests.Request for args - See https://requests.readthedocs.io/en/master/api/#requests.Request for args + Returns: + requests.Response -- The response to the request + """ + req = requests.Request(*args, **kwargs) + prepped = session.prepare_request(req) - Returns: - requests.Response -- The response to the request - """ - return request('POST', *args, **kwargs) + print('%s %s' % (prepped.method, prepped.path_url)) + response = session.send(prepped) + # print('Received %s' % (response.headers.get('Content-Type', 'unknown content',))) + return response -def show_shape(filename, basemap=True): - """Plots the shapefile in the given filename with optional basemap (ESRI or GeoJSON) - Arguments: - filename {string} -- The filename of the shapefile to display +def get(*args, **kwargs): + """Performs a GET request using the request wrapper - Keyword Arguments: - basemap {bool} -- Whether to display a basemap under the shapefile (default: {True}) - """ - shape = gpd.read_file(filename).to_crs(epsg=3857) - plot = shape.plot(alpha=0.5, edgecolor='k', figsize=(8, 8)) - if basemap: - ctx.add_basemap(plot) + See https://requests.readthedocs.io/en/master/api/#requests.Request for args -def show(response, varList=[], color_index=None, immediate=True): - """Shows a variety of responses possible from Harmony for its example data + Returns: + requests.Response -- The response to the request + """ + return request('GET', *args, **kwargs) - Handles NetCDF files with red_var, green_var, blue_var, and alpha_var bands, compositing output - into a single colored image, ESRI Shapefiles with basemaps, and any type of image that can be - read by PIL, including GeoTIFF - Arguments: - response {requests.Response} -- The response containing the data to display - varList {array} -- If set, only plot the variables listed in varList. Otherwise, plot all. +def post(*args, **kwargs): + """Performs a POST request using the request wrapper - Keyword Arguments: - color_index {number} -- Set for monochromatic images to put the output in a color band (0=red, 1=green, 2=blue) (default: {None}) - immediate {bool} -- True if the data should be shown immediately in the notebook (default: {True}) + See https://requests.readthedocs.io/en/master/api/#requests.Request for args - """ + Returns: + requests.Response -- The response to the request + """ + return request('POST', *args, **kwargs) - # show_netcdf (look at dimensions, decide how to display); show_image - plt.rcParams['figure.figsize'] = [16, 8] - arrays = [] +def show_shape(filename, basemap=True): + """Plots the shapefile in the given filename with optional basemap (ESRI or GeoJSON) - check_status(response) - content_type = response.headers['Content-Type'] - print('Content-type: ', content_type) + Arguments: + filename {string} -- The filename of the shapefile to display - if content_type == 'binary/octet-stream' or content_type == 'application/octet-stream': - print('WARNING: Let service developer know to set their content_type correctly!') + Keyword Arguments: + basemap {bool} -- Whether to display a basemap under the shapefile (default: {True}) + """ + shape = gpd.read_file(filename).to_crs(epsg=3857) + plot = shape.plot(alpha=0.5, edgecolor='k', figsize=(8, 8)) + if basemap: + ctx.add_basemap(plot) - if content_type == 'application/x-netcdf' or content_type == 'application/x-netcdf4' or content_type == 'application/netcdf' or content_type == 'binary/octet-stream' or content_type == 'application/octet-stream': - # Show NetCDF4 - data = H5File(BytesIO(response.content), 'r') - #If user didn't provide any specific vars to plot, pull all of them into varList - if (len(varList) == 0): - varList = data.keys() - print(varList) - - #Plot the variables requested - for var in varList: - if var in data and len(data[var].shape) > 0: - ds = data[var] - if (len(data[var].shape) < 3): - #Simple plot for 1D or 2D - np_data=ds[()] - plt.plot(np_data) - plt.show() - else: - #Setup for 3D display - # ds = data[var] - values = np.flip(ds[0,:], 0) - where = (values != ds.attrs.get('_FillValue', None)) - scale = ds.attrs.get('scale_factor', [1])[0] - offset = ds.attrs.get('add_offset', [0])[0] - array = np.where(where, values * scale + offset, values) - arrays.append(array) - else: - print('Error: ', var, 'not found in dataset') - if (len(arrays) != 0): - #plot the 3D data - plt.imshow(np.dstack(arrays)) - elif content_type in ['application/zip', 'application/shapefile+zip']: - # Show ESRI Shapefiles - tmp = tempfile.NamedTemporaryFile(suffix='.shp.zip', delete=False) - try: - tmp.write(response.content) - show_shape('zip://' + tmp.name, immediate) - finally: - os.unlink(tmp.name) - elif 'application/json' in content_type: - #Most likely an error - print(response.json()) - assert(False) - else: - # Show Images - if color_index == None: - plt.imshow(Image.open(BytesIO(response.content))) +def show(response, varList=[], color_index=None, immediate=True): + """Shows a variety of responses possible from Harmony for its example data + + Handles NetCDF files with red_var, green_var, blue_var, and alpha_var bands, compositing output + into a single colored image, ESRI Shapefiles with basemaps, and any type of image that can be + read by PIL, including GeoTIFF + + Arguments: + response {requests.Response} -- The response containing the data to display + varList {array} -- If set, only plot the variables listed in varList. Otherwise, plot all. + + Keyword Arguments: + color_index {number} -- Set for monochromatic images to put the output in a color band (0=red, 1=green, 2=blue) (default: {None}) + immediate {bool} -- True if the data should be shown immediately in the notebook (default: {True}) + + """ + + # show_netcdf (look at dimensions, decide how to display); show_image + plt.rcParams['figure.figsize'] = [16, 8] + arrays = [] + + check_status(response) + content_type = response.headers['Content-Type'] + print('Content-type: ', content_type) + + if ( + content_type == 'binary/octet-stream' + or content_type == 'application/octet-stream' + ): + print( + 'WARNING: Let service developer know to set their content_type correctly!' + ) + + if ( + content_type == 'application/x-netcdf' + or content_type == 'application/x-netcdf4' + or content_type == 'application/netcdf' + or content_type == 'binary/octet-stream' + or content_type == 'application/octet-stream' + ): + # Show NetCDF4 + data = H5File(BytesIO(response.content), 'r') + + # If user didn't provide any specific vars to plot, pull all of them into varList + if len(varList) == 0: + varList = data.keys() + print(varList) + + # Plot the variables requested + for var in varList: + if var in data and len(data[var].shape) > 0: + ds = data[var] + if len(data[var].shape) < 3: + # Simple plot for 1D or 2D + np_data = ds[()] + plt.plot(np_data) + plt.show() + else: + # Setup for 3D display + # ds = data[var] + values = np.flip(ds[0, :], 0) + where = values != ds.attrs.get('_FillValue', None) + scale = ds.attrs.get('scale_factor', [1])[0] + offset = ds.attrs.get('add_offset', [0])[0] + array = np.where(where, values * scale + offset, values) + arrays.append(array) + else: + print('Error: ', var, 'not found in dataset') + if len(arrays) != 0: + # plot the 3D data + plt.imshow(np.dstack(arrays)) + elif content_type in ['application/zip', 'application/shapefile+zip']: + # Show ESRI Shapefiles + tmp = tempfile.NamedTemporaryFile(suffix='.shp.zip', delete=False) + try: + tmp.write(response.content) + show_shape('zip://' + tmp.name, immediate) + finally: + os.unlink(tmp.name) + elif 'application/json' in content_type: + # Most likely an error + print(response.json()) + assert False else: - gray_image = Image.open(BytesIO(response.content)) - # Move 1-channel green_var TIFF to second channel of RGB - image = Image.new('RGB', gray_image.size) - # There's probably a better way to do this with numpy - if color_index == 0: - image.putdata([(g, 0, 0) for g in gray_image.getdata()]) - if color_index == 1: - image.putdata([(0, g, 0) for g in gray_image.getdata()]) - if color_index == 2: - image.putdata([(0, 0, g) for g in gray_image.getdata()]) - plt.imshow(image) - if immediate: - plt.show() + # Show Images + if color_index is None: + plt.imshow(Image.open(BytesIO(response.content))) + else: + gray_image = Image.open(BytesIO(response.content)) + # Move 1-channel green_var TIFF to second channel of RGB + image = Image.new('RGB', gray_image.size) + # There's probably a better way to do this with numpy + if color_index == 0: + image.putdata([(g, 0, 0) for g in gray_image.getdata()]) + if color_index == 1: + image.putdata([(0, g, 0) for g in gray_image.getdata()]) + if color_index == 2: + image.putdata([(0, 0, g) for g in gray_image.getdata()]) + plt.imshow(image) + if immediate: + plt.show() + def get_data_urls(response): - """Returns the data URLs in an async response - - Arguments: - response {response.Response} -- The async job response - - Returns: - string[] -- An array of URLs for data links - """ - return [link['href'] for link in response.json()['links'] if link.get('rel', 'data') == 'data'] - -def show_async(response, varList = []): - """Shows an asynchronous Harmony response. - - Polls the output, displaying it as it changes, displaying any http data - links in the response as they arrive, and ultimately ending once the request - is successful or failed - - Arguments: - response {response.Response} -- the response to display - varList {array} -- If set, only plot the variables listed in varList. Otherwise, plot all. - - Returns: - response.Response -- the response from the final successful or failed poll - """ - def show_response(response, link_count): - print('Async response at', datetime.now().strftime("%H:%M:%S")) - print(json.dumps(response.json(), indent=2)) - links = get_data_urls(response) - new_links = links[slice(link_count, None)] - for link in new_links: - if link.startswith('http'): - show(get(link), varList) - return len(links) - - check_status(response) - displayed_link_count = 0 - body = response.json() - displayed_link_count = show_response(response, displayed_link_count) - waiting_message_printed = False - while body['status'] not in ['successful', 'failed', 'canceled']: - if not waiting_message_printed: - print('Waiting for updates...') - waiting_message_printed = True - sleep(1) - progress = body['progress'] - status = body['status'] - response = session.get(response.url) + """Returns the data URLs in an async response + + Arguments: + response {response.Response} -- The async job response + + Returns: + string[] -- An array of URLs for data links + """ + return [ + link['href'] + for link in response.json()['links'] + if link.get('rel', 'data') == 'data' + ] + + +def show_async(response, varList=[]): + """Shows an asynchronous Harmony response. + + Polls the output, displaying it as it changes, displaying any http data + links in the response as they arrive, and ultimately ending once the request + is successful or failed + + Arguments: + response {response.Response} -- the response to display + varList {array} -- If set, only plot the variables listed in varList. Otherwise, plot all. + + Returns: + response.Response -- the response from the final successful or failed poll + """ + + def show_response(response, link_count): + print('Async response at', datetime.now().strftime("%H:%M:%S")) + print(json.dumps(response.json(), indent=2)) + links = get_data_urls(response) + new_links = links[slice(link_count, None)] + for link in new_links: + if link.startswith('http'): + show(get(link), varList) + return len(links) + + check_status(response) + displayed_link_count = 0 body = response.json() - if progress != body['progress'] or status != body['status']: - displayed_link_count = show_response(response, displayed_link_count) - waiting_message_printed = False + displayed_link_count = show_response(response, displayed_link_count) + waiting_message_printed = False + while body['status'] not in ['successful', 'failed', 'canceled']: + if not waiting_message_printed: + print('Waiting for updates...') + waiting_message_printed = True + sleep(1) + progress = body['progress'] + status = body['status'] + response = session.get(response.url) + body = response.json() + if progress != body['progress'] or status != body['status']: + displayed_link_count = show_response(response, displayed_link_count) + waiting_message_printed = False + + assert body['status'] not in ['failed'] + check_stac(response) + print('Async request is complete') + return response - assert(body['status'] not in ['failed']) - check_stac(response) - print('Async request is complete') - return response def print_async_status(body): - """Prints the status, progress and any messages for the async job - - Arguments: - body {json} -- the response body to display - - """ - print('JobID:',body['jobID'],'Status:',body['status'],'(',body['progress'],'%) Messages:', body['message']) - -def show_async_condensed(response, varList = [], show_results=True): - """Shows a condensed version of the asynchronous Harmony response. Useful for getting status if you don't care about the results. - - Polls the output, displaying status as it changes, and ultimately ending once the request - is successful or failed - - Arguments: - response {response.Response} -- the response to display - varList {array} -- If set, only plot the variables listed in varList. Otherwise, plot all. - show_results {bool} -- True will display the results as they arrive. (default: {True}) - """ - def show_response_condensed(response, varList, link_count): - links = get_data_urls(response) - new_links = links[slice(link_count, None)] - for link in new_links: - if link.startswith('http'): - show(get(link), varList) - return len(links) - - check_status(response) - - displayed_link_count = 0 - body = response.json() - print ('Getting results for request') - print_async_status(body) - if show_results: - displayed_link_count = show_response_condensed(response, varList, displayed_link_count) - while body['status'] not in ['successful', 'failed', 'canceled']: - sleep(0.5) - progress = body['progress'] - status = body['status'] - response = session.get(response.url) + """Prints the status, progress and any messages for the async job + + Arguments: + body {json} -- the response body to display + + """ + print( + 'JobID:', + body['jobID'], + 'Status:', + body['status'], + '(', + body['progress'], + '%) Messages:', + body['message'], + ) + + +def show_async_condensed(response, varList=[], show_results=True): + """Shows a condensed version of the asynchronous Harmony response. Useful for getting status if you don't care about the results. + + Polls the output, displaying status as it changes, and ultimately ending once the request + is successful or failed + + Arguments: + response {response.Response} -- the response to display + varList {array} -- If set, only plot the variables listed in varList. Otherwise, plot all. + show_results {bool} -- True will display the results as they arrive. (default: {True}) + """ + + def show_response_condensed(response, varList, link_count): + links = get_data_urls(response) + new_links = links[slice(link_count, None)] + for link in new_links: + if link.startswith('http'): + show(get(link), varList) + return len(links) + + check_status(response) + + displayed_link_count = 0 body = response.json() - if progress != body['progress'] or status != body['status']: - if show_results: - displayed_link_count = show_response_condensed(response, varList, displayed_link_count) - print_async_status(body) - - assert(body['status'] not in ['failed']) - check_stac(response) - print('Async request is complete') - + print('Getting results for request') + print_async_status(body) + if show_results: + displayed_link_count = show_response_condensed( + response, varList, displayed_link_count + ) + while body['status'] not in ['successful', 'failed', 'canceled']: + sleep(0.5) + progress = body['progress'] + status = body['status'] + response = session.get(response.url) + body = response.json() + if progress != body['progress'] or status != body['status']: + if show_results: + displayed_link_count = show_response_condensed( + response, varList, displayed_link_count + ) + print_async_status(body) + + assert body['status'] not in ['failed'] + check_stac(response) + print('Async request is complete') def check_bbox_subset(response, req_lat_min, req_lat_max, req_lon_min, req_lon_max): - """Asserts if the spatial extents of the data in the response are within the requested bbox of a spatial subset + """Asserts if the spatial extents of the data in the response are within the requested bbox of a spatial subset + + ##### CHECK_BBOX_SUBSET currently is not in use; placeholder for the next round of regression test work - ##### CHECK_BBOX_SUBSET currently is not in use; placeholder for the next round of regression test work + Arguments: + response {response.Response} -- the response to display + req_lat_min -- The minimimum latitude from the request bbox for a spatial subset + req_lat_max -- The maximum latitude from the request bbox for a spatial subset + req_lon_min -- The minimimum longitude from the request bbox for a spatial subset + req_lon_max -- The maximum longitude from the request bbox for a spatial subset + """ - Arguments: - response {response.Response} -- the response to display - req_lat_min -- The minimimum latitude from the request bbox for a spatial subset - req_lat_max -- The maximum latitude from the request bbox for a spatial subset - req_lon_min -- The minimimum longitude from the request bbox for a spatial subset - req_lon_max -- The maximum longitude from the request bbox for a spatial subset - """ + data = H5File(BytesIO(response.content), 'r') - data = H5File(BytesIO(response.content), 'r') + attr_data = data['lat'][:] - attr_data = data['lat'][:] + print('Orig min and max: ', attr_data.min(), attr_data.max()) + lat_min = (attr_data.min() + 180) % 360 - 180 + lat_max = (attr_data.max() + 180) % 360 - 180 + print(lat_min) + print(lat_max) - print('Orig min and max: ', attr_data.min(), attr_data.max() ) - lat_min = (attr_data.min() + 180) % 360 - 180 - lat_max = (attr_data.max() + 180) % 360 - 180 - print(lat_min) - print(lat_max) + assert lat_max <= req_lat_max + assert lat_min >= req_lat_min - assert lat_max <= req_lat_max - assert lat_min >= req_lat_min + attr_data = data['lon'][:] + lon_min = (attr_data.min() + 180) % 360 - 180 + lon_max = (attr_data.max() + 180) % 360 - 180 + print(lon_min) + print(lon_max) - attr_data = data['lon'][:] - lon_min = (attr_data.min() + 180) % 360 - 180 - lon_max = (attr_data.max() + 180) % 360 - 180 - print(lon_min) - print(lon_max) + assert lon_max <= req_lon_max + assert lon_min >= req_lon_min - assert lon_max <= req_lon_max - assert lon_min >= req_lon_min def check_status(response): - """Asserts if the response is a 200, if not, print out the response code + """Asserts if the response is a 200, if not, print out the response code - Arguments: - response {response.Response} -- the response to display - """ - if (response.status_code != 200): - errStr = 'Request failed with status code ' + str(response.status_code) - assert False, errStr + Arguments: + response {response.Response} -- the response to display + """ + if response.status_code != 200: + errStr = 'Request failed with status code ' + str(response.status_code) + assert False, errStr def check_stac(response): - """Asserts if the response contains a valid STAC catalog and prints it out. More robust assertions could - be done here in the future to confirm that the STAC metadata is valid per the request parameters - - Arguments: - response {response.Response} -- the response to display - """ - for i in range(len(response.json()['links'])): - if response.json()['links'][i]['title'] == 'STAC catalog': - stac_url = response.json()['links'][i]['href'] - - assert(stac_url) - cat = Catalog.open(stac_url) - - for i in cat.items(): - assert(i.id) - assert(i.datetime) - assert(i.bbox) - assert(i.assets.keys()) - print('STAC Item') - print('\t', 'ID:', i.id) - print('\t', 'Date:', i.datetime) - print('\t', 'Bounding Box:', i.bbox) - print('\t', 'File:', list(i.assets.keys())) + """Asserts if the response contains a valid STAC catalog and prints it out. More robust assertions could + be done here in the future to confirm that the STAC metadata is valid per the request parameters + + Arguments: + response {response.Response} -- the response to display + """ + for i in range(len(response.json()['links'])): + if response.json()['links'][i]['title'] == 'STAC catalog': + stac_url = response.json()['links'][i]['href'] + + assert stac_url + cat = Catalog.open(stac_url) + + for i in cat.items(): + assert i.id + assert i.datetime + assert i.bbox + assert i.assets.keys() + print('STAC Item') + print('\t', 'ID:', i.id) + print('\t', 'Date:', i.datetime) + print('\t', 'Bounding Box:', i.bbox) + print('\t', 'File:', list(i.assets.keys())) diff --git a/test/harmony-regression/version.txt b/test/harmony-regression/version.txt index 9faa1b7a..c946ee61 100644 --- a/test/harmony-regression/version.txt +++ b/test/harmony-regression/version.txt @@ -1 +1 @@ -0.1.5 +0.1.6 diff --git a/test/harmony/Harmony.ipynb b/test/harmony/Harmony.ipynb index 7d3dcb4e..514236d4 100644 --- a/test/harmony/Harmony.ipynb +++ b/test/harmony/Harmony.ipynb @@ -64,17 +64,20 @@ "metadata": {}, "outputs": [], "source": [ - "expected_service_names = ['harmony/service-example', \n", - " 'nasa/harmony-gdal-adapter', \n", - " 'podaac/l2-subsetter']\n", + "expected_service_names = [\n", + " 'harmony/service-example',\n", + " 'nasa/harmony-gdal-adapter',\n", + " 'podaac/l2-subsetter',\n", + "]\n", "\n", "versions_response = requests.get(f'{harmony_host_url}/versions')\n", "versions_response.raise_for_status()\n", "actual_services = versions_response.json()\n", "\n", "for expected_service_name in expected_service_names:\n", - " assert any([service.get('name') == expected_service_name\n", - " for service in actual_services]), f'Could not find service: {expected_service_name}'\n", + " assert any(\n", + " [service.get('name') == expected_service_name for service in actual_services]\n", + " ), f'Could not find service: {expected_service_name}'\n", "\n", "print('All expected services returned from /versions endpoint.')" ] diff --git a/test/harmony/util.py b/test/harmony/util.py index a17aac5c..a92e7d9e 100644 --- a/test/harmony/util.py +++ b/test/harmony/util.py @@ -1,5 +1,6 @@ import requests + def _url(harmony_host_url: str, uri: str) -> str: return f'{harmony_host_url}/{uri}' diff --git a/test/harmony/version.txt b/test/harmony/version.txt index 845639ee..9faa1b7a 100644 --- a/test/harmony/version.txt +++ b/test/harmony/version.txt @@ -1 +1 @@ -0.1.4 +0.1.5 diff --git a/test/hga/HGA_Regression.ipynb b/test/hga/HGA_Regression.ipynb index 5460adbf..9f6ded67 100644 --- a/test/hga/HGA_Regression.ipynb +++ b/test/hga/HGA_Regression.ipynb @@ -76,9 +76,13 @@ "\n", "from harmony import BBox, Client, Collection, Environment, Request\n", "\n", - "from test_utilities import (check_request_output, clean_test_artefacts,\n", - " get_test_granule_id, make_request_and_download_result,\n", - " print_success)" + "from test_utilities import (\n", + " check_request_output,\n", + " clean_test_artefacts,\n", + " get_test_granule_id,\n", + " make_request_and_download_result,\n", + " print_success,\n", + ")" ] }, { @@ -110,16 +114,13 @@ "collection_data = {\n", " 'https://harmony.uat.earthdata.nasa.gov': {\n", " 'env': Environment.UAT,\n", - " **non_production_collections\n", + " **non_production_collections,\n", " },\n", " 'https://harmony.sit.earthdata.nasa.gov': {\n", " 'env': Environment.SIT,\n", - " **non_production_collections\n", - " },\n", - " 'http://localhost:3000': {\n", - " 'env':Environment.LOCAL,\n", - " **non_production_collections\n", + " **non_production_collections,\n", " },\n", + " 'http://localhost:3000': {'env': Environment.LOCAL, **non_production_collections},\n", "}\n", "\n", "environment_information = collection_data.get(harmony_host_url)\n", @@ -152,35 +153,44 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1813212660-ASF',\n", - " 'G1244144597-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1813212660-ASF', 'G1244144597-EEDTEST'\n", + " )\n", "\n", " avnir_directory_one = 'avnir_test_one'\n", "\n", - " avnir_request_one = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-51.0, -0.05, -50.75, 0.25),\n", - " variables=['Band1', 'Band2'], format='image/tiff',\n", - " granule_id=granule_id)\n", + " avnir_request_one = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-51.0, -0.05, -50.75, 0.25),\n", + " variables=['Band1', 'Band2'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", "\n", - " avnir_output_one = make_request_and_download_result(harmony_client, avnir_request_one,\n", - " avnir_directory_one)\n", + " avnir_output_one = make_request_and_download_result(\n", + " harmony_client, avnir_request_one, avnir_directory_one\n", + " )\n", "\n", " avnir_variable_base_one = 'ALAV2A237403590-OORIRFU'\n", "\n", - " avnir_variables_one = {f'IMG-01-{avnir_variable_base_one}_000_band_1',\n", - " f'IMG-02-{avnir_variable_base_one}_000_band_1'}\n", - "\n", - " expected_avnir_results_one = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 2,\n", - " 'proj_cs':'WGS 84 / UTM zone 22N',\n", - " 'proj_epsg': '32622',\n", - " 'spatial_extent': [-0.05, 0.3, -50.99, -50.75],\n", - " 'variables': avnir_variables_one,\n", - " 'width': 3410, \n", - " 'height': 3238}\n", + " avnir_variables_one = {\n", + " f'IMG-01-{avnir_variable_base_one}_000_band_1',\n", + " f'IMG-02-{avnir_variable_base_one}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_one = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 2,\n", + " 'proj_cs': 'WGS 84 / UTM zone 22N',\n", + " 'proj_epsg': '32622',\n", + " 'spatial_extent': [-0.05, 0.3, -50.99, -50.75],\n", + " 'variables': avnir_variables_one,\n", + " 'width': 3410,\n", + " 'height': 3238,\n", + " }\n", "\n", " check_request_output(avnir_output_one[0], expected_avnir_results_one)\n", " clean_test_artefacts(avnir_directory_one)\n", @@ -208,35 +218,44 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1817336128-ASF',\n", - " 'G1244144599-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1817336128-ASF', 'G1244144599-EEDTEST'\n", + " )\n", "\n", " avnir_directory_two = 'avnir_test_two'\n", "\n", - " avnir_request_two = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-0.15, 14.8, 0.15, 14.9),\n", - " variables=['Band3', 'Band4'], format='image/tiff',\n", - " granule_id=granule_id)\n", + " avnir_request_two = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-0.15, 14.8, 0.15, 14.9),\n", + " variables=['Band3', 'Band4'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", "\n", - " avnir_output_two = make_request_and_download_result(harmony_client, avnir_request_two,\n", - " avnir_directory_two)\n", + " avnir_output_two = make_request_and_download_result(\n", + " harmony_client, avnir_request_two, avnir_directory_two\n", + " )\n", "\n", " avnir_variable_base_two = 'ALAV2A264953300-OORIRFU'\n", "\n", - " avnir_variables_two = {f'IMG-03-{avnir_variable_base_two}_000_band_1',\n", - " f'IMG-04-{avnir_variable_base_two}_000_band_1'}\n", - "\n", - " expected_avnir_results_two = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 2,\n", - " 'proj_cs':'WGS 84 / UTM zone 30N',\n", - " 'proj_epsg': '32630',\n", - " 'spatial_extent': [14.74, 14.96, -0.14, 0.14],\n", - " 'variables': avnir_variables_two,\n", - " 'width': 3392,\n", - " 'height': 1762}\n", + " avnir_variables_two = {\n", + " f'IMG-03-{avnir_variable_base_two}_000_band_1',\n", + " f'IMG-04-{avnir_variable_base_two}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_two = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 2,\n", + " 'proj_cs': 'WGS 84 / UTM zone 30N',\n", + " 'proj_epsg': '32630',\n", + " 'spatial_extent': [14.74, 14.96, -0.14, 0.14],\n", + " 'variables': avnir_variables_two,\n", + " 'width': 3392,\n", + " 'height': 1762,\n", + " }\n", "\n", " check_request_output(avnir_output_two[0], expected_avnir_results_two)\n", " clean_test_artefacts(avnir_directory_two)\n", @@ -264,35 +283,44 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1835935580-ASF',\n", - " 'G1244144598-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1835935580-ASF', 'G1244144598-EEDTEST'\n", + " )\n", "\n", " avnir_directory_three = 'avnir_test_three'\n", "\n", - " avnir_request_three = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(179.75, 66.5, -179.85, 66.75),\n", - " variables=['Band1', 'Band2'], format='image/tiff',\n", - " granule_id=granule_id)\n", + " avnir_request_three = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(179.75, 66.5, -179.85, 66.75),\n", + " variables=['Band1', 'Band2'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", "\n", - " avnir_output_three = make_request_and_download_result(harmony_client, avnir_request_three,\n", - " avnir_directory_three)\n", + " avnir_output_three = make_request_and_download_result(\n", + " harmony_client, avnir_request_three, avnir_directory_three\n", + " )\n", "\n", " avnir_variable_base_three = 'ALAV2A239652240-OORIRFU'\n", "\n", - " avnir_variables_three = {f'IMG-01-{avnir_variable_base_three}_002_band_1',\n", - " f'IMG-02-{avnir_variable_base_three}_002_band_1'}\n", - "\n", - " expected_avnir_results_three = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 2,\n", - " 'proj_cs':'WGS 84 / UTM zone 1N',\n", - " 'proj_epsg': '32601',\n", - " 'spatial_extent': [66.44, 66.81, 179.85, -179.95],\n", - " 'variables': avnir_variables_three,\n", - " 'width': 2694,\n", - " 'height': 3250}\n", + " avnir_variables_three = {\n", + " f'IMG-01-{avnir_variable_base_three}_002_band_1',\n", + " f'IMG-02-{avnir_variable_base_three}_002_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_three = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 2,\n", + " 'proj_cs': 'WGS 84 / UTM zone 1N',\n", + " 'proj_epsg': '32601',\n", + " 'spatial_extent': [66.44, 66.81, 179.85, -179.95],\n", + " 'variables': avnir_variables_three,\n", + " 'width': 2694,\n", + " 'height': 3250,\n", + " }\n", "\n", " check_request_output(avnir_output_three[0], expected_avnir_results_three)\n", " clean_test_artefacts(avnir_directory_three)\n", @@ -320,33 +348,40 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1849288272-ASF',\n", - " 'G1244144595-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1849288272-ASF', 'G1244144595-EEDTEST'\n", + " )\n", "\n", " avnir_directory_four = 'avnir_test_four'\n", "\n", - " avnir_request_four = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-99.15, 36.9, -98.85, 37.1),\n", - " variables=['Band4'], format='image/tiff',\n", - " granule_id=granule_id)\n", + " avnir_request_four = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-99.15, 36.9, -98.85, 37.1),\n", + " variables=['Band4'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", "\n", - " avnir_output_four = make_request_and_download_result(harmony_client, avnir_request_four,\n", - " avnir_directory_four)\n", + " avnir_output_four = make_request_and_download_result(\n", + " harmony_client, avnir_request_four, avnir_directory_four\n", + " )\n", "\n", " avnir_variable_base_four = 'ALAV2A278852850-OORIRFU'\n", " avnir_variables_four = {f'IMG-04-{avnir_variable_base_four}_000_band_1'}\n", "\n", - " expected_avnir_results_four = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs':'WGS 84 / UTM zone 14N',\n", - " 'proj_epsg': '32614',\n", - " 'spatial_extent': [36.85, 37.15, -99.13, -98.87],\n", - " 'variables': avnir_variables_four,\n", - " 'width': 3108,\n", - " 'height': 2774}\n", + " expected_avnir_results_four = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': 'WGS 84 / UTM zone 14N',\n", + " 'proj_epsg': '32614',\n", + " 'spatial_extent': [36.85, 37.15, -99.13, -98.87],\n", + " 'variables': avnir_variables_four,\n", + " 'width': 3108,\n", + " 'height': 2774,\n", + " }\n", "\n", " check_request_output(avnir_output_four[0], expected_avnir_results_four)\n", " clean_test_artefacts(avnir_directory_four)\n", @@ -375,41 +410,50 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1849288272-ASF',\n", - " 'G1244144595-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1849288272-ASF', 'G1244144595-EEDTEST'\n", + " )\n", "\n", " avnir_directory_five = 'avnir_test_five'\n", "\n", - " avnir_request_five = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-99.2, 36.95, -98.9, 37.05), format='image/tiff',\n", - " granule_id=granule_id)\n", + " avnir_request_five = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-99.2, 36.95, -98.9, 37.05),\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", "\n", - " avnir_output_five = make_request_and_download_result(harmony_client, avnir_request_five,\n", - " avnir_directory_five)\n", + " avnir_output_five = make_request_and_download_result(\n", + " harmony_client, avnir_request_five, avnir_directory_five\n", + " )\n", "\n", " avnir_variable_base_five = 'ALAV2A278852850-OORIRFU'\n", "\n", - " avnir_variables_five = {f'IMG-03-{avnir_variable_base_five}_000_band_1',\n", - " f'IMG-02-{avnir_variable_base_five}_000_band_1',\n", - " f'IMG-04-{avnir_variable_base_five}_000_band_1',\n", - " f'IMG-01-{avnir_variable_base_five}_000_band_1'}\n", - "\n", - " expected_avnir_results_five = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 4,\n", - " 'proj_cs':'WGS 84 / UTM zone 14N',\n", - " 'proj_epsg': '32614',\n", - " 'spatial_extent': [36.9, 37.1, -99.18, -98.92],\n", - " 'variables': avnir_variables_five,\n", - " 'width': 2853,\n", - " 'height': 1693}\n", + " avnir_variables_five = {\n", + " f'IMG-03-{avnir_variable_base_five}_000_band_1',\n", + " f'IMG-02-{avnir_variable_base_five}_000_band_1',\n", + " f'IMG-04-{avnir_variable_base_five}_000_band_1',\n", + " f'IMG-01-{avnir_variable_base_five}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_five = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 4,\n", + " 'proj_cs': 'WGS 84 / UTM zone 14N',\n", + " 'proj_epsg': '32614',\n", + " 'spatial_extent': [36.9, 37.1, -99.18, -98.92],\n", + " 'variables': avnir_variables_five,\n", + " 'width': 2853,\n", + " 'height': 1693,\n", + " }\n", "\n", " check_request_output(avnir_output_five[0], expected_avnir_results_five)\n", " clean_test_artefacts(avnir_directory_five)\n", "else:\n", - " print('Skipping test: HGA regression tests not configured for this environment.') " + " print('Skipping test: HGA regression tests not configured for this environment.')" ] }, { @@ -432,35 +476,46 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1835549401-ASF',\n", - " 'G1244144601-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1835549401-ASF', 'G1244144601-EEDTEST'\n", + " )\n", "\n", " avnir_directory_six = 'avnir_test_six'\n", "\n", - " avnir_request_six = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(18.07, -26.1, 18.25, -25.8),\n", - " variables=['Band1', 'Band3'], format='image/tiff',\n", - " granule_id=granule_id, height=30, width=75)\n", + " avnir_request_six = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(18.07, -26.1, 18.25, -25.8),\n", + " variables=['Band1', 'Band3'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " height=30,\n", + " width=75,\n", + " )\n", "\n", - " avnir_output_six = make_request_and_download_result(harmony_client, avnir_request_six,\n", - " avnir_directory_six)\n", + " avnir_output_six = make_request_and_download_result(\n", + " harmony_client, avnir_request_six, avnir_directory_six\n", + " )\n", "\n", " avnir_variable_base_six = 'ALAV2A151894120-OORIRFU'\n", "\n", - " avnir_variables_six = {f'IMG-01-{avnir_variable_base_six}_000_band_1',\n", - " f'IMG-03-{avnir_variable_base_six}_000_band_1'}\n", - "\n", - " expected_avnir_results_six = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 2,\n", - " 'proj_cs':'WGS 84 / UTM zone 34S',\n", - " 'proj_epsg': '32734',\n", - " 'spatial_extent': [-26.13, -25.77, 18.07, 18.25],\n", - " 'variables': avnir_variables_six,\n", - " 'width': 75,\n", - " 'height': 30}\n", + " avnir_variables_six = {\n", + " f'IMG-01-{avnir_variable_base_six}_000_band_1',\n", + " f'IMG-03-{avnir_variable_base_six}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_six = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 2,\n", + " 'proj_cs': 'WGS 84 / UTM zone 34S',\n", + " 'proj_epsg': '32734',\n", + " 'spatial_extent': [-26.13, -25.77, 18.07, 18.25],\n", + " 'variables': avnir_variables_six,\n", + " 'width': 75,\n", + " 'height': 30,\n", + " }\n", "\n", " check_request_output(avnir_output_six[0], expected_avnir_results_six)\n", " clean_test_artefacts(avnir_directory_six)\n", @@ -488,37 +543,48 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1906378530-ASF',\n", - " 'G1244144603-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1906378530-ASF', 'G1244144603-EEDTEST'\n", + " )\n", "\n", " avnir_directory_seven = 'avnir_test_seven'\n", "\n", - " avnir_request_seven = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-97.65, 49.65, -97.35, 49.95),\n", - " variables=['Band1', 'Band2', 'Band3', 'Band4'],\n", - " granule_id=granule_id, height=45, width=90, format='image/tiff')\n", + " avnir_request_seven = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-97.65, 49.65, -97.35, 49.95),\n", + " variables=['Band1', 'Band2', 'Band3', 'Band4'],\n", + " granule_id=granule_id,\n", + " height=45,\n", + " width=90,\n", + " format='image/tiff',\n", + " )\n", "\n", - " avnir_output_seven = make_request_and_download_result(harmony_client, avnir_request_seven,\n", - " avnir_directory_seven)\n", + " avnir_output_seven = make_request_and_download_result(\n", + " harmony_client, avnir_request_seven, avnir_directory_seven\n", + " )\n", "\n", " avnir_variable_base_seven = 'ALAV2A277832590-OORIRFU'\n", "\n", - " avnir_variables_seven = {f'IMG-01-{avnir_variable_base_seven}_000_band_1',\n", - " f'IMG-02-{avnir_variable_base_seven}_000_band_1',\n", - " f'IMG-04-{avnir_variable_base_seven}_000_band_1',\n", - " f'IMG-03-{avnir_variable_base_seven}_000_band_1',}\n", - "\n", - " expected_avnir_results_seven = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 4,\n", - " 'proj_cs':'WGS 84 / UTM zone 14N',\n", - " 'proj_epsg': '32614',\n", - " 'spatial_extent': [49.6, 49.99, -97.64, -97.36],\n", - " 'variables': avnir_variables_seven,\n", - " 'width': 90,\n", - " 'height': 45}\n", + " avnir_variables_seven = {\n", + " f'IMG-01-{avnir_variable_base_seven}_000_band_1',\n", + " f'IMG-02-{avnir_variable_base_seven}_000_band_1',\n", + " f'IMG-04-{avnir_variable_base_seven}_000_band_1',\n", + " f'IMG-03-{avnir_variable_base_seven}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_seven = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 4,\n", + " 'proj_cs': 'WGS 84 / UTM zone 14N',\n", + " 'proj_epsg': '32614',\n", + " 'spatial_extent': [49.6, 49.99, -97.64, -97.36],\n", + " 'variables': avnir_variables_seven,\n", + " 'width': 90,\n", + " 'height': 45,\n", + " }\n", "\n", " check_request_output(avnir_output_seven[0], expected_avnir_results_seven)\n", " clean_test_artefacts(avnir_directory_seven)\n", @@ -546,36 +612,45 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1813329318-ASF',\n", - " 'G1244144596-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1813329318-ASF', 'G1244144596-EEDTEST'\n", + " )\n", "\n", " avnir_directory_eight = 'avnir_test_eight'\n", "\n", - " avnir_request_eight = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-79.9, 9.2, -78.9, 10.3),\n", - " granule_id=granule_id, format='image/tiff')\n", + " avnir_request_eight = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-79.9, 9.2, -78.9, 10.3),\n", + " granule_id=granule_id,\n", + " format='image/tiff',\n", + " )\n", "\n", - " avnir_output_eight = make_request_and_download_result(harmony_client, avnir_request_eight,\n", - " avnir_directory_eight)\n", + " avnir_output_eight = make_request_and_download_result(\n", + " harmony_client, avnir_request_eight, avnir_directory_eight\n", + " )\n", "\n", " avnir_variable_base_eight = 'ALAV2A150623400-OORIRFU'\n", "\n", - " avnir_variables_eight = { f'IMG-01-{avnir_variable_base_eight}_000_band_1',\n", - " f'IMG-02-{avnir_variable_base_eight}_000_band_1',\n", - " f'IMG-04-{avnir_variable_base_eight}_000_band_1',\n", - " f'IMG-03-{avnir_variable_base_eight}_000_band_1', }\n", - "\n", - " expected_avnir_results_eight = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 4,\n", - " 'proj_cs':'WGS 84 / UTM zone 17N',\n", - " 'proj_epsg': '32617',\n", - " 'spatial_extent': [9.4, 10.24, -79.7, -79.2],\n", - " 'variables': avnir_variables_eight,\n", - " 'width': 7215,\n", - " 'height': 8000}\n", + " avnir_variables_eight = {\n", + " f'IMG-01-{avnir_variable_base_eight}_000_band_1',\n", + " f'IMG-02-{avnir_variable_base_eight}_000_band_1',\n", + " f'IMG-04-{avnir_variable_base_eight}_000_band_1',\n", + " f'IMG-03-{avnir_variable_base_eight}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_eight = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 4,\n", + " 'proj_cs': 'WGS 84 / UTM zone 17N',\n", + " 'proj_epsg': '32617',\n", + " 'spatial_extent': [9.4, 10.24, -79.7, -79.2],\n", + " 'variables': avnir_variables_eight,\n", + " 'width': 7215,\n", + " 'height': 8000,\n", + " }\n", "\n", " check_request_output(avnir_output_eight[0], expected_avnir_results_eight)\n", " clean_test_artefacts(avnir_directory_eight)\n", @@ -605,32 +680,39 @@ "if environment_information is not None:\n", " avnir_directory_nine = 'avnir_test_nine'\n", "\n", - " avnir_request_nine = Request(collection=environment_information['avnir_collection'],\n", - " variables=['all'],\n", - " spatial=BBox(18.4, -26.1, 18.6, -26.0),\n", - " format='image/tiff')\n", + " avnir_request_nine = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " variables=['all'],\n", + " spatial=BBox(18.4, -26.1, 18.6, -26.0),\n", + " format='image/tiff',\n", + " )\n", "\n", - " avnir_output_nine = make_request_and_download_result(harmony_client, avnir_request_nine,\n", - " avnir_directory_nine)\n", + " avnir_output_nine = make_request_and_download_result(\n", + " harmony_client, avnir_request_nine, avnir_directory_nine\n", + " )\n", "\n", " avnir_variable_base_nine = 'ALAV2A151894120-OORIRFU'\n", "\n", - " avnir_variables_nine = {f'IMG-01-{avnir_variable_base_nine}_000_band_1',\n", - " f'IMG-02-{avnir_variable_base_nine}_000_band_1',\n", - " f'IMG-03-{avnir_variable_base_nine}_000_band_1',\n", - " f'IMG-04-{avnir_variable_base_nine}_000_band_1'}\n", - "\n", - " expected_avnir_results_nine = {'authority': 'EPSG',\n", - " 'cs': 'Projected',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 4,\n", - " 'proj_cs':'WGS 84 / UTM zone 34S',\n", - " 'proj_epsg': '32734',\n", - " 'spatial_extent': [-26.13, -25.96, 18.41, 18.59],\n", - " 'variables': avnir_variables_nine,\n", - " 'width': 2195,\n", - " 'height': 1516}\n", + " avnir_variables_nine = {\n", + " f'IMG-01-{avnir_variable_base_nine}_000_band_1',\n", + " f'IMG-02-{avnir_variable_base_nine}_000_band_1',\n", + " f'IMG-03-{avnir_variable_base_nine}_000_band_1',\n", + " f'IMG-04-{avnir_variable_base_nine}_000_band_1',\n", + " }\n", + "\n", + " expected_avnir_results_nine = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Projected',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 4,\n", + " 'proj_cs': 'WGS 84 / UTM zone 34S',\n", + " 'proj_epsg': '32734',\n", + " 'spatial_extent': [-26.13, -25.96, 18.41, 18.59],\n", + " 'variables': avnir_variables_nine,\n", + " 'width': 2195,\n", + " 'height': 1516,\n", + " }\n", "\n", " check_request_output(avnir_output_nine[0], expected_avnir_results_nine)\n", " clean_test_artefacts(avnir_directory_nine)\n", @@ -674,7 +756,7 @@ "# 'G1244144597-EEDTEST')\n", "#\n", "# avnir_directory_ten = 'avnir_test_ten'\n", - "# \n", + "#\n", "# avnir_request_ten = Request(collection=environment_information['avnir_collection'],\n", "# spatial=BBox(-51.0, -0.05, -50.75, 0.25), variables=['Band2'],\n", "# granule_id=granule_id, format='image/png')\n", @@ -722,30 +804,38 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1813212660-ASF',\n", - " 'G1244144597-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1813212660-ASF', 'G1244144597-EEDTEST'\n", + " )\n", "\n", " avnir_directory_eleven = 'avnir_test_eleven'\n", "\n", - " avnir_request_eleven = Request(collection=environment_information['avnir_collection'],\n", - " spatial=BBox(-51.0, -0.05, -50.75, 0.25), variables=['Band2'],\n", - " granule_id=granule_id, format='image/gif')\n", - "\n", - " avnir_output_eleven = make_request_and_download_result(harmony_client, avnir_request_eleven,\n", - " avnir_directory_eleven)\n", - " expected_avnir_results_eleven = {'authority': None,\n", - " 'cs': None,\n", - " 'file_type': 'gif',\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'spatial_extent': None,\n", - " 'variables': {},\n", - " 'reference_image': 'reference_images/avnir_reference_eleven.gif',\n", - " 'width': 3410,\n", - " 'height': 3238}\n", + " avnir_request_eleven = Request(\n", + " collection=environment_information['avnir_collection'],\n", + " spatial=BBox(-51.0, -0.05, -50.75, 0.25),\n", + " variables=['Band2'],\n", + " granule_id=granule_id,\n", + " format='image/gif',\n", + " )\n", + "\n", + " avnir_output_eleven = make_request_and_download_result(\n", + " harmony_client, avnir_request_eleven, avnir_directory_eleven\n", + " )\n", + " expected_avnir_results_eleven = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'file_type': 'gif',\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'spatial_extent': None,\n", + " 'variables': {},\n", + " 'reference_image': 'reference_images/avnir_reference_eleven.gif',\n", + " 'width': 3410,\n", + " 'height': 3238,\n", + " }\n", "\n", " check_request_output(avnir_output_eleven[0], expected_avnir_results_eleven)\n", " clean_test_artefacts(avnir_directory_eleven)\n", @@ -777,36 +867,39 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1715962900-ASF',\n", - " 'G1244144618-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1715962900-ASF', 'G1244144618-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_one = 'sentinel_test_one'\n", "\n", - " sentinel_request_one = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-115.5, 33, -115.25, 33.1),\n", - " variables=['science/grids/data/amplitude'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_one = make_request_and_download_result(harmony_client, sentinel_request_one,\n", - " sentinel_directory_one)\n", - "\n", - " sentinel_variables_one = {'amplitude',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_one = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 4,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_one.nc',\n", - " 'spatial_extent': [33.0, 33.1, -115.5, -115.25],\n", - " 'variables': sentinel_variables_one,\n", - " 'width': 300,\n", - " 'height': 120}\n", + " sentinel_request_one = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-115.5, 33, -115.25, 33.1),\n", + " variables=['science/grids/data/amplitude'],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_one = make_request_and_download_result(\n", + " harmony_client, sentinel_request_one, sentinel_directory_one\n", + " )\n", + "\n", + " sentinel_variables_one = {'amplitude', 'lat', 'lon', 'latitude_longitude'}\n", + "\n", + " expected_sentinel_results_one = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 4,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_one.nc',\n", + " 'spatial_extent': [33.0, 33.1, -115.5, -115.25],\n", + " 'variables': sentinel_variables_one,\n", + " 'width': 300,\n", + " 'height': 120,\n", + " }\n", "\n", " check_request_output(sentinel_output_one[0], expected_sentinel_results_one)\n", " clean_test_artefacts(sentinel_directory_one)\n", @@ -834,34 +927,45 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1944514838-ASF',\n", - " 'G1244144611-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1944514838-ASF', 'G1244144611-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_two = 'sentinel_test_two'\n", "\n", - " sentinel_request_two = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(37.25, 26.95, 38.15, 27.25),\n", - " variables=['science/grids/data/coherence'],\n", - " granule_id=granule_id, format='image/tiff')\n", - "\n", - " sentinel_output_two = make_request_and_download_result(harmony_client, sentinel_request_two,\n", - " sentinel_directory_two)\n", - "\n", - " sentinel_variables_two = {('S1-GUNW-A-R-014-tops-20191127_20191115-153047-28315N_26592N-PP-8a6f-v2_0_3_'\n", - " 'science_grids_data_coherence_subsetted__science_grids_data_coherence__subsetted_band_1')}\n", - "\n", - " expected_sentinel_results_two = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_two.tiff',\n", - " 'spatial_extent': [26.95, 27.25, 37.25, 38.15],\n", - " 'variables': sentinel_variables_two,\n", - " 'width': 1080,\n", - " 'height': 360}\n", + " sentinel_request_two = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(37.25, 26.95, 38.15, 27.25),\n", + " variables=['science/grids/data/coherence'],\n", + " granule_id=granule_id,\n", + " format='image/tiff',\n", + " )\n", + "\n", + " sentinel_output_two = make_request_and_download_result(\n", + " harmony_client, sentinel_request_two, sentinel_directory_two\n", + " )\n", + "\n", + " sentinel_variables_two = {\n", + " (\n", + " 'S1-GUNW-A-R-014-tops-20191127_20191115-153047-28315N_26592N-PP-8a6f-v2_0_3_'\n", + " 'science_grids_data_coherence_subsetted__science_grids_data_coherence__subsetted_band_1'\n", + " )\n", + " }\n", + "\n", + " expected_sentinel_results_two = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_two.tiff',\n", + " 'spatial_extent': [26.95, 27.25, 37.25, 38.15],\n", + " 'variables': sentinel_variables_two,\n", + " 'width': 1080,\n", + " 'height': 360,\n", + " }\n", "\n", " check_request_output(sentinel_output_two[0], expected_sentinel_results_two)\n", " clean_test_artefacts(sentinel_directory_two)\n", @@ -889,34 +993,44 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1944514838-ASF',\n", - " 'G1244144611-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1944514838-ASF', 'G1244144611-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_three = 'sentinel_test_three'\n", - " \n", - " sentinel_request_three = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(37.5, 27.0, 38.25, 27.5),\n", - " variables=['science/grids/data/connectedComponents'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_three = make_request_and_download_result(harmony_client, sentinel_request_three,\n", - " sentinel_directory_three)\n", - "\n", - "\n", - " sentinel_variables_three = {'connectedComponents', 'lat', 'lon', 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_three = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 4,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_three.nc',\n", - " 'spatial_extent': [27.0, 27.5, 37.5, 38.25],\n", - " 'variables': sentinel_variables_three,\n", - " 'width': 900,\n", - " 'height': 600}\n", + "\n", + " sentinel_request_three = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(37.5, 27.0, 38.25, 27.5),\n", + " variables=['science/grids/data/connectedComponents'],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_three = make_request_and_download_result(\n", + " harmony_client, sentinel_request_three, sentinel_directory_three\n", + " )\n", + "\n", + " sentinel_variables_three = {\n", + " 'connectedComponents',\n", + " 'lat',\n", + " 'lon',\n", + " 'latitude_longitude',\n", + " }\n", + "\n", + " expected_sentinel_results_three = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 4,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_three.nc',\n", + " 'spatial_extent': [27.0, 27.5, 37.5, 38.25],\n", + " 'variables': sentinel_variables_three,\n", + " 'width': 900,\n", + " 'height': 600,\n", + " }\n", "\n", " check_request_output(sentinel_output_three[0], expected_sentinel_results_three)\n", " clean_test_artefacts(sentinel_directory_three)\n", @@ -944,40 +1058,45 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1944514838-ASF',\n", - " 'G1244144611-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1944514838-ASF', 'G1244144611-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_four = 'sentinel_test_four'\n", "\n", - " sentinel_request_four = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(36.9, 26.9, 39.1, 27.5),\n", - " variables=['science/grids/data/unwrappedPhase'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_four = make_request_and_download_result(harmony_client, sentinel_request_four,\n", - " sentinel_directory_four)\n", - "\n", - " sentinel_base_variable_four = ('S1_GUNW_A_R_014_tops_20191127_20191115_153047_28315N_26592N_PP_8a6f_v2_0_3_'\n", - " 'science_grids_data_unwrappedPhase_subsetted__'\n", - " 'science_grids_data_unwrappedPhase__colored_band')\n", - "\n", - " sentinel_variables_four = {'unwrappedPhase',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_four = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 4,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_four.nc',\n", - " 'spatial_extent': [26.9, 27.5, 36.9, 39.1],\n", - " 'variables': sentinel_variables_four,\n", - " 'width': 2640,\n", - " 'height': 720}\n", + " sentinel_request_four = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(36.9, 26.9, 39.1, 27.5),\n", + " variables=['science/grids/data/unwrappedPhase'],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_four = make_request_and_download_result(\n", + " harmony_client, sentinel_request_four, sentinel_directory_four\n", + " )\n", + "\n", + " sentinel_base_variable_four = (\n", + " 'S1_GUNW_A_R_014_tops_20191127_20191115_153047_28315N_26592N_PP_8a6f_v2_0_3_'\n", + " 'science_grids_data_unwrappedPhase_subsetted__'\n", + " 'science_grids_data_unwrappedPhase__colored_band'\n", + " )\n", + "\n", + " sentinel_variables_four = {'unwrappedPhase', 'lat', 'lon', 'latitude_longitude'}\n", + "\n", + " expected_sentinel_results_four = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 4,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_four.nc',\n", + " 'spatial_extent': [26.9, 27.5, 36.9, 39.1],\n", + " 'variables': sentinel_variables_four,\n", + " 'width': 2640,\n", + " 'height': 720,\n", + " }\n", "\n", " check_request_output(sentinel_output_four[0], expected_sentinel_results_four)\n", " clean_test_artefacts(sentinel_directory_four)\n", @@ -1007,38 +1126,47 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1715962900-ASF',\n", - " 'G1244144618-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1715962900-ASF', 'G1244144618-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_five = 'sentinel_test_five'\n", "\n", - " sentinel_request_five = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-115.5, 33, -115.25, 33.1),\n", - " variables=['science/grids/data/amplitude',\n", - " 'science/grids/data/coherence'],\n", - " granule_id=granule_id, height=60, width=150)\n", - "\n", - " sentinel_output_five = make_request_and_download_result(harmony_client, sentinel_request_five,\n", - " sentinel_directory_five)\n", - "\n", - " sentinel_variables_five = {'amplitude',\n", - " 'coherence',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_five = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 5,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_five.nc',\n", - " 'spatial_extent': [33.0, 33.1, -115.5, -115.25],\n", - " 'variables': sentinel_variables_five,\n", - " 'width': 150,\n", - " 'height': 60}\n", + " sentinel_request_five = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-115.5, 33, -115.25, 33.1),\n", + " variables=['science/grids/data/amplitude', 'science/grids/data/coherence'],\n", + " granule_id=granule_id,\n", + " height=60,\n", + " width=150,\n", + " )\n", + "\n", + " sentinel_output_five = make_request_and_download_result(\n", + " harmony_client, sentinel_request_five, sentinel_directory_five\n", + " )\n", + "\n", + " sentinel_variables_five = {\n", + " 'amplitude',\n", + " 'coherence',\n", + " 'lat',\n", + " 'lon',\n", + " 'latitude_longitude',\n", + " }\n", + "\n", + " expected_sentinel_results_five = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 5,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_five.nc',\n", + " 'spatial_extent': [33.0, 33.1, -115.5, -115.25],\n", + " 'variables': sentinel_variables_five,\n", + " 'width': 150,\n", + " 'height': 60,\n", + " }\n", "\n", " check_request_output(sentinel_output_five[0], expected_sentinel_results_five)\n", " clean_test_artefacts(sentinel_directory_five)\n", @@ -1066,41 +1194,52 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1715962900-ASF',\n", - " 'G1244144618-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1715962900-ASF', 'G1244144618-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_six = 'sentinel_test_six'\n", "\n", - " sentinel_request_six = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-115.52, 33.05, -115.27, 33.1),\n", - " variables=['science/grids/data/amplitude',\n", - " 'science/grids/data/coherence',\n", - " 'science/grids/data/connectedComponents'],\n", - " granule_id=granule_id, height=30, width=75)\n", - "\n", - " sentinel_output_six = make_request_and_download_result(harmony_client, sentinel_request_six,\n", - " sentinel_directory_six)\n", - "\n", - "\n", - " sentinel_variables_six = {'amplitude',\n", - " 'coherence',\n", - " 'connectedComponents',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_six = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 6,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_six.nc',\n", - " 'spatial_extent': [33.05, 33.1, -115.52, -115.27],\n", - " 'variables': sentinel_variables_six,\n", - " 'width': 75,\n", - " 'height': 30}\n", + " sentinel_request_six = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-115.52, 33.05, -115.27, 33.1),\n", + " variables=[\n", + " 'science/grids/data/amplitude',\n", + " 'science/grids/data/coherence',\n", + " 'science/grids/data/connectedComponents',\n", + " ],\n", + " granule_id=granule_id,\n", + " height=30,\n", + " width=75,\n", + " )\n", + "\n", + " sentinel_output_six = make_request_and_download_result(\n", + " harmony_client, sentinel_request_six, sentinel_directory_six\n", + " )\n", + "\n", + " sentinel_variables_six = {\n", + " 'amplitude',\n", + " 'coherence',\n", + " 'connectedComponents',\n", + " 'lat',\n", + " 'lon',\n", + " 'latitude_longitude',\n", + " }\n", + "\n", + " expected_sentinel_results_six = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 6,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_six.nc',\n", + " 'spatial_extent': [33.05, 33.1, -115.52, -115.27],\n", + " 'variables': sentinel_variables_six,\n", + " 'width': 75,\n", + " 'height': 30,\n", + " }\n", "\n", " check_request_output(sentinel_output_six[0], expected_sentinel_results_six)\n", " clean_test_artefacts(sentinel_directory_six)\n", @@ -1128,42 +1267,52 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1715962900-ASF',\n", - " 'G1244144618-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1715962900-ASF', 'G1244144618-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_seven = 'sentinel_test_seven'\n", "\n", - " sentinel_request_seven = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-115.45, 32.95, -115.15, 33.09),\n", - " variables=['science/grids/data/amplitude',\n", - " 'science/grids/data/coherence',\n", - " 'science/grids/data/connectedComponents',\n", - " 'science/grids/data/unwrappedPhase'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_seven = make_request_and_download_result(harmony_client, sentinel_request_seven,\n", - " sentinel_directory_seven)\n", - "\n", - " sentinel_variables_seven = {'amplitude',\n", - " 'coherence',\n", - " 'connectedComponents',\n", - " 'unwrappedPhase',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_seven = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 7,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_seven.nc',\n", - " 'spatial_extent': [32.95, 33.09, -115.45, -115.15],\n", - " 'variables': sentinel_variables_seven,\n", - " 'width': 360,\n", - " 'height': 168}\n", + " sentinel_request_seven = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-115.45, 32.95, -115.15, 33.09),\n", + " variables=[\n", + " 'science/grids/data/amplitude',\n", + " 'science/grids/data/coherence',\n", + " 'science/grids/data/connectedComponents',\n", + " 'science/grids/data/unwrappedPhase',\n", + " ],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_seven = make_request_and_download_result(\n", + " harmony_client, sentinel_request_seven, sentinel_directory_seven\n", + " )\n", + "\n", + " sentinel_variables_seven = {\n", + " 'amplitude',\n", + " 'coherence',\n", + " 'connectedComponents',\n", + " 'unwrappedPhase',\n", + " 'lat',\n", + " 'lon',\n", + " 'latitude_longitude',\n", + " }\n", + "\n", + " expected_sentinel_results_seven = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 7,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_seven.nc',\n", + " 'spatial_extent': [32.95, 33.09, -115.45, -115.15],\n", + " 'variables': sentinel_variables_seven,\n", + " 'width': 360,\n", + " 'height': 168,\n", + " }\n", "\n", " check_request_output(sentinel_output_seven[0], expected_sentinel_results_seven)\n", " clean_test_artefacts(sentinel_directory_seven)\n", @@ -1191,36 +1340,39 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1697222082-ASF',\n", - " 'G1244144612-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1697222082-ASF', 'G1244144612-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_eight = 'sentinel_test_eight'\n", "\n", - " sentinel_request_eight = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-69.8, -37.2, -68.8, -36.9),\n", - " variables=['science/grids/data/amplitude'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_eight = make_request_and_download_result(harmony_client, sentinel_request_eight,\n", - " sentinel_directory_eight)\n", - "\n", - " sentinel_variables_eight = {'amplitude',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_eight = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 4,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_eight.nc',\n", - " 'spatial_extent': [-37.2, -36.9, -69.8, -68.8],\n", - " 'variables': sentinel_variables_eight,\n", - " 'width': 1200,\n", - " 'height': 360}\n", + " sentinel_request_eight = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-69.8, -37.2, -68.8, -36.9),\n", + " variables=['science/grids/data/amplitude'],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_eight = make_request_and_download_result(\n", + " harmony_client, sentinel_request_eight, sentinel_directory_eight\n", + " )\n", + "\n", + " sentinel_variables_eight = {'amplitude', 'lat', 'lon', 'latitude_longitude'}\n", + "\n", + " expected_sentinel_results_eight = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 4,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_eight.nc',\n", + " 'spatial_extent': [-37.2, -36.9, -69.8, -68.8],\n", + " 'variables': sentinel_variables_eight,\n", + " 'width': 1200,\n", + " 'height': 360,\n", + " }\n", "\n", " check_request_output(sentinel_output_eight[0], expected_sentinel_results_eight)\n", " clean_test_artefacts(sentinel_directory_eight)\n", @@ -1248,36 +1400,39 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1697222082-ASF',\n", - " 'G1244144612-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1697222082-ASF', 'G1244144612-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_nine = 'sentinel_test_nine'\n", "\n", - " sentinel_request_nine = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-71.6, -39.0, -67.5, -35.0),\n", - " variables=['science/grids/data/coherence'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_nine = make_request_and_download_result(harmony_client, sentinel_request_nine,\n", - " sentinel_directory_nine)\n", - "\n", - " sentinel_variables_nine = {'coherence',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_nine = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 4,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_nine.nc',\n", - " 'spatial_extent': [-38.14, -36.0, -71.41, -67.96],\n", - " 'variables': sentinel_variables_nine,\n", - " 'width': 4143,\n", - " 'height': 2559}\n", + " sentinel_request_nine = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-71.6, -39.0, -67.5, -35.0),\n", + " variables=['science/grids/data/coherence'],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_nine = make_request_and_download_result(\n", + " harmony_client, sentinel_request_nine, sentinel_directory_nine\n", + " )\n", + "\n", + " sentinel_variables_nine = {'coherence', 'lat', 'lon', 'latitude_longitude'}\n", + "\n", + " expected_sentinel_results_nine = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 4,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_nine.nc',\n", + " 'spatial_extent': [-38.14, -36.0, -71.41, -67.96],\n", + " 'variables': sentinel_variables_nine,\n", + " 'width': 4143,\n", + " 'height': 2559,\n", + " }\n", "\n", " check_request_output(sentinel_output_nine[0], expected_sentinel_results_nine)\n", " clean_test_artefacts(sentinel_directory_nine)\n", @@ -1305,38 +1460,45 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1722540208-ASF',\n", - " 'G1244144616-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1722540208-ASF', 'G1244144616-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_ten = 'sentinel_test_ten'\n", "\n", - " sentinel_request_ten = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(86.85, 43.75, 88.15, 44.85),\n", - " variables=['science/grids/data/coherence',\n", - " 'science/grids/data/unwrappedPhase'],\n", - " granule_id=granule_id)\n", - "\n", - " sentinel_output_ten = make_request_and_download_result(harmony_client, sentinel_request_ten,\n", - " sentinel_directory_ten)\n", - "\n", - " sentinel_variables_ten = {'coherence',\n", - " 'unwrappedPhase',\n", - " 'lat',\n", - " 'lon',\n", - " 'latitude_longitude'}\n", - "\n", - " expected_sentinel_results_ten = {'authority': None,\n", - " 'cs': None,\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 5,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_ten.nc',\n", - " 'spatial_extent': [43.75, 44.85, 86.85, 88.15],\n", - " 'variables': sentinel_variables_ten,\n", - " 'width': 1560,\n", - " 'height': 1320}\n", + " sentinel_request_ten = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(86.85, 43.75, 88.15, 44.85),\n", + " variables=['science/grids/data/coherence', 'science/grids/data/unwrappedPhase'],\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " sentinel_output_ten = make_request_and_download_result(\n", + " harmony_client, sentinel_request_ten, sentinel_directory_ten\n", + " )\n", + "\n", + " sentinel_variables_ten = {\n", + " 'coherence',\n", + " 'unwrappedPhase',\n", + " 'lat',\n", + " 'lon',\n", + " 'latitude_longitude',\n", + " }\n", + "\n", + " expected_sentinel_results_ten = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 5,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_ten.nc',\n", + " 'spatial_extent': [43.75, 44.85, 86.85, 88.15],\n", + " 'variables': sentinel_variables_ten,\n", + " 'width': 1560,\n", + " 'height': 1320,\n", + " }\n", "\n", " check_request_output(sentinel_output_ten[0], expected_sentinel_results_ten)\n", " clean_test_artefacts(sentinel_directory_ten)\n", @@ -1423,34 +1585,41 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1715962900-ASF',\n", - " 'G1244144618-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1715962900-ASF', 'G1244144618-EEDTEST'\n", + " )\n", "\n", " sentinel_directory_twelve = 'sentinel_test_twelve'\n", "\n", - " sentinel_request_twelve = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-115.5, 33, -115.45, 33.05),\n", - " variables=['science/grids/data/amplitude'],\n", - " granule_id=granule_id, format='image/gif')\n", + " sentinel_request_twelve = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-115.5, 33, -115.45, 33.05),\n", + " variables=['science/grids/data/amplitude'],\n", + " granule_id=granule_id,\n", + " format='image/gif',\n", + " )\n", "\n", - " sentinel_output_twelve = make_request_and_download_result(harmony_client, sentinel_request_twelve,\n", - " sentinel_directory_twelve)\n", + " sentinel_output_twelve = make_request_and_download_result(\n", + " harmony_client, sentinel_request_twelve, sentinel_directory_twelve\n", + " )\n", "\n", " sentinel_variables_twelve = {}\n", "\n", - " expected_sentinel_results_twelve = {'authority': None,\n", - " 'cs': None,\n", - " 'file_type': 'gif',\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/sentinel_reference_twelve.gif',\n", - " 'spatial_extent': None,\n", - " 'variables': sentinel_variables_twelve,\n", - " 'width': 60,\n", - " 'height': 60}\n", + " expected_sentinel_results_twelve = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'file_type': 'gif',\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/sentinel_reference_twelve.gif',\n", + " 'spatial_extent': None,\n", + " 'variables': sentinel_variables_twelve,\n", + " 'width': 60,\n", + " 'height': 60,\n", + " }\n", "\n", " check_request_output(sentinel_output_twelve[0], expected_sentinel_results_twelve)\n", " clean_test_artefacts(sentinel_directory_twelve)\n", @@ -1476,21 +1645,28 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1715962900-ASF',\n", - " 'G1244144618-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1715962900-ASF', 'G1244144618-EEDTEST'\n", + " )\n", "\n", - " sentinel_request_thirteen = Request(collection=environment_information['sentinel_collection'],\n", - " spatial=BBox(-115.45, 32.95, -115.15, 33.09),\n", - " granule_id=granule_id,\n", - " ignore_errors=False,\n", - " variables=['science/grids/data/amplitude',\n", - " 'science/grids/imagingGeometry/azimuthAngle'])\n", + " sentinel_request_thirteen = Request(\n", + " collection=environment_information['sentinel_collection'],\n", + " spatial=BBox(-115.45, 32.95, -115.15, 33.09),\n", + " granule_id=granule_id,\n", + " ignore_errors=False,\n", + " variables=[\n", + " 'science/grids/data/amplitude',\n", + " 'science/grids/imagingGeometry/azimuthAngle',\n", + " ],\n", + " )\n", "\n", " sentinel_job_id_thirteen = harmony_client.submit(sentinel_request_thirteen)\n", - " try: \n", + " try:\n", " harmony_client.wait_for_processing(sentinel_job_id_thirteen, show_progress=True)\n", " except Exception as exception:\n", - " assert 'datasets are incompatible and cannot be combined' in str(exception), 'Exception not raised correctly'\n", + " assert 'datasets are incompatible and cannot be combined' in str(\n", + " exception\n", + " ), 'Exception not raised correctly'\n", " print_success('Test raised expected Exception')\n", "else:\n", " print('Skipping test: HGA regression tests not configured for this environment.')" @@ -1521,35 +1697,42 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1422449017-ASF',\n", - " 'G1244144733-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1422449017-ASF', 'G1244144733-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_one = 'uavsar_test_one'\n", "\n", - " uavsar_request_one = Request(collection=environment_information['uavsar_collection'],\n", - " spatial=BBox(-145.75, 64.0, -145.70, 64.1),\n", - " variables=['Band1'], format='image/tiff',\n", - " granule_id=granule_id)\n", + " uavsar_request_one = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " spatial=BBox(-145.75, 64.0, -145.70, 64.1),\n", + " variables=['Band1'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", "\n", - " uavsar_output_one = make_request_and_download_result(harmony_client, uavsar_request_one,\n", - " uavsar_directory_one)\n", + " uavsar_output_one = make_request_and_download_result(\n", + " harmony_client, uavsar_request_one, uavsar_directory_one\n", + " )\n", "\n", " uavsar_variable_base_one = 'djNEON_00101_17095_012_170910_L090_CX_01_pauli_Band1_subsetted__Band1__subsetted_band'\n", - " \n", + "\n", " uavsar_variables_one = {f'{uavsar_variable_base_one}_1'}\n", "\n", - " expected_uavsar_results_one = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_one.tiff',\n", - " 'spatial_extent': [64.0, 64.1, -145.75, -145.7],\n", - " 'variables': uavsar_variables_one,\n", - " 'width': 450, \n", - " 'height': 1800}\n", + " expected_uavsar_results_one = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_one.tiff',\n", + " 'spatial_extent': [64.0, 64.1, -145.75, -145.7],\n", + " 'variables': uavsar_variables_one,\n", + " 'width': 450,\n", + " 'height': 1800,\n", + " }\n", "\n", " check_request_output(uavsar_output_one[0], expected_uavsar_results_one)\n", " clean_test_artefacts(uavsar_directory_one)\n", @@ -1575,33 +1758,42 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1422449017-ASF',\n", - " 'G1244144733-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1422449017-ASF', 'G1244144733-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_two = 'uavsar_test_two'\n", "\n", - " uavsar_request_two = Request(collection=environment_information['uavsar_collection'],\n", - " spatial=BBox(-145.9, 63.75, -145.8, 63.80),\n", - " variables=['Band2'], format='image/tiff',\n", - " granule_id=granule_id)\n", - "\n", - " uavsar_output_two = make_request_and_download_result(harmony_client, uavsar_request_two,\n", - " uavsar_directory_two)\n", - "\n", - " uavsar_variables_two = {'djNEON_00101_17095_012_170910_L090_CX_01_pauli_Band2_subsetted__Band2__subsetted_band_1'}\n", - "\n", - " expected_uavsar_results_two = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_two.tiff',\n", - " 'spatial_extent': [63.75, 63.80, -145.9, -145.8],\n", - " 'variables': uavsar_variables_two,\n", - " 'width': 900, \n", - " 'height': 900}\n", + " uavsar_request_two = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " spatial=BBox(-145.9, 63.75, -145.8, 63.80),\n", + " variables=['Band2'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_two = make_request_and_download_result(\n", + " harmony_client, uavsar_request_two, uavsar_directory_two\n", + " )\n", + "\n", + " uavsar_variables_two = {\n", + " 'djNEON_00101_17095_012_170910_L090_CX_01_pauli_Band2_subsetted__Band2__subsetted_band_1'\n", + " }\n", + "\n", + " expected_uavsar_results_two = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_two.tiff',\n", + " 'spatial_extent': [63.75, 63.80, -145.9, -145.8],\n", + " 'variables': uavsar_variables_two,\n", + " 'width': 900,\n", + " 'height': 900,\n", + " }\n", "\n", " check_request_output(uavsar_output_two[0], expected_uavsar_results_two)\n", " clean_test_artefacts(uavsar_directory_two)\n", @@ -1629,33 +1821,42 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1422449017-ASF',\n", - " 'G1244144733-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1422449017-ASF', 'G1244144733-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_three = 'uavsar_test_three'\n", "\n", - " uavsar_request_three = Request(collection=environment_information['uavsar_collection'],\n", - " spatial=BBox(-145.85, 63.85, -145.80, 63.90),\n", - " variables=['Band3'], format='image/tiff',\n", - " granule_id=granule_id)\n", - "\n", - " uavsar_output_three = make_request_and_download_result(harmony_client, uavsar_request_three,\n", - " uavsar_directory_three)\n", - "\n", - " uavsar_variables_three = {'djNEON_00101_17095_012_170910_L090_CX_01_pauli_Band3_subsetted__Band3__subsetted_band_1'}\n", - "\n", - " expected_uavsar_results_three = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_three.tiff',\n", - " 'spatial_extent': [63.85, 63.90, -145.85, -145.80],\n", - " 'variables': uavsar_variables_three,\n", - " 'width': 450, \n", - " 'height': 900}\n", + " uavsar_request_three = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " spatial=BBox(-145.85, 63.85, -145.80, 63.90),\n", + " variables=['Band3'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_three = make_request_and_download_result(\n", + " harmony_client, uavsar_request_three, uavsar_directory_three\n", + " )\n", + "\n", + " uavsar_variables_three = {\n", + " 'djNEON_00101_17095_012_170910_L090_CX_01_pauli_Band3_subsetted__Band3__subsetted_band_1'\n", + " }\n", + "\n", + " expected_uavsar_results_three = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_three.tiff',\n", + " 'spatial_extent': [63.85, 63.90, -145.85, -145.80],\n", + " 'variables': uavsar_variables_three,\n", + " 'width': 450,\n", + " 'height': 900,\n", + " }\n", "\n", " check_request_output(uavsar_output_three[0], expected_uavsar_results_three)\n", " clean_test_artefacts(uavsar_directory_three)\n", @@ -1683,34 +1884,43 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1366089385-ASF',\n", - " 'G1244144737-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1366089385-ASF', 'G1244144737-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_four = 'uavsar_test_four'\n", "\n", - " uavsar_request_four = Request(collection=environment_information['uavsar_collection'],\n", - " spatial=BBox(-123.39, 40.05, -123.34, 40.06),\n", - " variables=['Band1', 'Band2'], format='image/tiff',\n", - " granule_id=granule_id)\n", - "\n", - " uavsar_output_four = make_request_and_download_result(harmony_client, uavsar_request_four,\n", - " uavsar_directory_four)\n", - "\n", - " uavsar_variables_four = {'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__Band1__subsetted_band_1',\n", - " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__Band2__subsetted_band_1'}\n", - "\n", - " expected_uavsar_results_four = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 2,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_four.tiff',\n", - " 'spatial_extent': [40.05, 40.06, -123.39, -123.34],\n", - " 'variables': uavsar_variables_four,\n", - " 'width': 900, \n", - " 'height': 180}\n", + " uavsar_request_four = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " spatial=BBox(-123.39, 40.05, -123.34, 40.06),\n", + " variables=['Band1', 'Band2'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_four = make_request_and_download_result(\n", + " harmony_client, uavsar_request_four, uavsar_directory_four\n", + " )\n", + "\n", + " uavsar_variables_four = {\n", + " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__Band1__subsetted_band_1',\n", + " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__Band2__subsetted_band_1',\n", + " }\n", + "\n", + " expected_uavsar_results_four = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 2,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_four.tiff',\n", + " 'spatial_extent': [40.05, 40.06, -123.39, -123.34],\n", + " 'variables': uavsar_variables_four,\n", + " 'width': 900,\n", + " 'height': 180,\n", + " }\n", "\n", " check_request_output(uavsar_output_four[0], expected_uavsar_results_four)\n", " clean_test_artefacts(uavsar_directory_four)\n", @@ -1738,34 +1948,43 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1366089385-ASF',\n", - " 'G1244144737-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1366089385-ASF', 'G1244144737-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_five = 'uavsar_test_five'\n", "\n", - " uavsar_request_five = Request(collection=environment_information['uavsar_collection'],\n", - " spatial=BBox(-123.69, 39.95, -123.59, 40.0),\n", - " format='image/tiff', granule_id=granule_id)\n", - "\n", - " uavsar_output_five = make_request_and_download_result(harmony_client, uavsar_request_five,\n", - " uavsar_directory_five)\n", - "\n", - " uavsar_variables_five = {'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__all__subsetted_band_1',\n", - " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__all__subsetted_band_2', \n", - " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__all__subsetted_band_3',}\n", - "\n", - " expected_uavsar_results_five = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 3,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_five.tiff',\n", - " 'spatial_extent': [39.95, 40.0, -123.69, -123.59],\n", - " 'variables': uavsar_variables_five,\n", - " 'width': 1800, \n", - " 'height': 900}\n", + " uavsar_request_five = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " spatial=BBox(-123.69, 39.95, -123.59, 40.0),\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_five = make_request_and_download_result(\n", + " harmony_client, uavsar_request_five, uavsar_directory_five\n", + " )\n", + "\n", + " uavsar_variables_five = {\n", + " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__all__subsetted_band_1',\n", + " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__all__subsetted_band_2',\n", + " 'eelriv_06508_16103_002_161206_L090_CX_02_pauli_subsetted__all__subsetted_band_3',\n", + " }\n", + "\n", + " expected_uavsar_results_five = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 3,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_five.tiff',\n", + " 'spatial_extent': [39.95, 40.0, -123.69, -123.59],\n", + " 'variables': uavsar_variables_five,\n", + " 'width': 1800,\n", + " 'height': 900,\n", + " }\n", "\n", " check_request_output(uavsar_output_five[0], expected_uavsar_results_five)\n", " clean_test_artefacts(uavsar_directory_five)\n", @@ -1793,37 +2012,48 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1232482059-ASF',\n", - " 'G1244144732-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1232482059-ASF', 'G1244144732-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_six = 'uavsar_test_six'\n", "\n", - " uavsar_request_six = Request(collection=environment_information['uavsar_collection'],\n", - " variables=['all'],\n", - " spatial=BBox(11.57, -0.03, 11.60, 0.02),\n", - " format='image/tiff', granule_id=granule_id)\n", - "\n", - " uavsar_output_six = make_request_and_download_result(harmony_client, uavsar_request_six,\n", - " uavsar_directory_six)\n", - "\n", - " uavsar_base_six = 'eighty_14047_16008_006_160225_L090_CX_01_pauli_subsetted__all__subsetted'\n", - "\n", - " uavsar_variables_six = {f'{uavsar_base_six}_band_1',\n", - " f'{uavsar_base_six}_band_2',\n", - " f'{uavsar_base_six}_band_3'}\n", - "\n", - " expected_uavsar_results_six = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 3,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_six.tiff',\n", - " 'spatial_extent': [-0.03, 0.02, 11.57, 11.60],\n", - " 'variables': uavsar_variables_six,\n", - " 'width': 540, \n", - " 'height': 900}\n", + " uavsar_request_six = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " variables=['all'],\n", + " spatial=BBox(11.57, -0.03, 11.60, 0.02),\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_six = make_request_and_download_result(\n", + " harmony_client, uavsar_request_six, uavsar_directory_six\n", + " )\n", + "\n", + " uavsar_base_six = (\n", + " 'eighty_14047_16008_006_160225_L090_CX_01_pauli_subsetted__all__subsetted'\n", + " )\n", + "\n", + " uavsar_variables_six = {\n", + " f'{uavsar_base_six}_band_1',\n", + " f'{uavsar_base_six}_band_2',\n", + " f'{uavsar_base_six}_band_3',\n", + " }\n", + "\n", + " expected_uavsar_results_six = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 3,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_six.tiff',\n", + " 'spatial_extent': [-0.03, 0.02, 11.57, 11.60],\n", + " 'variables': uavsar_variables_six,\n", + " 'width': 540,\n", + " 'height': 900,\n", + " }\n", "\n", " check_request_output(uavsar_output_six[0], expected_uavsar_results_six)\n", " clean_test_artefacts(uavsar_directory_six)\n", @@ -1849,33 +2079,44 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1402334529-ASF',\n", - " 'G1244144730-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1402334529-ASF', 'G1244144730-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_seven = 'uavsar_test_seven'\n", "\n", - " uavsar_request_seven = Request(collection=environment_information['uavsar_collection'],\n", - " variables=['Band1'], spatial=BBox(-139, 61, -137, 62),\n", - " format='image/tiff', height=870, width=1129,\n", - " granule_id=granule_id)\n", - "\n", - " uavsar_output_seven = make_request_and_download_result(harmony_client, uavsar_request_seven,\n", - " uavsar_directory_seven)\n", - "\n", - " uavsar_variables_seven = {'kluane_01902_17064_005_170615_L090_CX_01_pauli_Band1_regridded_subsetted__Band1__resized_band_1'}\n", - "\n", - " expected_uavsar_results_seven = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_seven.tiff',\n", - " 'spatial_extent': [61.57, 61.76, -138.36, -137.85],\n", - " 'variables': uavsar_variables_seven,\n", - " 'width': 1129, \n", - " 'height': 870}\n", + " uavsar_request_seven = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " variables=['Band1'],\n", + " spatial=BBox(-139, 61, -137, 62),\n", + " format='image/tiff',\n", + " height=870,\n", + " width=1129,\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_seven = make_request_and_download_result(\n", + " harmony_client, uavsar_request_seven, uavsar_directory_seven\n", + " )\n", + "\n", + " uavsar_variables_seven = {\n", + " 'kluane_01902_17064_005_170615_L090_CX_01_pauli_Band1_regridded_subsetted__Band1__resized_band_1'\n", + " }\n", + "\n", + " expected_uavsar_results_seven = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_seven.tiff',\n", + " 'spatial_extent': [61.57, 61.76, -138.36, -137.85],\n", + " 'variables': uavsar_variables_seven,\n", + " 'width': 1129,\n", + " 'height': 870,\n", + " }\n", "\n", " check_request_output(uavsar_output_seven[0], expected_uavsar_results_seven)\n", " clean_test_artefacts(uavsar_directory_seven)\n", @@ -1901,33 +2142,42 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1223713114-ASF',\n", - " 'G1244144731-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1223713114-ASF', 'G1244144731-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_eight = 'uavsar_test_eight'\n", "\n", - " uavsar_request_eight = Request(collection=environment_information['uavsar_collection'],\n", - " spatial=BBox(-72.3, -40.02, -72.0, -40.0),\n", - " variables=['Band2'], format='image/tiff',\n", - " granule_id=granule_id)\n", - "\n", - " uavsar_output_eight = make_request_and_download_result(harmony_client, uavsar_request_eight,\n", - " uavsar_directory_eight)\n", - "\n", - " uavsar_variables_eight = {'ChiVol_00700_15030_010_150330_L090_CX_01_pauli_Band2_subsetted__Band2__subsetted_band_1'}\n", - "\n", - " expected_uavsar_results_eight = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_eight.tiff',\n", - " 'spatial_extent': [-40.02, -40.0, -72.17, -72.0],\n", - " 'variables': uavsar_variables_eight,\n", - " 'width': 3004, \n", - " 'height': 360}\n", + " uavsar_request_eight = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " spatial=BBox(-72.3, -40.02, -72.0, -40.0),\n", + " variables=['Band2'],\n", + " format='image/tiff',\n", + " granule_id=granule_id,\n", + " )\n", + "\n", + " uavsar_output_eight = make_request_and_download_result(\n", + " harmony_client, uavsar_request_eight, uavsar_directory_eight\n", + " )\n", + "\n", + " uavsar_variables_eight = {\n", + " 'ChiVol_00700_15030_010_150330_L090_CX_01_pauli_Band2_subsetted__Band2__subsetted_band_1'\n", + " }\n", + "\n", + " expected_uavsar_results_eight = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_eight.tiff',\n", + " 'spatial_extent': [-40.02, -40.0, -72.17, -72.0],\n", + " 'variables': uavsar_variables_eight,\n", + " 'width': 3004,\n", + " 'height': 360,\n", + " }\n", "\n", " check_request_output(uavsar_output_eight[0], expected_uavsar_results_eight)\n", " clean_test_artefacts(uavsar_directory_eight)\n", @@ -1955,32 +2205,41 @@ "if environment_information is not None:\n", " uavsar_directory_nine = 'uavsar_test_nine'\n", "\n", - " uavsar_request_nine = Request(collection=environment_information['uavsar_collection'],\n", - " variables=['all'],\n", - " spatial=BBox(-89.02, 28.98, -89.00, 29.00),\n", - " temporal={'start': datetime(2016, 11, 17, 23, 27, 0),\n", - " 'stop': datetime(2016, 11, 17, 23, 28, 0)},\n", - " format='image/tiff')\n", - "\n", - " uavsar_output_nine = make_request_and_download_result(harmony_client, uavsar_request_nine,\n", - " uavsar_directory_nine)\n", - "\n", - " uavsar_variables_nine = {'gulfco_27086_16101_004_161117_L090_CX_01_pauli_subsetted__all__subsetted_band_1',\n", - " 'gulfco_27086_16101_004_161117_L090_CX_01_pauli_subsetted__all__subsetted_band_2',\n", - " 'gulfco_27086_16101_004_161117_L090_CX_01_pauli_subsetted__all__subsetted_band_3',}\n", - "\n", - " expected_uavsar_results_nine = {'authority': 'EPSG',\n", - " 'cs': 'Geographic',\n", - " 'gcs': 'WGS 84',\n", - " 'gcs_epsg': '4326',\n", - " 'n_bands': 3,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_nine.tiff',\n", - " 'spatial_extent': [28.98, 29.00, -89.02, -89.00],\n", - " 'variables': uavsar_variables_nine,\n", - " 'width': 360, \n", - " 'height': 360}\n", + " uavsar_request_nine = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " variables=['all'],\n", + " spatial=BBox(-89.02, 28.98, -89.00, 29.00),\n", + " temporal={\n", + " 'start': datetime(2016, 11, 17, 23, 27, 0),\n", + " 'stop': datetime(2016, 11, 17, 23, 28, 0),\n", + " },\n", + " format='image/tiff',\n", + " )\n", + "\n", + " uavsar_output_nine = make_request_and_download_result(\n", + " harmony_client, uavsar_request_nine, uavsar_directory_nine\n", + " )\n", + "\n", + " uavsar_variables_nine = {\n", + " 'gulfco_27086_16101_004_161117_L090_CX_01_pauli_subsetted__all__subsetted_band_1',\n", + " 'gulfco_27086_16101_004_161117_L090_CX_01_pauli_subsetted__all__subsetted_band_2',\n", + " 'gulfco_27086_16101_004_161117_L090_CX_01_pauli_subsetted__all__subsetted_band_3',\n", + " }\n", + "\n", + " expected_uavsar_results_nine = {\n", + " 'authority': 'EPSG',\n", + " 'cs': 'Geographic',\n", + " 'gcs': 'WGS 84',\n", + " 'gcs_epsg': '4326',\n", + " 'n_bands': 3,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_nine.tiff',\n", + " 'spatial_extent': [28.98, 29.00, -89.02, -89.00],\n", + " 'variables': uavsar_variables_nine,\n", + " 'width': 360,\n", + " 'height': 360,\n", + " }\n", "\n", " check_request_output(uavsar_output_nine[0], expected_uavsar_results_nine)\n", " clean_test_artefacts(uavsar_directory_nine)\n", @@ -2043,7 +2302,7 @@ "# 'reference_image': 'reference_images/uavsar_reference_ten.png',\n", "# 'spatial_extent': [64.0, 64.1, -145.75, -145.70],\n", "# 'variables': uavsar_variables_ten,\n", - "# 'width': 450, \n", + "# 'width': 450,\n", "# 'height': 1800}\n", "#\n", "# check_request_output(uavsar_output_ten[0], expected_uavsar_results_ten)\n", @@ -2072,31 +2331,39 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], 'G1422449017-ASF',\n", - " 'G1244144733-EEDTEST')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], 'G1422449017-ASF', 'G1244144733-EEDTEST'\n", + " )\n", "\n", " uavsar_directory_eleven = 'uavsar_test_eleven'\n", "\n", - " uavsar_request_eleven = Request(collection=environment_information['uavsar_collection'],\n", - " variables=['Band1'], spatial=BBox(-145.75, 64.0, -145.70, 64.1),\n", - " granule_id=granule_id, format='image/gif')\n", - "\n", - " uavsar_output_eleven = make_request_and_download_result(harmony_client, uavsar_request_eleven,\n", - " uavsar_directory_eleven)\n", - "\n", - " expected_uavsar_results_eleven = {'authority': None,\n", - " 'cs': None,\n", - " 'file_type': 'gif',\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 1,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/uavsar_reference_eleven.tiff',\n", - " 'spatial_extent': [64.0, 64.1, -145.75, -145.70],\n", - " 'variables': {},\n", - " 'width': 450, \n", - " 'height': 1800}\n", + " uavsar_request_eleven = Request(\n", + " collection=environment_information['uavsar_collection'],\n", + " variables=['Band1'],\n", + " spatial=BBox(-145.75, 64.0, -145.70, 64.1),\n", + " granule_id=granule_id,\n", + " format='image/gif',\n", + " )\n", + "\n", + " uavsar_output_eleven = make_request_and_download_result(\n", + " harmony_client, uavsar_request_eleven, uavsar_directory_eleven\n", + " )\n", + "\n", + " expected_uavsar_results_eleven = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'file_type': 'gif',\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 1,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/uavsar_reference_eleven.tiff',\n", + " 'spatial_extent': [64.0, 64.1, -145.75, -145.70],\n", + " 'variables': {},\n", + " 'width': 450,\n", + " 'height': 1800,\n", + " }\n", "\n", " check_request_output(uavsar_output_eleven[0], expected_uavsar_results_eleven)\n", " clean_test_artefacts(uavsar_directory_eleven)\n", @@ -2128,14 +2395,20 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " granule_id = get_test_granule_id(environment_information['env'], None,\n", - " 'G1240564919-POCLOUD')\n", + " granule_id = get_test_granule_id(\n", + " environment_information['env'], None, 'G1240564919-POCLOUD'\n", + " )\n", "\n", " mur_directory_one = 'mur_test_one'\n", "\n", - " mur_request_one = Request(collection=environment_information['mur_collection'],\n", - " variables=['analysed_sst'], granule_id=granule_id,\n", - " format='image/png', width=900, height=450)\n", + " mur_request_one = Request(\n", + " collection=environment_information['mur_collection'],\n", + " variables=['analysed_sst'],\n", + " granule_id=granule_id,\n", + " format='image/png',\n", + " width=900,\n", + " height=450,\n", + " )\n", "\n", " mur_output_one = make_request_and_download_result(\n", " harmony_client, mur_request_one, mur_directory_one\n", @@ -2143,19 +2416,21 @@ "\n", " mur_variables_one = {'Band1', 'Band2', 'Band3', 'Band4'}\n", "\n", - " expected_mur_results_one = {'authority': None,\n", - " 'cs': None,\n", - " 'file_type': 'png',\n", - " 'gcs': None,\n", - " 'gcs_epsg': None,\n", - " 'n_bands': 4,\n", - " 'proj_cs': None,\n", - " 'proj_epsg': None,\n", - " 'reference_image': 'reference_images/mur_reference_one.png',\n", - " 'spatial_extent': None,\n", - " 'variables': mur_variables_one,\n", - " 'width': 900,\n", - " 'height': 450}\n", + " expected_mur_results_one = {\n", + " 'authority': None,\n", + " 'cs': None,\n", + " 'file_type': 'png',\n", + " 'gcs': None,\n", + " 'gcs_epsg': None,\n", + " 'n_bands': 4,\n", + " 'proj_cs': None,\n", + " 'proj_epsg': None,\n", + " 'reference_image': 'reference_images/mur_reference_one.png',\n", + " 'spatial_extent': None,\n", + " 'variables': mur_variables_one,\n", + " 'width': 900,\n", + " 'height': 450,\n", + " }\n", "\n", " check_request_output(mur_output_one[0], expected_mur_results_one)\n", " clean_test_artefacts(mur_directory_one)\n", diff --git a/test/hga/test_utilities.py b/test/hga/test_utilities.py index fab6f5fc..74783e48 100644 --- a/test/hga/test_utilities.py +++ b/test/hga/test_utilities.py @@ -12,10 +12,11 @@ from pyproj import Transformer -def get_test_granule_id(environment: Environment, production_granule_id: str, - non_prod_granule_id: str) -> str: - """ Select either the production or non-production granule concept ID - based on the environment against which the tests are being run. +def get_test_granule_id( + environment: Environment, production_granule_id: str, non_prod_granule_id: str +) -> str: + """Select either the production or non-production granule concept ID + based on the environment against which the tests are being run. """ if environment == Environment.PROD: @@ -26,11 +27,12 @@ def get_test_granule_id(environment: Environment, production_granule_id: str, return granule_id -def make_request_and_download_result(harmony_client: Client, request: Request, - output_directory: str): - """ Use the supplied Harmony client (e.g., Prod, UAT, SIT) to make the - specified request. Wait for the request to complete, and then save the - output to a user-specified directory. +def make_request_and_download_result( + harmony_client: Client, request: Request, output_directory: str +): + """Use the supplied Harmony client (e.g., Prod, UAT, SIT) to make the + specified request. Wait for the request to complete, and then save the + output to a user-specified directory. """ try: @@ -38,23 +40,25 @@ def make_request_and_download_result(harmony_client: Client, request: Request, harmony_client.wait_for_processing(job_id, show_progress=True) makedirs(output_directory, exist_ok=True) - return [result.result() - for result - in harmony_client.download_all(job_id, output_directory, - overwrite=True)] + return [ + result.result() + for result in harmony_client.download_all( + job_id, output_directory, overwrite=True + ) + ] except Exception as exception: print(f'Could not download granule: {str(exception)}') raise exception def clean_test_artefacts(output_directory: str): - """ Remove test outputs between tests. """ + """Remove test outputs between tests.""" rmtree(output_directory) def check_request_output(output_path: str, expected: Dict): - """ Extract information from a Harmony GDAL Adapter request output - and compare to the expected results. + """Extract information from a Harmony GDAL Adapter request output + and compare to the expected results. """ if output_path.endswith(('.nc', '.nc4')): @@ -71,9 +75,12 @@ def check_request_output(output_path: str, expected: Dict): assert output['gcs_epsg'] == expected['gcs_epsg'], f'{base_error} GCS EPSG' assert output['authority'] == expected['authority'], f'{base_error} authority' if output['spatial_extent'] is not None: - assert_almost_equal(output['spatial_extent'], - expected['spatial_extent'], decimal=2, - err_msg=f'{base_error} spatial extent') + assert_almost_equal( + output['spatial_extent'], + expected['spatial_extent'], + decimal=2, + err_msg=f'{base_error} spatial extent', + ) assert output['n_bands'] == expected['n_bands'], f'{base_error} number of bands' assert output['variables'] == expected['variables'], f'{base_error} set of variable' @@ -81,9 +88,9 @@ def check_request_output(output_path: str, expected: Dict): assert output['width'] == expected['width'], f'{base_error} width' if 'file_type' in expected: - assert what_file_type(output_path) == expected['file_type'], ( - f'{base_error} file type' - ) + assert ( + what_file_type(output_path) == expected['file_type'] + ), f'{base_error} file type' if 'reference_image' in expected: assert compare_to_reference_image( @@ -94,9 +101,9 @@ def check_request_output(output_path: str, expected: Dict): def get_netcdf_information(file_path: str): - """ Extract information from a NetCDF-4 output file. The spatial extents - for projected outputs are not converted to geographic coordinates in - this function. + """Extract information from a NetCDF-4 output file. The spatial extents + for projected outputs are not converted to geographic coordinates in + this function. """ with NetCDF4Dataset(file_path) as dataset: @@ -129,21 +136,23 @@ def get_netcdf_information(file_path: str): min_y = None max_y = None - return {'cs': None, - 'proj_cs': None, - 'gcs': None, - 'authority': None, - 'proj_epsg': None, - 'gcs_epsg': None, - 'spatial_extent': [min_y, max_y, min_x, max_x], - 'variables': variable_set, - 'n_bands': bands, - 'height': height, - 'width': width} + return { + 'cs': None, + 'proj_cs': None, + 'gcs': None, + 'authority': None, + 'proj_epsg': None, + 'gcs_epsg': None, + 'spatial_extent': [min_y, max_y, min_x, max_x], + 'variables': variable_set, + 'n_bands': bands, + 'height': height, + 'width': width, + } def get_geotiff_information(file_path: str): - """ Read information from a specified GeoTIFF file. """ + """Read information from a specified GeoTIFF file.""" dataset = GdalOpen(file_path) bands = dataset.RasterCount width = dataset.RasterXSize @@ -151,11 +160,17 @@ def get_geotiff_information(file_path: str): geo_transform = dataset.GetGeoTransform() minx = geo_transform[0] - miny = (geo_transform[3] + dataset.RasterXSize * geo_transform[4] - + dataset.RasterYSize * geo_transform[5]) - maxx = (geo_transform[0] + dataset.RasterXSize * geo_transform[1] - + dataset.RasterYSize * geo_transform[2]) - maxy = (geo_transform[3]) + miny = ( + geo_transform[3] + + dataset.RasterXSize * geo_transform[4] + + dataset.RasterYSize * geo_transform[5] + ) + maxx = ( + geo_transform[0] + + dataset.RasterXSize * geo_transform[1] + + dataset.RasterYSize * geo_transform[2] + ) + maxy = geo_transform[3] proj = SpatialReference(wkt=dataset.GetProjection()) gcs = proj.GetAttrValue('GEOGCS', 0) @@ -171,15 +186,23 @@ def get_geotiff_information(file_path: str): transformer = Transformer.from_crs(f'epsg:{proj_epsg}', 'epsg:4326') min_extent = transformer.transform(minx, miny) max_extent = transformer.transform(maxx, maxy) - spatial_extent = [round(min_extent[0], 2), round(max_extent[0], 2), - round(min_extent[1], 2), round(max_extent[1], 2)] + spatial_extent = [ + round(min_extent[0], 2), + round(max_extent[0], 2), + round(min_extent[1], 2), + round(max_extent[1], 2), + ] elif proj.IsProjected() == 0 and proj.IsGeographic() == 1: cs = 'Geographic' proj_cs = None proj_epsg = None gcs_epsg = proj.GetAttrValue('AUTHORITY', 1) - spatial_extent = [round(miny, 2), round(maxy, 2), round(minx, 2), - round(maxx, 2)] + spatial_extent = [ + round(miny, 2), + round(maxy, 2), + round(minx, 2), + round(maxx, 2), + ] else: cs = None proj_cs = None @@ -192,35 +215,41 @@ def get_geotiff_information(file_path: str): gdinfo = GdalInfo(file_path) if file_path.endswith(('.tiff', '.tif', '.png')): - variable_set = {gdinfo_line.strip().partition('= ')[2] - for gdinfo_line in gdinfo.split('\n') - if 'Description' in gdinfo_line} + variable_set = { + gdinfo_line.strip().partition('= ')[2] + for gdinfo_line in gdinfo.split('\n') + if 'Description' in gdinfo_line + } if len(variable_set) == 0: - variable_set = {f'{gdinfo_line[:4]}{gdinfo_line[5]}' - for gdinfo_line in gdinfo.split('\n') - if 'Block=' in gdinfo_line} + variable_set = { + f'{gdinfo_line[:4]}{gdinfo_line[5]}' + for gdinfo_line in gdinfo.split('\n') + if 'Block=' in gdinfo_line + } else: variable_set = {} gdinfo = None - return {'cs': cs, - 'proj_cs': proj_cs, - 'gcs': gcs, - 'authority': authority, - 'proj_epsg': proj_epsg, - 'gcs_epsg': gcs_epsg, - 'spatial_extent': spatial_extent, - 'variables': variable_set, - 'n_bands': bands, - 'height': height, - 'width': width} + return { + 'cs': cs, + 'proj_cs': proj_cs, + 'gcs': gcs, + 'authority': authority, + 'proj_epsg': proj_epsg, + 'gcs_epsg': gcs_epsg, + 'spatial_extent': spatial_extent, + 'variables': variable_set, + 'n_bands': bands, + 'height': height, + 'width': width, + } def compare_to_reference_image(test_output: str, reference_image: str) -> bool: - """ Compare the array values of the output results to those of a reference - image. + """Compare the array values of the output results to those of a reference + image. """ if test_output.endswith(('.nc', '.nc4')): @@ -240,28 +269,31 @@ def compare_to_reference_image(test_output: str, reference_image: str) -> bool: def compare_netcdf_files(test_output: str, reference_image: str) -> bool: - """ Check all variables from the reference image are in the test output - and have the same value. This function assumes the NetCDF-4 file to be - flat. + """Check all variables from the reference image are in the test output + and have the same value. This function assumes the NetCDF-4 file to be + flat. """ - with NetCDF4Dataset(test_output) as output_ds, \ - NetCDF4Dataset(reference_image) as reference_ds: - - variables_equal = all([ - variable_name in output_ds.variables - and array_equal(output_ds[variable_name][:], ref_variable[:]) - for variable_name, ref_variable in reference_ds.variables.items() - ]) + with NetCDF4Dataset(test_output) as output_ds, NetCDF4Dataset( + reference_image + ) as reference_ds: + + variables_equal = all( + [ + variable_name in output_ds.variables + and array_equal(output_ds[variable_name][:], ref_variable[:]) + for variable_name, ref_variable in reference_ds.variables.items() + ] + ) return variables_equal def print_error(error_string: str) -> str: - """Print an error, with formatting for red text. """ + """Print an error, with formatting for red text.""" print(f'\033[91m{error_string}\033[0m') def print_success(success_string: str) -> str: - """ Print a success message, with formatting for green text. """ + """Print a success message, with formatting for green text.""" print(f'\033[92mSuccess: {success_string}\033[0m') diff --git a/test/hga/version.txt b/test/hga/version.txt index 9faa1b7a..c946ee61 100644 --- a/test/hga/version.txt +++ b/test/hga/version.txt @@ -1 +1 @@ -0.1.5 +0.1.6 diff --git a/test/hoss/HOSS_Regression.ipynb b/test/hoss/HOSS_Regression.ipynb index c86292b4..4cc78224 100644 --- a/test/hoss/HOSS_Regression.ipynb +++ b/test/hoss/HOSS_Regression.ipynb @@ -51,9 +51,13 @@ "\n", "from harmony import BBox, Client, Collection, Dimension, Environment, Request\n", "\n", - "from utilities import (compare_results_to_reference_file, print_success,\n", - " remove_results_files, submit_and_download,\n", - " test_is_configured)" + "from utilities import (\n", + " compare_results_to_reference_file,\n", + " print_success,\n", + " remove_results_files,\n", + " submit_and_download,\n", + " test_is_configured,\n", + ")" ] }, { @@ -95,10 +99,12 @@ "metadata": {}, "outputs": [], "source": [ - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", @@ -140,20 +146,26 @@ "metadata": {}, "outputs": [], "source": [ - "hoss_non_prod_information = {'collection': Collection(id='C1238392622-EEDTEST'),\n", - " 'granule_id': 'G1245840464-EEDTEST',\n", - " 'temporal_collection': Collection(id='C1245662776-EEDTEST'),\n", - " 'temporal_granule_id': 'G1245662797-EEDTEST',\n", - " 'bounds_collection': Collection(id='C1245618475-EEDTEST'),\n", - " 'bounds_granule_id': 'G1255863984-EEDTEST'}\n", + "hoss_non_prod_information = {\n", + " 'collection': Collection(id='C1238392622-EEDTEST'),\n", + " 'granule_id': 'G1245840464-EEDTEST',\n", + " 'temporal_collection': Collection(id='C1245662776-EEDTEST'),\n", + " 'temporal_granule_id': 'G1245662797-EEDTEST',\n", + " 'bounds_collection': Collection(id='C1245618475-EEDTEST'),\n", + " 'bounds_granule_id': 'G1255863984-EEDTEST',\n", + "}\n", "\n", - "hoss_prod_information = {'collection': Collection(id='C1996546500-GHRC_DAAC'),\n", - " 'granule_id': 'G2040739631-GHRC_DAAC'}\n", + "hoss_prod_information = {\n", + " 'collection': Collection(id='C1996546500-GHRC_DAAC'),\n", + " 'granule_id': 'G2040739631-GHRC_DAAC',\n", + "}\n", "\n", - "hoss_env = {Environment.LOCAL: hoss_non_prod_information,\n", - " Environment.SIT: hoss_non_prod_information,\n", - " Environment.UAT: hoss_non_prod_information,\n", - " Environment.PROD: hoss_prod_information}\n", + "hoss_env = {\n", + " Environment.LOCAL: hoss_non_prod_information,\n", + " Environment.SIT: hoss_non_prod_information,\n", + " Environment.UAT: hoss_non_prod_information,\n", + " Environment.PROD: hoss_prod_information,\n", + "}\n", "\n", "if harmony_environment in hoss_env:\n", " hoss_info = hoss_env[harmony_environment]\n", @@ -191,17 +203,21 @@ "if test_is_configured(hoss_info, 'collection'):\n", " hoss_var_bbox_file_name = 'hoss_var_bbox.nc4'\n", " hoss_var_bbox_bbox = BBox(w=-150, s=0, e=-105, n=15)\n", - " hoss_var_bbox_request = Request(collection=hoss_info['collection'],\n", - " granule_id=[hoss_info['granule_id']],\n", - " variables=['atmosphere_cloud_liquid_water_content'],\n", - " spatial=hoss_var_bbox_bbox)\n", + " hoss_var_bbox_request = Request(\n", + " collection=hoss_info['collection'],\n", + " granule_id=[hoss_info['granule_id']],\n", + " variables=['atmosphere_cloud_liquid_water_content'],\n", + " spatial=hoss_var_bbox_bbox,\n", + " )\n", "\n", " submit_and_download(harmony_client, hoss_var_bbox_request, hoss_var_bbox_file_name)\n", - " assert exists(hoss_var_bbox_file_name), 'Unsuccessful HOSS variable, bounding box request.'\n", + " assert exists(\n", + " hoss_var_bbox_file_name\n", + " ), 'Unsuccessful HOSS variable, bounding box request.'\n", "\n", - " compare_results_to_reference_file(hoss_var_bbox_file_name,\n", - " 'reference_files/hoss_var_bbox_reference.nc4',\n", - " '/')\n", + " compare_results_to_reference_file(\n", + " hoss_var_bbox_file_name, 'reference_files/hoss_var_bbox_reference.nc4', '/'\n", + " )\n", "\n", " print_success('HOSS variable and bounding box request.')\n", "else:\n", @@ -240,17 +256,21 @@ "if test_is_configured(hoss_info, 'collection'):\n", " grid_edge_file_name = 'hoss_grid_edge.nc4'\n", " grid_edge_bbox = BBox(w=-15, s=-60, e=15, n=-30)\n", - " grid_edge_request = Request(collection=hoss_info['collection'],\n", - " granule_id=[hoss_info['granule_id']],\n", - " variables=['atmosphere_cloud_liquid_water_content'],\n", - " spatial=grid_edge_bbox)\n", + " grid_edge_request = Request(\n", + " collection=hoss_info['collection'],\n", + " granule_id=[hoss_info['granule_id']],\n", + " variables=['atmosphere_cloud_liquid_water_content'],\n", + " spatial=grid_edge_bbox,\n", + " )\n", "\n", " submit_and_download(harmony_client, grid_edge_request, grid_edge_file_name)\n", - " assert exists(grid_edge_file_name), 'Unsuccessful HOSS request crossing longitudinal edge.'\n", + " assert exists(\n", + " grid_edge_file_name\n", + " ), 'Unsuccessful HOSS request crossing longitudinal edge.'\n", "\n", - " compare_results_to_reference_file(grid_edge_file_name,\n", - " 'reference_files/hoss_grid_edge_reference.nc4',\n", - " '/')\n", + " compare_results_to_reference_file(\n", + " grid_edge_file_name, 'reference_files/hoss_grid_edge_reference.nc4', '/'\n", + " )\n", "\n", " print_success('HOSS request crossing longitudinal edge.')\n", "else:\n", @@ -276,16 +296,18 @@ "source": [ "if test_is_configured(hoss_info, 'collection'):\n", " no_bbox_file_name = 'hoss_no_bbox.nc4'\n", - " no_bbox_request = Request(collection=hoss_info['collection'],\n", - " granule_id=[hoss_info['granule_id']],\n", - " variables=['sst_dtime', 'wind_speed'])\n", + " no_bbox_request = Request(\n", + " collection=hoss_info['collection'],\n", + " granule_id=[hoss_info['granule_id']],\n", + " variables=['sst_dtime', 'wind_speed'],\n", + " )\n", "\n", " submit_and_download(harmony_client, no_bbox_request, no_bbox_file_name)\n", " assert exists(no_bbox_file_name), 'Unsuccessful HOSS request without bounding box.'\n", "\n", - " compare_results_to_reference_file(no_bbox_file_name,\n", - " 'reference_files/hoss_no_bbox_reference.nc4',\n", - " '/')\n", + " compare_results_to_reference_file(\n", + " no_bbox_file_name, 'reference_files/hoss_no_bbox_reference.nc4', '/'\n", + " )\n", "\n", " print_success('HOSS request without bounding box.')\n", "else:\n", @@ -312,16 +334,18 @@ "if test_is_configured(hoss_info, 'collection'):\n", " hoss_all_vars_file_name = 'hoss_all_vars.nc4'\n", " hoss_all_vars_bbox = BBox(w=-150, s=0, e=-105, n=15)\n", - " hoss_all_vars_request = Request(collection=hoss_info['collection'],\n", - " granule_id=[hoss_info['granule_id']],\n", - " spatial=hoss_all_vars_bbox)\n", + " hoss_all_vars_request = Request(\n", + " collection=hoss_info['collection'],\n", + " granule_id=[hoss_info['granule_id']],\n", + " spatial=hoss_all_vars_bbox,\n", + " )\n", "\n", " submit_and_download(harmony_client, hoss_all_vars_request, hoss_all_vars_file_name)\n", " assert exists(hoss_all_vars_file_name), 'Unsuccessful HOSS all-variable request.'\n", "\n", - " compare_results_to_reference_file(hoss_all_vars_file_name,\n", - " 'reference_files/hoss_all_vars_reference.nc4',\n", - " '/')\n", + " compare_results_to_reference_file(\n", + " hoss_all_vars_file_name, 'reference_files/hoss_all_vars_reference.nc4', '/'\n", + " )\n", "\n", " print_success('HOSS all-variable request.')\n", "else:\n", @@ -371,19 +395,22 @@ "source": [ "if test_is_configured(hoss_info, 'temporal_collection'):\n", " hoss_temporal_file_name = 'hoss_temporal.nc4'\n", - " hoss_temporal_request = Request(collection=hoss_info['temporal_collection'],\n", - " granule_id=hoss_info['temporal_granule_id'],\n", - " variables=['H1000'],\n", - " temporal={'start': datetime(2021, 6, 9, 12, 0, 0),\n", - " 'stop': datetime(2021, 6, 9, 16, 0, 0)})\n", - "\n", + " hoss_temporal_request = Request(\n", + " collection=hoss_info['temporal_collection'],\n", + " granule_id=hoss_info['temporal_granule_id'],\n", + " variables=['H1000'],\n", + " temporal={\n", + " 'start': datetime(2021, 6, 9, 12, 0, 0),\n", + " 'stop': datetime(2021, 6, 9, 16, 0, 0),\n", + " },\n", + " )\n", "\n", " submit_and_download(harmony_client, hoss_temporal_request, hoss_temporal_file_name)\n", " assert exists(hoss_temporal_file_name), 'Unsuccessful HOSS temporal request.'\n", "\n", - " compare_results_to_reference_file(hoss_temporal_file_name,\n", - " 'reference_files/hoss_temporal_reference.nc4',\n", - " '/')\n", + " compare_results_to_reference_file(\n", + " hoss_temporal_file_name, 'reference_files/hoss_temporal_reference.nc4', '/'\n", + " )\n", "\n", " print_success('HOSS temporal request.')\n", "else:\n", @@ -411,18 +438,25 @@ "source": [ "if test_is_configured(hoss_info, 'collection'):\n", " hoss_named_dims_file_name = 'hoss_named_dimensions.nc4'\n", - " hoss_named_dims_request = Request(collection=hoss_info['collection'],\n", - " granule_id=[hoss_info['granule_id']],\n", - " variables=['atmosphere_cloud_liquid_water_content'],\n", - " dimensions=[Dimension('latitude', -20, -5),\n", - " Dimension('longitude', 70, 85)])\n", - "\n", - " submit_and_download(harmony_client, hoss_named_dims_request, hoss_named_dims_file_name)\n", - " assert exists(hoss_named_dims_file_name), 'Unsuccessful HOSS named dimensions request.'\n", - "\n", - " compare_results_to_reference_file(hoss_named_dims_file_name,\n", - " 'reference_files/hoss_named_dimensions_reference.nc4',\n", - " '/')\n", + " hoss_named_dims_request = Request(\n", + " collection=hoss_info['collection'],\n", + " granule_id=[hoss_info['granule_id']],\n", + " variables=['atmosphere_cloud_liquid_water_content'],\n", + " dimensions=[Dimension('latitude', -20, -5), Dimension('longitude', 70, 85)],\n", + " )\n", + "\n", + " submit_and_download(\n", + " harmony_client, hoss_named_dims_request, hoss_named_dims_file_name\n", + " )\n", + " assert exists(\n", + " hoss_named_dims_file_name\n", + " ), 'Unsuccessful HOSS named dimensions request.'\n", + "\n", + " compare_results_to_reference_file(\n", + " hoss_named_dims_file_name,\n", + " 'reference_files/hoss_named_dimensions_reference.nc4',\n", + " '/',\n", + " )\n", "\n", " print_success('HOSS named dimensions request.')\n", "else:\n", @@ -446,13 +480,17 @@ "metadata": {}, "outputs": [], "source": [ - "hoss_shape_non_prod_information = {'collection': Collection(id='C1245618475-EEDTEST'),\n", - " 'shape_file_path': 'amazon_basin.geo.json',\n", - " 'granule_id': 'G1245649517-EEDTEST'}\n", + "hoss_shape_non_prod_information = {\n", + " 'collection': Collection(id='C1245618475-EEDTEST'),\n", + " 'shape_file_path': 'amazon_basin.geo.json',\n", + " 'granule_id': 'G1245649517-EEDTEST',\n", + "}\n", "\n", - "hoss_shape_env = {Environment.LOCAL: hoss_shape_non_prod_information,\n", - " Environment.SIT: hoss_shape_non_prod_information,\n", - " Environment.UAT: hoss_shape_non_prod_information}\n", + "hoss_shape_env = {\n", + " Environment.LOCAL: hoss_shape_non_prod_information,\n", + " Environment.SIT: hoss_shape_non_prod_information,\n", + " Environment.UAT: hoss_shape_non_prod_information,\n", + "}\n", "\n", "if harmony_environment in hoss_shape_env:\n", " hoss_shape_info = hoss_shape_env[harmony_environment]\n", @@ -469,20 +507,26 @@ "source": [ "if test_is_configured(hoss_shape_info, 'collection'):\n", " hoss_shape_file_name = 'hoss_shape_file.nc4'\n", - " hoss_shape_request = Request(collection=hoss_shape_info['collection'],\n", - " granule_id=hoss_shape_info['granule_id'],\n", - " shape=hoss_shape_info['shape_file_path'],\n", - " variables=['Grid/precipitationCal'])\n", + " hoss_shape_request = Request(\n", + " collection=hoss_shape_info['collection'],\n", + " granule_id=hoss_shape_info['granule_id'],\n", + " shape=hoss_shape_info['shape_file_path'],\n", + " variables=['Grid/precipitationCal'],\n", + " )\n", " submit_and_download(harmony_client, hoss_shape_request, hoss_shape_file_name)\n", - " assert exists(hoss_shape_file_name), 'Unsuccessful HOSS-Geographic polygon spatial subset request.'\n", + " assert exists(\n", + " hoss_shape_file_name\n", + " ), 'Unsuccessful HOSS-Geographic polygon spatial subset request.'\n", "\n", - " compare_results_to_reference_file(hoss_shape_file_name,\n", - " 'reference_files/hoss_shape_file_reference.nc4',\n", - " '/Grid')\n", + " compare_results_to_reference_file(\n", + " hoss_shape_file_name, 'reference_files/hoss_shape_file_reference.nc4', '/Grid'\n", + " )\n", "\n", " print_success('HOSS-Geographic polygon spatial subset request.')\n", "else:\n", - " print(f'Skipping - HOSS-Geographic is not configured for this test in {harmony_environment}.')" + " print(\n", + " f'Skipping - HOSS-Geographic is not configured for this test in {harmony_environment}.'\n", + " )" ] }, { @@ -505,19 +549,23 @@ "if test_is_configured(hoss_info, 'bounds_collection'):\n", " hoss_bounds_file_name = 'hoss_bounds.nc4'\n", " hoss_bounds_bbox = BBox(w=60, s=-15, e=75, n=0)\n", - " hoss_bounds_request = Request(collection=hoss_info['bounds_collection'],\n", - " granule_id=[hoss_info['bounds_granule_id']],\n", - " variables=['Grid/precipitationCal'],\n", - " spatial=hoss_bounds_bbox,\n", - " temporal={'start': datetime(2020, 1, 18, 18, 45, 0),\n", - " 'stop': datetime(2020, 1, 18, 19, 45, 0)})\n", + " hoss_bounds_request = Request(\n", + " collection=hoss_info['bounds_collection'],\n", + " granule_id=[hoss_info['bounds_granule_id']],\n", + " variables=['Grid/precipitationCal'],\n", + " spatial=hoss_bounds_bbox,\n", + " temporal={\n", + " 'start': datetime(2020, 1, 18, 18, 45, 0),\n", + " 'stop': datetime(2020, 1, 18, 19, 45, 0),\n", + " },\n", + " )\n", "\n", " submit_and_download(harmony_client, hoss_bounds_request, hoss_bounds_file_name)\n", " assert exists(hoss_bounds_file_name), 'Unsuccessful HOSS bounds request.'\n", "\n", - " compare_results_to_reference_file(hoss_bounds_file_name,\n", - " 'reference_files/hoss_bounds_reference.nc4',\n", - " '/Grid')\n", + " compare_results_to_reference_file(\n", + " hoss_bounds_file_name, 'reference_files/hoss_bounds_reference.nc4', '/Grid'\n", + " )\n", "\n", " print_success('HOSS bounds request.')\n", "else:\n", @@ -560,14 +608,14 @@ " 'granule_id': 'G1245804356-EEDTEST',\n", " 'temporal_range': {\n", " 'start': datetime(2008, 7, 2, 0, 0, 0),\n", - " 'stop': datetime(2008, 7, 2, 1, 0, 0)\n", - " }\n", + " 'stop': datetime(2008, 7, 2, 1, 0, 0),\n", + " },\n", "}\n", "\n", "hoss_projected_env = {\n", " Environment.LOCAL: hoss_projected_non_prod_information,\n", " Environment.SIT: hoss_projected_non_prod_information,\n", - " Environment.UAT: hoss_projected_non_prod_information\n", + " Environment.UAT: hoss_projected_non_prod_information,\n", "}\n", "\n", "if harmony_environment in hoss_projected_env:\n", @@ -585,21 +633,27 @@ "source": [ "if test_is_configured(hoss_projected_info, 'collection'):\n", " hoss_projected_filename = 'hoss_projected_north_slope.nc4'\n", - " hoss_projected_request = Request(collection=hoss_projected_info['collection'],\n", - " granule_id=hoss_projected_info['granule_id'],\n", - " spatial=hoss_projected_info['bbox'],\n", - " variables=['NEE', 'lat', 'lon'],\n", - " temporal=hoss_projected_info['temporal_range'])\n", + " hoss_projected_request = Request(\n", + " collection=hoss_projected_info['collection'],\n", + " granule_id=hoss_projected_info['granule_id'],\n", + " spatial=hoss_projected_info['bbox'],\n", + " variables=['NEE', 'lat', 'lon'],\n", + " temporal=hoss_projected_info['temporal_range'],\n", + " )\n", " submit_and_download(harmony_client, hoss_projected_request, hoss_projected_filename)\n", " assert exists(hoss_projected_filename), 'Unsuccessful HOSS spatial subset request.'\n", "\n", - " compare_results_to_reference_file(hoss_projected_filename,\n", - " 'reference_files/hoss_projected_north_slope_reference.nc4',\n", - " '/')\n", + " compare_results_to_reference_file(\n", + " hoss_projected_filename,\n", + " 'reference_files/hoss_projected_north_slope_reference.nc4',\n", + " '/',\n", + " )\n", "\n", " print_success('HOSS-projection-gridded with bounding box.')\n", "else:\n", - " print(f'Skipping - HOSS Projection-Gridded is not configured for this test in {harmony_environment}.')" + " print(\n", + " f'Skipping - HOSS Projection-Gridded is not configured for this test in {harmony_environment}.'\n", + " )" ] }, { @@ -625,17 +679,25 @@ "source": [ "if test_is_configured(hoss_projected_info, 'collection'):\n", " hoss_projected_shape_filename = 'hoss_projected_north_slope_shape.nc4'\n", - " hoss_projected_request = Request(collection=hoss_projected_info['collection'],\n", - " granule_id=hoss_projected_info['granule_id'],\n", - " shape=hoss_projected_info['shape_file_path'],\n", - " variables=['NEE', 'lat', 'lon'],\n", - " temporal=hoss_projected_info['temporal_range'])\n", - " submit_and_download(harmony_client, hoss_projected_request, hoss_projected_shape_filename)\n", - " assert exists(hoss_projected_shape_filename), 'Unsuccessful HOSS shapefile spatial subset request.'\n", - "\n", - " compare_results_to_reference_file(hoss_projected_shape_filename,\n", - " 'reference_files/hoss_projected_north_slope_reference.nc4',\n", - " '/')\n", + " hoss_projected_request = Request(\n", + " collection=hoss_projected_info['collection'],\n", + " granule_id=hoss_projected_info['granule_id'],\n", + " shape=hoss_projected_info['shape_file_path'],\n", + " variables=['NEE', 'lat', 'lon'],\n", + " temporal=hoss_projected_info['temporal_range'],\n", + " )\n", + " submit_and_download(\n", + " harmony_client, hoss_projected_request, hoss_projected_shape_filename\n", + " )\n", + " assert exists(\n", + " hoss_projected_shape_filename\n", + " ), 'Unsuccessful HOSS shapefile spatial subset request.'\n", + "\n", + " compare_results_to_reference_file(\n", + " hoss_projected_shape_filename,\n", + " 'reference_files/hoss_projected_north_slope_reference.nc4',\n", + " '/',\n", + " )\n", "\n", " print_success('Subsetting projected grid with shapefile.')\n", "else:\n", diff --git a/test/hoss/utilities.py b/test/hoss/utilities.py index bc101c43..49f3fe06 100644 --- a/test/hoss/utilities.py +++ b/test/hoss/utilities.py @@ -3,6 +3,7 @@ notebook to increase the readability of the regression test suite. """ + from os import listdir, remove, replace from typing import Dict, Optional @@ -11,21 +12,22 @@ import xarray as xr -def compare_results_to_reference_file(results_file_name: str, - reference_file_name: str, - group_name: Optional[str]) -> None: - """ Use native `xarray` functionality to compare data values and metadata - attributes of downloaded results to a reference file. +def compare_results_to_reference_file( + results_file_name: str, reference_file_name: str, group_name: Optional[str] +) -> None: + """Use native `xarray` functionality to compare data values and metadata + attributes of downloaded results to a reference file. - If future tests require checks of hierarchical output with multiple - groups, see same-named function in `variable-subsetter/utilities.py`. + If future tests require checks of hierarchical output with multiple + groups, see same-named function in `variable-subsetter/utilities.py`. """ - reference_data = xr.open_dataset(reference_file_name, group=group_name) + reference_data = xr.open_dataset(reference_file_name, group=group_name) results_data = xr.open_dataset(results_file_name, group=group_name) - assert results_data.equals(reference_data), ('Output and reference files ' - 'do not match.') + assert results_data.equals(reference_data), ( + 'Output and reference files ' 'do not match.' + ) reference_data.close() results_data.close() @@ -33,11 +35,12 @@ def compare_results_to_reference_file(results_file_name: str, results_data = None -def submit_and_download(harmony_client: Client, request: Request, - output_file_name: str): - """ Submit a Harmony request via a `harmony-py` client. Wait for the - Harmony job to finish, then download the results to the specified file - path. +def submit_and_download( + harmony_client: Client, request: Request, output_file_name: str +): + """Submit a Harmony request via a `harmony-py` client. Wait for the + Harmony job to finish, then download the results to the specified file + path. """ downloaded_filename = None @@ -45,10 +48,10 @@ def submit_and_download(harmony_client: Client, request: Request, try: job_id = harmony_client.submit(request) - for filename in [file_future.result() - for file_future - in harmony_client.download_all(job_id, - overwrite=True)]: + for filename in [ + file_future.result() + for file_future in harmony_client.download_all(job_id, overwrite=True) + ]: print(f'Downloaded: {filename}') downloaded_filename = filename @@ -63,8 +66,8 @@ def submit_and_download(harmony_client: Client, request: Request, def remove_results_files() -> None: - """ Remove all NetCDF-4 files downloaded during the Swath Projector - regression tests. + """Remove all NetCDF-4 files downloaded during the Swath Projector + regression tests. """ directory_files = listdir() @@ -75,23 +78,22 @@ def remove_results_files() -> None: def print_error(error_string: str) -> str: - """Print an error, with formatting for red text. """ + """Print an error, with formatting for red text.""" print(f'\033[91m{error_string}\033[0m') def print_success(success_string: str) -> str: - """ Print a success message, with formatting for green text. """ + """Print a success message, with formatting for green text.""" print(f'\033[92mSuccess: {success_string}\033[0m') -def test_is_configured(configuration_settings: Dict, - collection_key: str) -> bool: - """ A helper function to determine if a test should be run given the - environment information available. +def test_is_configured(configuration_settings: Dict, collection_key: str) -> bool: + """A helper function to determine if a test should be run given the + environment information available. - The specific check is whether the configuration object exists and, if - so, if the collection that will be used in the test has been saved in - that configuration object. + The specific check is whether the configuration object exists and, if + so, if the collection that will be used in the test has been saved in + that configuration object. """ return ( diff --git a/test/hoss/version.txt b/test/hoss/version.txt index 11808190..699c6c6d 100644 --- a/test/hoss/version.txt +++ b/test/hoss/version.txt @@ -1 +1 @@ -0.1.7 +0.1.8 diff --git a/test/hybig/HyBIG_Regression.ipynb b/test/hybig/HyBIG_Regression.ipynb index b74f7834..2e918334 100644 --- a/test/hybig/HyBIG_Regression.ipynb +++ b/test/hybig/HyBIG_Regression.ipynb @@ -81,7 +81,12 @@ "from rasterio.transform import Affine\n", "from rasterio.crs import CRS\n", "\n", - "from utility import print_success, print_error, assert_dataset_produced_correct_results, build_file_list\n", + "from utility import (\n", + " print_success,\n", + " print_error,\n", + " assert_dataset_produced_correct_results,\n", + " build_file_list,\n", + ")\n", "\n", "\n", "reference_dir = Path('./reference_data')" @@ -108,29 +113,29 @@ "source": [ "non_production_collection = {\n", " 'aster_collection': Collection(id='C1256584478-EEDTEST'),\n", - " 'measures_collection': Collection(id='C1258119317-EEDTEST')\n", + " 'measures_collection': Collection(id='C1258119317-EEDTEST'),\n", "}\n", "\n", "non_prod_granule_data = {\n", " 'aster_granules': ['G1256584570-EEDTEST'],\n", - " 'measures_granules': ['G1258119387-EEDTEST']\n", + " 'measures_granules': ['G1258119387-EEDTEST'],\n", "}\n", "\n", "collection_data = {\n", " 'https://harmony.uat.earthdata.nasa.gov': {\n", " **non_production_collection,\n", " **non_prod_granule_data,\n", - " 'env': Environment.UAT\n", + " 'env': Environment.UAT,\n", " },\n", " 'https://harmony.sit.earthdata.nasa.gov': {\n", " **non_production_collection,\n", " **non_prod_granule_data,\n", - " 'env': Environment.SIT\n", + " 'env': Environment.SIT,\n", " },\n", " 'http://localhost:3000': {\n", " **non_production_collection,\n", " **non_prod_granule_data,\n", - " 'env': Environment.LOCAL\n", + " 'env': Environment.LOCAL,\n", " },\n", "}\n", "\n", @@ -164,9 +169,11 @@ " 'height': 3641,\n", " 'count': 1,\n", " 'crs': CRS.from_epsg(4326),\n", - " 'transform': Affine(0.00027464982147761604, 0.0, 22.0, 0.0, -0.00027464982147761604, 1.0)\n", + " 'transform': Affine(\n", + " 0.00027464982147761604, 0.0, 22.0, 0.0, -0.00027464982147761604, 1.0\n", + " ),\n", "}\n", - "aster_basename = 'ASTGTMV003_N00E022_dem'\n" + "aster_basename = 'ASTGTMV003_N00E022_dem'" ] }, { @@ -193,13 +200,15 @@ "source": [ "if environment_information is not None:\n", "\n", - " scale_extent = [22, 0, 23, 1] \n", + " scale_extent = [22, 0, 23, 1]\n", "\n", - " aster_request = Request(collection=environment_information['aster_collection'],\n", - " granule_id=environment_information['aster_granules'][0],\n", - " scale_extent=scale_extent,\n", - " crs='EPSG:4326',\n", - " format='image/png')\n", + " aster_request = Request(\n", + " collection=environment_information['aster_collection'],\n", + " granule_id=environment_information['aster_granules'][0],\n", + " scale_extent=scale_extent,\n", + " crs='EPSG:4326',\n", + " format='image/png',\n", + " )\n", "\n", " aster_job_id = harmony_client.submit(aster_request)\n", " harmony_client.wait_for_processing(aster_job_id, show_progress=True)\n", @@ -209,17 +218,23 @@ " with TemporaryDirectory() as temp_dir:\n", " downloaded_grid_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(aster_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " aster_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " test_files = build_file_list(aster_basename, Path(temp_dir), 'PNG')\n", " for file_name in test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", " print_success('all test files generated')\n", "\n", " expected_output_metadata = {\n", - " **common_aster_metadata, 'driver': 'PNG', 'count': 1, 'nodata': 255\n", + " **common_aster_metadata,\n", + " 'driver': 'PNG',\n", + " 'count': 1,\n", + " 'nodata': 255,\n", " }\n", "\n", " assert_dataset_produced_correct_results(\n", @@ -250,13 +265,15 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " \n", + "\n", " scale_extent = [22, 0, 23, 1]\n", - " aster_request = Request(collection=environment_information['aster_collection'],\n", - " granule_id=environment_information['aster_granules'][0],\n", - " scale_extent=scale_extent,\n", - " crs='EPSG:4326',\n", - " format='image/jpeg')\n", + " aster_request = Request(\n", + " collection=environment_information['aster_collection'],\n", + " granule_id=environment_information['aster_granules'][0],\n", + " scale_extent=scale_extent,\n", + " crs='EPSG:4326',\n", + " format='image/jpeg',\n", + " )\n", "\n", " aster_job_id = harmony_client.submit(aster_request)\n", " harmony_client.wait_for_processing(aster_job_id, show_progress=True)\n", @@ -266,18 +283,23 @@ " with TemporaryDirectory() as temp_dir:\n", " downloaded_grid_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(aster_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " aster_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " test_files = build_file_list(aster_basename, Path(temp_dir), 'JPEG')\n", " for file_name in test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", " print_success('all test files generated')\n", "\n", " expected_output_metadata = {\n", - " 'driver': 'JPEG', **common_aster_metadata, 'count': 3\n", - " } \n", + " 'driver': 'JPEG',\n", + " **common_aster_metadata,\n", + " 'count': 3,\n", + " }\n", "\n", " assert_dataset_produced_correct_results(\n", " test_files[0], expected_output_metadata, reference_files[0], 'JPEG'\n", @@ -313,21 +335,25 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " \n", + "\n", " reversed_xscale_extent = [23, 0, 22, 1]\n", - " aster_request = Request(collection=environment_information['aster_collection'],\n", - " granule_id=environment_information['aster_granules'][0],\n", - " scale_extent=reversed_xscale_extent,\n", - " crs='EPSG:4326',\n", - " format='image/jpeg')\n", + " aster_request = Request(\n", + " collection=environment_information['aster_collection'],\n", + " granule_id=environment_information['aster_granules'][0],\n", + " scale_extent=reversed_xscale_extent,\n", + " crs='EPSG:4326',\n", + " format='image/jpeg',\n", + " )\n", "\n", " aster_job_id = harmony_client.submit(aster_request)\n", - " try: \n", + " try:\n", " harmony_client.wait_for_processing(aster_job_id, show_progress=True)\n", " print_error('Exception not rasied for bad Scale Extents')\n", " assert False, 'Fail Scale Extent test'\n", " except Exception as exception:\n", - " assert 'Harmony ScaleExtents must be in order [xmin,ymin,xmax,ymax]' in str(exception), 'Exception not raised correctly'\n", + " assert 'Harmony ScaleExtents must be in order [xmin,ymin,xmax,ymax]' in str(\n", + " exception\n", + " ), 'Exception not raised correctly'\n", " print_success('Exception Raised Correctly for Scale Extents')\n", "else:\n", " print('Skipping test: HyBIG regression tests not configured for this environment.')" @@ -349,21 +375,25 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " \n", - " scale_size = [.002, -.002]\n", - " aster_request = Request(collection=environment_information['aster_collection'],\n", - " granule_id=environment_information['aster_granules'][0],\n", - " scale_size=scale_size,\n", - " crs='EPSG:4326',\n", - " format='image/jpeg')\n", + "\n", + " scale_size = [0.002, -0.002]\n", + " aster_request = Request(\n", + " collection=environment_information['aster_collection'],\n", + " granule_id=environment_information['aster_granules'][0],\n", + " scale_size=scale_size,\n", + " crs='EPSG:4326',\n", + " format='image/jpeg',\n", + " )\n", "\n", " try:\n", - " aster_job_id = harmony_client.submit(aster_request) \n", + " aster_job_id = harmony_client.submit(aster_request)\n", " harmony_client.wait_for_processing(aster_job_id, show_progress=True)\n", " print_error('Exception not rasied for negative Scale Size.')\n", " assert False, 'Fail Scale sizes test'\n", " except Exception as exception:\n", - " assert '\"scaleSize[1]\" should be >= 0' in str(exception), 'Exception not raised correctly'\n", + " assert '\"scaleSize[1]\" should be >= 0' in str(\n", + " exception\n", + " ), 'Exception not raised correctly'\n", " print_success('Exception raised correctly for negative Scale Size.')\n", "else:\n", " print('Skipping test: HyBIG regression tests not configured for this environment.')" @@ -392,7 +422,7 @@ " 'width': 7200,\n", " 'height': 3600,\n", " 'crs': CRS.from_epsg(4326),\n", - " 'transform': Affine(0.05, 0.0, -180.0, 0.0, -0.05, 90.0)\n", + " 'transform': Affine(0.05, 0.0, -180.0, 0.0, -0.05, 90.0),\n", "}\n", "measures_basename = 'VCF5KYR_1991001_001_2018224205008'" ] @@ -417,9 +447,11 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " measures_request = Request(collection=environment_information['measures_collection'],\n", - " granule_id=environment_information['measures_granules'][0],\n", - " format='image/png')\n", + " measures_request = Request(\n", + " collection=environment_information['measures_collection'],\n", + " granule_id=environment_information['measures_granules'][0],\n", + " format='image/png',\n", + " )\n", "\n", " measures_job_id = harmony_client.submit(measures_request)\n", " harmony_client.wait_for_processing(measures_job_id, show_progress=True)\n", @@ -429,18 +461,23 @@ " with TemporaryDirectory() as temp_dir:\n", " downloaded_grid_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(measures_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " measures_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " test_files = build_file_list(measures_basename, Path(temp_dir), 'PNG')\n", " for file_name in test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", "\n", " print_success('all test files generated')\n", "\n", " expected_output_metadata = {\n", - " **common_measures_metadata, 'driver': 'PNG', 'count': 1 \n", + " **common_measures_metadata,\n", + " 'driver': 'PNG',\n", + " 'count': 1,\n", " }\n", "\n", " assert_dataset_produced_correct_results(\n", @@ -470,9 +507,11 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " measures_request = Request(collection=environment_information['measures_collection'],\n", - " granule_id=environment_information['measures_granules'][0],\n", - " format='image/jpeg')\n", + " measures_request = Request(\n", + " collection=environment_information['measures_collection'],\n", + " granule_id=environment_information['measures_granules'][0],\n", + " format='image/jpeg',\n", + " )\n", "\n", " measures_job_id = harmony_client.submit(measures_request)\n", " harmony_client.wait_for_processing(measures_job_id, show_progress=True)\n", @@ -482,17 +521,23 @@ " with TemporaryDirectory() as temp_dir:\n", " downloaded_grid_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(measures_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " measures_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " test_files = build_file_list(measures_basename, Path(temp_dir), 'JPG')\n", " for file_name in test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", " print_success('all test files generated')\n", "\n", " expected_output_metadata = {\n", - " **common_measures_metadata, 'driver': 'JPEG','count': 3, 'nodata': None,\n", + " **common_measures_metadata,\n", + " 'driver': 'JPEG',\n", + " 'count': 3,\n", + " 'nodata': None,\n", " }\n", "\n", " assert_dataset_produced_correct_results(\n", @@ -529,27 +574,35 @@ " scale_sizes = [1.0, 2.0]\n", " scale_size_reference_dir = reference_dir / 'scale_size'\n", "\n", - " scale_size_request = Request(collection=environment_information['measures_collection'],\n", - " granule_id=environment_information['measures_granules'][0],\n", - " scale_size=scale_sizes,\n", - " crs='EPSG:4326', format='image/png')\n", + " scale_size_request = Request(\n", + " collection=environment_information['measures_collection'],\n", + " granule_id=environment_information['measures_granules'][0],\n", + " scale_size=scale_sizes,\n", + " crs='EPSG:4326',\n", + " format='image/png',\n", + " )\n", "\n", " scale_size_job_id = harmony_client.submit(scale_size_request)\n", " harmony_client.wait_for_processing(scale_size_job_id, show_progress=True)\n", "\n", - " reference_files = build_file_list(measures_basename, scale_size_reference_dir, 'PNG')\n", + " reference_files = build_file_list(\n", + " measures_basename, scale_size_reference_dir, 'PNG'\n", + " )\n", "\n", " with TemporaryDirectory() as temp_dir:\n", " downloaded_scale_size_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(scale_size_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " scale_size_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " test_files = build_file_list(measures_basename, Path(temp_dir), 'PNG')\n", "\n", " for file_name in test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", "\n", " print_success('all test files generated')\n", "\n", @@ -559,7 +612,7 @@ " 'count': 1,\n", " 'transform': Affine(1.0, 0.0, -180.0, 0.0, -2.0, 90.0),\n", " 'height': 90,\n", - " 'width': 360\n", + " 'width': 360,\n", " }\n", "\n", " assert_dataset_produced_correct_results(\n", @@ -591,26 +644,35 @@ "if environment_information is not None:\n", " dimensions_reference_dir = reference_dir / 'dimensions'\n", "\n", - " dimensions_request = Request(collection=environment_information['measures_collection'],\n", - " granule_id=environment_information['measures_granules'][0],\n", - " height=180, width=180, format='image/png')\n", + " dimensions_request = Request(\n", + " collection=environment_information['measures_collection'],\n", + " granule_id=environment_information['measures_granules'][0],\n", + " height=180,\n", + " width=180,\n", + " format='image/png',\n", + " )\n", "\n", " dimensions_job_id = harmony_client.submit(dimensions_request)\n", " harmony_client.wait_for_processing(dimensions_job_id, show_progress=True)\n", "\n", - " reference_files = build_file_list(measures_basename, dimensions_reference_dir, 'PNG')\n", + " reference_files = build_file_list(\n", + " measures_basename, dimensions_reference_dir, 'PNG'\n", + " )\n", "\n", " with TemporaryDirectory() as temp_dir:\n", " downloaded_dimensions_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(dimensions_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " dimensions_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " test_files = build_file_list(measures_basename, Path(temp_dir), 'PNG')\n", "\n", " for file_name in test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", "\n", " print_success('all test files generated')\n", "\n", @@ -620,14 +682,16 @@ " 'count': 1,\n", " 'transform': Affine(2.0, 0.0, -180.0, 0.0, -1.0, 90.0),\n", " 'height': 180,\n", - " 'width': 180\n", + " 'width': 180,\n", " }\n", "\n", " assert_dataset_produced_correct_results(\n", " test_files[0], expected_output_metadata, reference_files[0], 'PNG'\n", " )\n", "\n", - " print_success('Conversion of MEaSUREs GeoTIFF to PNG specifying dimensions. Success')\n", + " print_success(\n", + " 'Conversion of MEaSUREs GeoTIFF to PNG specifying dimensions. Success'\n", + " )\n", "else:\n", " print('Skipping test: HyBIG regression tests not configured for this environment.')" ] @@ -692,10 +756,14 @@ " iceland_scale_size = [0.001, 0.0029]\n", " tiled_reference_dir = reference_dir / 'tiled'\n", "\n", - " tiled_request = Request(collection=environment_information['measures_collection'],\n", - " granule_id=environment_information['measures_granules'][0],\n", - " scale_extent=iceland_extent, scale_size=iceland_scale_size,\n", - " crs='EPSG:4326', format='image/png')\n", + " tiled_request = Request(\n", + " collection=environment_information['measures_collection'],\n", + " granule_id=environment_information['measures_granules'][0],\n", + " scale_extent=iceland_extent,\n", + " scale_size=iceland_scale_size,\n", + " crs='EPSG:4326',\n", + " format='image/png',\n", + " )\n", "\n", " tiled_job_id = harmony_client.submit(tiled_request)\n", " harmony_client.wait_for_processing(tiled_job_id, show_progress=True)\n", @@ -712,29 +780,29 @@ " 'driver': 'PNG',\n", " 'count': 1,\n", " 'height': 3448,\n", - " 'width': 4096\n", + " 'width': 4096,\n", " }\n", "\n", " expected_tile_0_metadata = {\n", " **common_output_metadata,\n", - " 'transform': Affine(0.001, 0.0, -30.0, 0.0, -0.002900232018561485, 70.0)\n", + " 'transform': Affine(0.001, 0.0, -30.0, 0.0, -0.002900232018561485, 70.0),\n", " }\n", " expected_tile_1_metadata = {\n", " **common_output_metadata,\n", - " 'transform': Affine(0.001, 0.0, -25.904, 0.0, -0.002900232018561485, 70.0)\n", + " 'transform': Affine(0.001, 0.0, -25.904, 0.0, -0.002900232018561485, 70.0),\n", " }\n", " expected_tile_2_metadata = {\n", " **common_output_metadata,\n", - " 'transform': Affine(0.001, 0.0, -21.808, 0.0, -0.002900232018561485, 70.0)\n", + " 'transform': Affine(0.001, 0.0, -21.808, 0.0, -0.002900232018561485, 70.0),\n", " }\n", " expected_tile_3_metadata = {\n", " **common_output_metadata,\n", - " 'transform': Affine(0.001, 0.0, -17.712, 0.0, -0.002900232018561485, 70.0)\n", + " 'transform': Affine(0.001, 0.0, -17.712, 0.0, -0.002900232018561485, 70.0),\n", " }\n", " expected_tile_4_metadata = {\n", " **common_output_metadata,\n", " 'transform': Affine(0.001, 0.0, -13.616, 0.0, -0.002900232018561485, 70.0),\n", - " 'width': 3616\n", + " 'width': 3616,\n", " }\n", "\n", " tile_information = [\n", @@ -748,8 +816,9 @@ " with TemporaryDirectory() as temp_dir:\n", " downloaded_tiled_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(tiled_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " tiled_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", "\n", " # Perform the same checks for both tiles:\n", @@ -758,7 +827,9 @@ " tile_test_files = build_file_list(tile_basename, Path(temp_dir), 'PNG')\n", "\n", " for file_name in tile_test_files:\n", - " assert file_name.exists() == True, f'File does not exist {file_name.resolve()}'\n", + " assert (\n", + " file_name.exists() == True\n", + " ), f'File does not exist {file_name.resolve()}'\n", "\n", " print_success(f'All {tile_basename} test files generated')\n", " assert_dataset_produced_correct_results(\n", diff --git a/test/hybig/utility.py b/test/hybig/utility.py index 0c5adbf8..7e052f89 100644 --- a/test/hybig/utility.py +++ b/test/hybig/utility.py @@ -1,4 +1,5 @@ """Simple utility functions used in the regridder test notebook.""" + from pathlib import Path import rasterio from numpy.testing import assert_array_almost_equal @@ -13,6 +14,7 @@ def print_success(success_string: str) -> str: """Print a success message, with formatting for green text.""" print(f'\033[92mSuccess: {success_string}\033[0m') + def assert_dataset_produced_correct_results( generated_file: Path, expected_metadata: dict, reference_file: Path, file_type: str ) -> None: @@ -30,6 +32,7 @@ def assert_dataset_produced_correct_results( print_success('Generated image contains correct data.') + def build_file_list(basename: str, path: Path, file_type: str) -> list[Path]: if file_type == 'PNG': exts = ['.png', '.pgw', '.png.aux.xml'] diff --git a/test/hybig/version.txt b/test/hybig/version.txt index 1750564f..5a5831ab 100644 --- a/test/hybig/version.txt +++ b/test/hybig/version.txt @@ -1 +1 @@ -0.0.6 +0.0.7 diff --git a/test/n2z/N2Z_Regression.ipynb b/test/n2z/N2Z_Regression.ipynb index af9d4af1..718ee65f 100644 --- a/test/n2z/N2Z_Regression.ipynb +++ b/test/n2z/N2Z_Regression.ipynb @@ -77,7 +77,7 @@ " assert_result_has_correct_number_of_stores,\n", " download_zarr_store,\n", " get_zarr_store_location,\n", - " print_success \n", + " print_success,\n", ")\n", "from tempfile import TemporaryDirectory\n", "import xarray as xr" @@ -109,16 +109,16 @@ "collection_data = {\n", " 'https://harmony.uat.earthdata.nasa.gov': {\n", " **non_production_collection,\n", - " 'env': Environment.UAT\n", + " 'env': Environment.UAT,\n", " },\n", " 'https://harmony.sit.earthdata.nasa.gov': {\n", " **non_production_collection,\n", - " 'env': Environment.SIT\n", + " 'env': Environment.SIT,\n", " },\n", " 'http://localhost:3000': {\n", " **non_production_collection,\n", " 'env': Environment.LOCAL,\n", - " 'endpoint_url': 'http://localhost:4566'\n", + " 'endpoint_url': 'http://localhost:4566',\n", " },\n", "}\n", "\n", @@ -147,18 +147,20 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " imrg_request1 = Request(collection=environment_information['imerg_collection'], \n", - " max_results=1,\n", - " concatenate=False,\n", - " format='application/x-zarr')\n", + " imrg_request1 = Request(\n", + " collection=environment_information['imerg_collection'],\n", + " max_results=1,\n", + " concatenate=False,\n", + " format='application/x-zarr',\n", + " )\n", "\n", " job_id = harmony_client.submit(imrg_request1)\n", " harmony_client.wait_for_processing(job_id, show_progress=True)\n", " results1 = harmony_client.result_json(job_id)\n", " assert_result_has_correct_number_of_stores(results1, 1)\n", - " print_success('One granule, not aggregated, creates a single Zarr store.') \n", + " print_success('One granule, not aggregated, creates a single Zarr store.')\n", "else:\n", - " print('Skipping test: N2Z regression tests not configured for this environment.') " + " print('Skipping test: N2Z regression tests not configured for this environment.')" ] }, { @@ -179,10 +181,12 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " imrg_request2 = Request(collection=environment_information['imerg_collection'],\n", - " max_results=3,\n", - " concatenate=False,\n", - " format='application/x-zarr')\n", + " imrg_request2 = Request(\n", + " collection=environment_information['imerg_collection'],\n", + " max_results=3,\n", + " concatenate=False,\n", + " format='application/x-zarr',\n", + " )\n", "\n", " job_id2 = harmony_client.submit(imrg_request2)\n", " harmony_client.wait_for_processing(job_id2, show_progress=True)\n", @@ -211,24 +215,28 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " imrg_request3 = Request(collection=environment_information['imerg_collection'],\n", - " max_results=2,\n", - " concatenate=True,\n", - " format='application/x-zarr')\n", + " imrg_request3 = Request(\n", + " collection=environment_information['imerg_collection'],\n", + " max_results=2,\n", + " concatenate=True,\n", + " format='application/x-zarr',\n", + " )\n", "\n", " job_id3 = harmony_client.submit(imrg_request3)\n", " harmony_client.wait_for_processing(job_id3, show_progress=True)\n", " results3 = harmony_client.result_json(job_id3)\n", " assert_result_has_correct_number_of_stores(results3, 1)\n", - " \n", + "\n", " zarr_store = get_zarr_store_location(results3)\n", " with TemporaryDirectory() as tmp_zarr_store:\n", " download_zarr_store(zarr_store, tmp_zarr_store, endpoint_url)\n", " downloaded_ds = xr.open_dataset(tmp_zarr_store, group='/Grid', engine='zarr')\n", " reference_ds = xr.open_dataset('reference_data/sample.zarr', group='/Grid')\n", - " assert downloaded_ds.equals(reference_ds), 'reference and output Zarr stores did not match'\n", - " \n", - " print_success('Two granules aggregated create a single Zarr store.') " + " assert downloaded_ds.equals(\n", + " reference_ds\n", + " ), 'reference and output Zarr stores did not match'\n", + "\n", + " print_success('Two granules aggregated create a single Zarr store.')" ] }, { @@ -249,15 +257,17 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " imrg_request4 = Request(collection=environment_information['imerg_collection'],\n", - " max_results=2,\n", - " format='application/x-zarr')\n", + " imrg_request4 = Request(\n", + " collection=environment_information['imerg_collection'],\n", + " max_results=2,\n", + " format='application/x-zarr',\n", + " )\n", "\n", " job_id4 = harmony_client.submit(imrg_request4)\n", " harmony_client.wait_for_processing(job_id4, show_progress=True)\n", " results4 = harmony_client.result_json(job_id4)\n", " assert_result_has_correct_number_of_stores(results4, 2)\n", - " print_success('Two granules create two Zarr stores.') " + " print_success('Two granules create two Zarr stores.')" ] } ], diff --git a/test/n2z/reference_data/sample.zarr/.zattrs b/test/n2z/reference_data/sample.zarr/.zattrs index 182210e5..69088a3d 100644 --- a/test/n2z/reference_data/sample.zarr/.zattrs +++ b/test/n2z/reference_data/sample.zarr/.zattrs @@ -1,4 +1,4 @@ { "FileHeader": "DOI=10.5067/GPM/IMERG/3B-HH/06;\nDOIauthority=http://dx.doi.org/;\nDOIshortName=3IMERGHH;\nAlgorithmID=3IMERGHH;\nAlgorithmVersion=3IMERGH_6.3;\nFileName=3B-HHR.MS.MRG.3IMERG.20200101-S003000-E005959.0030.V06B.HDF5;\nSatelliteName=MULTI;\nInstrumentName=MERGED;\nGenerationDateTime=2020-05-04T06:20:22.000Z;\nStartGranuleDateTime=2020-01-01T00:30:00.000Z;\nStopGranuleDateTime=2020-01-01T00:59:59.999Z;\nGranuleNumber=;\nNumberOfSwaths=0;\nNumberOfGrids=1;\nGranuleStart=;\nTimeInterval=HALF_HOUR;\nProcessingSystem=PPS;\nProductVersion=V06B;\nEmptyGranule=NOT_EMPTY;\nMissingData=;\n", "FileInfo": "DataFormatVersion=6a;\nTKCodeBuildVersion=0;\nMetadataVersion=6a;\nFormatPackage=HDF5-1.8.9;\nBlueprintFilename=GPM.V6.3IMERGHH.blueprint.xml;\nBlueprintVersion=BV_62;\nTKIOVersion=3.93;\nMetadataStyle=PVL;\nEndianType=LITTLE_ENDIAN;\n" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/.zgroup b/test/n2z/reference_data/sample.zarr/.zgroup index 3b7daf22..3f3fad2d 100644 --- a/test/n2z/reference_data/sample.zarr/.zgroup +++ b/test/n2z/reference_data/sample.zarr/.zgroup @@ -1,3 +1,3 @@ { "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/.zmetadata b/test/n2z/reference_data/sample.zarr/.zmetadata index 6fa2bb3f..e905482e 100644 --- a/test/n2z/reference_data/sample.zarr/.zmetadata +++ b/test/n2z/reference_data/sample.zarr/.zmetadata @@ -572,4 +572,4 @@ } }, "zarr_consolidated_format": 1 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/.zattrs index 7cb79b97..01550ac4 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/.zattrs @@ -1,3 +1,3 @@ { "GridHeader": "BinMethod=ARITHMETIC_MEAN;\nRegistration=CENTER;\nLatitudeResolution=0.1;\nLongitudeResolution=0.1;\nNorthBoundingCoordinate=90;\nSouthBoundingCoordinate=-90;\nEastBoundingCoordinate=180;\nWestBoundingCoordinate=-180;\nOrigin=SOUTHWEST;\n" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/.zgroup b/test/n2z/reference_data/sample.zarr/Grid/.zgroup index 3b7daf22..3f3fad2d 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/.zgroup +++ b/test/n2z/reference_data/sample.zarr/Grid/.zgroup @@ -1,3 +1,3 @@ { "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zarray b/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zarray index fcd0b062..255c8e54 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zattrs index 33570c9c..690a9afb 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/HQobservationTime/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999, "coordinates": "time lon lat", "units": "minutes" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zarray b/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zarray index 3ccba1e7..ed4a8241 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zattrs index 92808d64..5502ea4a 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/HQprecipSource/.zattrs @@ -8,4 +8,4 @@ ], "_FillValue": -9999, "coordinates": "time lon lat" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zarray b/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zarray index 38a4f6c8..67d8be97 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zattrs index 9b4a8806..30741c9c 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/HQprecipitation/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999.900390625, "coordinates": "time lon lat", "units": "mm/hr" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zarray b/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zarray index 3ccba1e7..ed4a8241 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zattrs index 92808d64..5502ea4a 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/IRkalmanFilterWeight/.zattrs @@ -8,4 +8,4 @@ ], "_FillValue": -9999, "coordinates": "time lon lat" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zarray b/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zarray index 38a4f6c8..67d8be97 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zattrs index 9b4a8806..30741c9c 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/IRprecipitation/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999.900390625, "coordinates": "time lon lat", "units": "mm/hr" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lat/.zarray b/test/n2z/reference_data/sample.zarr/Grid/lat/.zarray index 9627600d..fcfd953b 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lat/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/lat/.zarray @@ -17,4 +17,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lat/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/lat/.zattrs index 86b57ed9..b346a407 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lat/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/lat/.zattrs @@ -9,4 +9,4 @@ "bounds": "lat_bnds", "standard_name": "latitude", "units": "degrees_north" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zarray b/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zarray index 20d4a621..ea08a370 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zarray @@ -19,4 +19,4 @@ 2 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zattrs index 67bef891..332433ea 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/lat_bnds/.zattrs @@ -7,4 +7,4 @@ ], "coordinates": "lat latv", "units": "degrees_north" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lon/.zarray b/test/n2z/reference_data/sample.zarr/Grid/lon/.zarray index ba87a459..5a9ce255 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lon/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/lon/.zarray @@ -17,4 +17,4 @@ 3600 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lon/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/lon/.zattrs index 21299b10..e119f686 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lon/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/lon/.zattrs @@ -9,4 +9,4 @@ "bounds": "lon_bnds", "standard_name": "longitude", "units": "degrees_east" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zarray b/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zarray index a6fb5f4d..408bd3c0 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zarray @@ -19,4 +19,4 @@ 2 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zattrs index a76c6414..e3badd12 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/lon_bnds/.zattrs @@ -7,4 +7,4 @@ ], "coordinates": "lon lonv", "units": "degrees_east" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zarray b/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zarray index 38a4f6c8..67d8be97 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zattrs index 9b4a8806..30741c9c 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/precipitationCal/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999.900390625, "coordinates": "time lon lat", "units": "mm/hr" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zarray b/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zarray index 38a4f6c8..67d8be97 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zattrs index a3b5c463..1c76395b 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/precipitationQualityIndex/.zattrs @@ -8,4 +8,4 @@ ], "_FillValue": -9999.900390625, "coordinates": "time lon lat" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zarray b/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zarray index 38a4f6c8..67d8be97 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zattrs index 9b4a8806..30741c9c 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/precipitationUncal/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999.900390625, "coordinates": "time lon lat", "units": "mm/hr" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zarray b/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zarray index 3ccba1e7..ed4a8241 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zattrs index caaeceb7..79131069 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/probabilityLiquidPrecipitation/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999, "coordinates": "time lon lat", "units": "percent" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/randomError/.zarray b/test/n2z/reference_data/sample.zarr/Grid/randomError/.zarray index 38a4f6c8..67d8be97 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/randomError/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/randomError/.zarray @@ -21,4 +21,4 @@ 1800 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/randomError/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/randomError/.zattrs index 9b4a8806..30741c9c 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/randomError/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/randomError/.zattrs @@ -10,4 +10,4 @@ "_FillValue": -9999.900390625, "coordinates": "time lon lat", "units": "mm/hr" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/time/.zarray b/test/n2z/reference_data/sample.zarr/Grid/time/.zarray index da78cbb8..c0aff712 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/time/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/time/.zarray @@ -17,4 +17,4 @@ 2 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/time/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/time/.zattrs index 0fb7d693..136cffdb 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/time/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/time/.zattrs @@ -10,4 +10,4 @@ "calendar": "julian", "standard_name": "time", "units": "seconds since 1970-01-01 00:00:00 UTC" -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zarray b/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zarray index 1795fc4e..f21ce487 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zarray +++ b/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zarray @@ -19,4 +19,4 @@ 2 ], "zarr_format": 2 -} \ No newline at end of file +} diff --git a/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zattrs b/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zattrs index 9db9eed5..e89ba355 100644 --- a/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zattrs +++ b/test/n2z/reference_data/sample.zarr/Grid/time_bnds/.zattrs @@ -7,4 +7,4 @@ ], "coordinates": "time nv", "units": "seconds since 1970-01-01 00:00:00 UTC" -} \ No newline at end of file +} diff --git a/test/n2z/utility.py b/test/n2z/utility.py index b84fbaea..85938f26 100644 --- a/test/n2z/utility.py +++ b/test/n2z/utility.py @@ -12,13 +12,16 @@ def print_success(success_string: str) -> str: def get_zarr_stores(results: dict) -> List[dict]: """Return all results items that are type "application/x-zarr".""" - return [link - for link in results.get('links', []) - if link.get('type') == 'application/x-zarr'] + return [ + link + for link in results.get('links', []) + if link.get('type') == 'application/x-zarr' + ] -def assert_result_has_correct_number_of_stores(results: dict, - expected_stores: int) -> None: +def assert_result_has_correct_number_of_stores( + results: dict, expected_stores: int +) -> None: """ Verify correct number of zarr stores returned. @@ -36,9 +39,9 @@ def get_zarr_store_location(results: dict) -> str: return zarr_stores[0]['href'] -def download_zarr_store(zarr_s3_url: str, - local_directory: str, - endpoint_url: str = None) -> None: +def download_zarr_store( + zarr_s3_url: str, local_directory: str, endpoint_url: str = None +) -> None: """Download a zarr store from S3 to the desired location. zarr_s3_url - location of the zarr store on s3 diff --git a/test/n2z/version.txt b/test/n2z/version.txt index 9faa1b7a..c946ee61 100644 --- a/test/n2z/version.txt +++ b/test/n2z/version.txt @@ -1 +1 @@ -0.1.5 +0.1.6 diff --git a/test/regridder/Regridder_Regression.ipynb b/test/regridder/Regridder_Regression.ipynb index 3047186d..1859e7c9 100644 --- a/test/regridder/Regridder_Regression.ipynb +++ b/test/regridder/Regridder_Regression.ipynb @@ -71,7 +71,7 @@ "outputs": [], "source": [ "from harmony import Collection, Environment, Client, Request\n", - "from utility import print_success \n", + "from utility import print_success\n", "from tempfile import TemporaryDirectory\n", "import xarray as xr\n", "\n", @@ -99,12 +99,12 @@ "source": [ "non_production_collection = {\n", " 'merra_collection': Collection(id='C1245662776-EEDTEST'),\n", - " 'atl16_collection': Collection(id='C1238589498-EEDTEST')\n", + " 'atl16_collection': Collection(id='C1238589498-EEDTEST'),\n", "}\n", "\n", "non_prod_granule_data = {\n", " 'merra_granules': ['G1245662793-EEDTEST', 'G1245662791-EEDTEST'],\n", - " 'atl16_granules': ['G1245614996-EEDTEST', 'G1245614968-EEDTEST']\n", + " 'atl16_granules': ['G1245614996-EEDTEST', 'G1245614968-EEDTEST'],\n", "}\n", "\n", "\n", @@ -112,17 +112,17 @@ " 'https://harmony.uat.earthdata.nasa.gov': {\n", " **non_production_collection,\n", " **non_prod_granule_data,\n", - " 'env': Environment.UAT\n", + " 'env': Environment.UAT,\n", " },\n", " 'https://harmony.sit.earthdata.nasa.gov': {\n", " **non_production_collection,\n", " **non_prod_granule_data,\n", - " 'env': Environment.SIT\n", + " 'env': Environment.SIT,\n", " },\n", " 'http://localhost:3000': {\n", " **non_production_collection,\n", - " **non_prod_granule_data, \n", - " 'env': Environment.LOCAL\n", + " **non_prod_granule_data,\n", + " 'env': Environment.LOCAL,\n", " },\n", "}\n", "\n", @@ -153,26 +153,41 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " merra_request = Request(collection=environment_information['merra_collection'],\n", - " granule_id=environment_information['merra_granules'][0],\n", - " interpolation='Elliptical Weighted Averaging',\n", - " scale_size=(2.5, 2.5), scale_extent=(-180, -90, 180, 90),\n", - " crs='+proj=latlong +datum=WGS84 +no_defs', height=72, width=144) \n", + " merra_request = Request(\n", + " collection=environment_information['merra_collection'],\n", + " granule_id=environment_information['merra_granules'][0],\n", + " interpolation='Elliptical Weighted Averaging',\n", + " scale_size=(2.5, 2.5),\n", + " scale_extent=(-180, -90, 180, 90),\n", + " crs='+proj=latlong +datum=WGS84 +no_defs',\n", + " height=72,\n", + " width=144,\n", + " )\n", " merra_job_id = harmony_client.submit(merra_request)\n", " harmony_client.wait_for_processing(merra_job_id, show_progress=True)\n", " with TemporaryDirectory() as temp_dir:\n", " downloaded_grid_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(merra_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " merra_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", - " reference_ds = xr.open_dataset('reference_data/MERRA2_400.tavg1_2d_slv_Nx.20210605_regridded.nc4', engine='netcdf4')\n", - " output_ds = xr.open_dataset(Path(temp_dir, downloaded_grid_outputs[0]), engine='netcdf4')\n", - " assert output_ds.equals(reference_ds), 'Generated data did not match reference dataset'\n", + " reference_ds = xr.open_dataset(\n", + " 'reference_data/MERRA2_400.tavg1_2d_slv_Nx.20210605_regridded.nc4',\n", + " engine='netcdf4',\n", + " )\n", + " output_ds = xr.open_dataset(\n", + " Path(temp_dir, downloaded_grid_outputs[0]), engine='netcdf4'\n", + " )\n", + " assert output_ds.equals(\n", + " reference_ds\n", + " ), 'Generated data did not match reference dataset'\n", "\n", " print_success('Regrid MERRA2 Data Success')\n", "else:\n", - " print('Skipping test: regridder regression tests not configured for this environment.')" + " print(\n", + " 'Skipping test: regridder regression tests not configured for this environment.'\n", + " )" ] }, { @@ -197,29 +212,43 @@ "outputs": [], "source": [ "if environment_information is not None:\n", - " atl16_request = Request(collection=environment_information['atl16_collection'],\n", - " granule_id=environment_information['atl16_granules'][0],\n", - " interpolation='Elliptical Weighted Averaging',\n", - " scale_size=(5.0, 5.0),\n", - " scale_extent=(-180, -90, 180, 90),\n", - " crs='EPSG:4326', height=36, width=72) \n", + " atl16_request = Request(\n", + " collection=environment_information['atl16_collection'],\n", + " granule_id=environment_information['atl16_granules'][0],\n", + " interpolation='Elliptical Weighted Averaging',\n", + " scale_size=(5.0, 5.0),\n", + " scale_extent=(-180, -90, 180, 90),\n", + " crs='EPSG:4326',\n", + " height=36,\n", + " width=72,\n", + " )\n", " atl16_job_id = harmony_client.submit(atl16_request)\n", " harmony_client.wait_for_processing(atl16_job_id, show_progress=True)\n", "\n", " with TemporaryDirectory() as temp_dir:\n", " downloaded_grid_outputs = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(atl16_job_id, overwrite=True, directory=temp_dir)\n", + " for file_future in harmony_client.download_all(\n", + " atl16_job_id, overwrite=True, directory=temp_dir\n", + " )\n", " ]\n", - " reference_ds = xr.open_dataset('reference_data/ATL16_20200308000030_11040601_004_01_regridded.h5', engine='netcdf4')\n", - " output_ds = xr.open_dataset(Path(temp_dir, downloaded_grid_outputs[0]), engine='netcdf4')\n", + " reference_ds = xr.open_dataset(\n", + " 'reference_data/ATL16_20200308000030_11040601_004_01_regridded.h5',\n", + " engine='netcdf4',\n", + " )\n", + " output_ds = xr.open_dataset(\n", + " Path(temp_dir, downloaded_grid_outputs[0]), engine='netcdf4'\n", + " )\n", "\n", - " assert output_ds.equals(reference_ds), 'Regridded ATL16 data did not match reference dataset'\n", + " assert output_ds.equals(\n", + " reference_ds\n", + " ), 'Regridded ATL16 data did not match reference dataset'\n", "\n", " print_success('Regrid ATL16 Data Success')\n", "else:\n", - " print('Skipping test: regridder regression tests not configured for this environment.')" + " print(\n", + " 'Skipping test: regridder regression tests not configured for this environment.'\n", + " )" ] } ], diff --git a/test/regridder/version.txt b/test/regridder/version.txt index 845639ee..9faa1b7a 100644 --- a/test/regridder/version.txt +++ b/test/regridder/version.txt @@ -1 +1 @@ -0.1.4 +0.1.5 diff --git a/test/swath-projector/SwathProjector_Regression.ipynb b/test/swath-projector/SwathProjector_Regression.ipynb index 563bf11a..8f8b499e 100644 --- a/test/swath-projector/SwathProjector_Regression.ipynb +++ b/test/swath-projector/SwathProjector_Regression.ipynb @@ -53,8 +53,12 @@ "\n", "from harmony import Client, Collection, Environment, Request\n", "\n", - "from utilities import (compare_results_to_reference_file, print_success,\n", - " remove_results_files, submit_and_download)" + "from utilities import (\n", + " compare_results_to_reference_file,\n", + " print_success,\n", + " remove_results_files,\n", + " submit_and_download,\n", + ")" ] }, { @@ -96,10 +100,12 @@ "metadata": {}, "outputs": [], "source": [ - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", @@ -125,12 +131,16 @@ "metadata": {}, "outputs": [], "source": [ - "swath_projector_non_prod_information = {'collection': Collection(id='C1233860183-EEDTEST'),\n", - " 'granule_id': 'G1233860549-EEDTEST'}\n", + "swath_projector_non_prod_information = {\n", + " 'collection': Collection(id='C1233860183-EEDTEST'),\n", + " 'granule_id': 'G1233860549-EEDTEST',\n", + "}\n", "\n", - "swath_projector_env = {Environment.LOCAL: swath_projector_non_prod_information,\n", - " Environment.SIT: swath_projector_non_prod_information,\n", - " Environment.UAT: swath_projector_non_prod_information}\n", + "swath_projector_env = {\n", + " Environment.LOCAL: swath_projector_non_prod_information,\n", + " Environment.SIT: swath_projector_non_prod_information,\n", + " Environment.UAT: swath_projector_non_prod_information,\n", + "}\n", "\n", "if harmony_environment in swath_projector_env:\n", " swath_projector_info = swath_projector_env[harmony_environment]\n", @@ -169,20 +179,26 @@ "source": [ "if swath_projector_info is not None:\n", " epsg_file_name = 'swath_projector_epsg.nc4'\n", - " epsg_request = Request(collection=swath_projector_info['collection'],\n", - " granule_id=[swath_projector_info['granule_id']],\n", - " crs='EPSG:4326', scale_extent=[42, -27, 52, -10],\n", - " temporal={'start': datetime(2020, 1, 15), 'stop': datetime(2020, 1, 16)})\n", + " epsg_request = Request(\n", + " collection=swath_projector_info['collection'],\n", + " granule_id=[swath_projector_info['granule_id']],\n", + " crs='EPSG:4326',\n", + " scale_extent=[42, -27, 52, -10],\n", + " temporal={'start': datetime(2020, 1, 15), 'stop': datetime(2020, 1, 16)},\n", + " )\n", "\n", " submit_and_download(harmony_client, epsg_request, epsg_file_name)\n", " assert exists(epsg_file_name), 'Unsuccessful Swath Projector EPSG code request.'\n", "\n", - " compare_results_to_reference_file(epsg_file_name,\n", - " 'reference_files/swath_projector_epsg_reference.nc4')\n", + " compare_results_to_reference_file(\n", + " epsg_file_name, 'reference_files/swath_projector_epsg_reference.nc4'\n", + " )\n", "\n", " print_success('Swath Projector EPSG code request.')\n", "else:\n", - " print(f'The Swath Projector is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'The Swath Projector is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { @@ -205,20 +221,28 @@ "if swath_projector_info is not None:\n", " proj4_string_file_name = 'swath_projector_proj4.nc4'\n", " proj4_lcc = '+proj=lcc +lat_1=43 +lat_2=62 +lat_0=30 +lon_0=10 +x_0=0 +y_0=0 +ellps=intl +units=m +no_defs'\n", - " proj4_string_request = Request(collection=swath_projector_info['collection'],\n", - " granule_id=[swath_projector_info['granule_id']],\n", - " crs=proj4_lcc, interpolation='near',\n", - " temporal={'start': datetime(2020, 1, 15), 'stop': datetime(2020, 1, 16)})\n", + " proj4_string_request = Request(\n", + " collection=swath_projector_info['collection'],\n", + " granule_id=[swath_projector_info['granule_id']],\n", + " crs=proj4_lcc,\n", + " interpolation='near',\n", + " temporal={'start': datetime(2020, 1, 15), 'stop': datetime(2020, 1, 16)},\n", + " )\n", "\n", " submit_and_download(harmony_client, proj4_string_request, proj4_string_file_name)\n", - " assert exists(proj4_string_file_name), 'Unsuccessful Swath Projector interpolation and Proj4 request.'\n", + " assert exists(\n", + " proj4_string_file_name\n", + " ), 'Unsuccessful Swath Projector interpolation and Proj4 request.'\n", "\n", - " compare_results_to_reference_file(proj4_string_file_name,\n", - " 'reference_files/swath_projector_proj4_reference.nc4')\n", + " compare_results_to_reference_file(\n", + " proj4_string_file_name, 'reference_files/swath_projector_proj4_reference.nc4'\n", + " )\n", "\n", " print_success('Swath Projector interpolation and Proj4 request')\n", "else:\n", - " print(f'The Swath Projector is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'The Swath Projector is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { diff --git a/test/swath-projector/utilities.py b/test/swath-projector/utilities.py index e49fa745..b8240835 100644 --- a/test/swath-projector/utilities.py +++ b/test/swath-projector/utilities.py @@ -3,6 +3,7 @@ increase the readability of the regression test suite. """ + from os import listdir, remove, replace from harmony import Client, Request @@ -10,17 +11,19 @@ import xarray as xr -def compare_results_to_reference_file(results_file_name: str, - reference_file_name: str) -> None: - """ Use native `xarray` functionality to compare data values and metadata - attributes of downloaded results to a reference file. +def compare_results_to_reference_file( + results_file_name: str, reference_file_name: str +) -> None: + """Use native `xarray` functionality to compare data values and metadata + attributes of downloaded results to a reference file. """ - reference_data = xr.open_dataset(reference_file_name, group=None) + reference_data = xr.open_dataset(reference_file_name, group=None) results_data = xr.open_dataset(results_file_name, group=None) - assert results_data.equals(reference_data), ('Output and reference files ' - 'do not match.') + assert results_data.equals(reference_data), ( + 'Output and reference files ' 'do not match.' + ) reference_data.close() results_data.close() @@ -28,11 +31,12 @@ def compare_results_to_reference_file(results_file_name: str, results_data = None -def submit_and_download(harmony_client: Client, request: Request, - output_file_name: str): - """ Submit a Harmony request via a `harmony-py` client. Wait for the - Harmony job to finish, then download the results to the specified file - path. +def submit_and_download( + harmony_client: Client, request: Request, output_file_name: str +): + """Submit a Harmony request via a `harmony-py` client. Wait for the + Harmony job to finish, then download the results to the specified file + path. """ downloaded_filename = None @@ -40,10 +44,10 @@ def submit_and_download(harmony_client: Client, request: Request, try: job_id = harmony_client.submit(request) - for filename in [file_future.result() - for file_future - in harmony_client.download_all(job_id, - overwrite=True)]: + for filename in [ + file_future.result() + for file_future in harmony_client.download_all(job_id, overwrite=True) + ]: print(f'Downloaded: {filename}') downloaded_filename = filename @@ -58,8 +62,8 @@ def submit_and_download(harmony_client: Client, request: Request, def remove_results_files() -> None: - """ Remove all NetCDF-4 files downloaded during the Swath Projector - regression tests. + """Remove all NetCDF-4 files downloaded during the Swath Projector + regression tests. """ directory_files = listdir() @@ -70,10 +74,10 @@ def remove_results_files() -> None: def print_error(error_string: str) -> str: - """Print an error, with formatting for red text. """ + """Print an error, with formatting for red text.""" print(f'\033[91m{error_string}\033[0m') def print_success(success_string: str) -> str: - """ Print a success message, with formatting for green text. """ + """Print a success message, with formatting for green text.""" print(f'\033[92mSuccess: {success_string}\033[0m') diff --git a/test/swath-projector/version.txt b/test/swath-projector/version.txt index 9faa1b7a..c946ee61 100644 --- a/test/swath-projector/version.txt +++ b/test/swath-projector/version.txt @@ -1 +1 @@ -0.1.5 +0.1.6 diff --git a/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb b/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb index 1f231420..bc0951b6 100644 --- a/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb +++ b/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb @@ -51,8 +51,12 @@ "\n", "from harmony import BBox, Client, Collection, Environment, Request\n", "\n", - "from utilities import (compare_results_to_reference_file, print_success,\n", - " remove_results_files, submit_and_download)" + "from utilities import (\n", + " compare_results_to_reference_file,\n", + " print_success,\n", + " remove_results_files,\n", + " submit_and_download,\n", + ")" ] }, { @@ -94,10 +98,12 @@ "metadata": {}, "outputs": [], "source": [ - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", @@ -133,18 +139,25 @@ "metadata": {}, "outputs": [], "source": [ - "traj_sub_non_prod_information = {'collection': Collection(id='C1242267295-EEDTEST'),\n", - " 'granule_id': 'G1242274836-EEDTEST',\n", - " 'shape_file_path': 'amazon_basin.geo.json',\n", - " 'requested_variables': ['/BEAM0000/agbd'],\n", - " 'retrieved_variables': ['/BEAM0000/agbd', '/BEAM0000/delta_time',\n", - " '/BEAM0000/lat_lowestmode',\n", - " '/BEAM0000/lon_lowestmode',\n", - " '/BEAM0000/shot_number']}\n", - "\n", - "trajectory_subsetter_env = {Environment.LOCAL: traj_sub_non_prod_information,\n", - " Environment.SIT: traj_sub_non_prod_information,\n", - " Environment.UAT: traj_sub_non_prod_information}\n", + "traj_sub_non_prod_information = {\n", + " 'collection': Collection(id='C1242267295-EEDTEST'),\n", + " 'granule_id': 'G1242274836-EEDTEST',\n", + " 'shape_file_path': 'amazon_basin.geo.json',\n", + " 'requested_variables': ['/BEAM0000/agbd'],\n", + " 'retrieved_variables': [\n", + " '/BEAM0000/agbd',\n", + " '/BEAM0000/delta_time',\n", + " '/BEAM0000/lat_lowestmode',\n", + " '/BEAM0000/lon_lowestmode',\n", + " '/BEAM0000/shot_number',\n", + " ],\n", + "}\n", + "\n", + "trajectory_subsetter_env = {\n", + " Environment.LOCAL: traj_sub_non_prod_information,\n", + " Environment.SIT: traj_sub_non_prod_information,\n", + " Environment.UAT: traj_sub_non_prod_information,\n", + "}\n", "\n", "if harmony_environment in trajectory_subsetter_env:\n", " trajectory_subsetter_info = trajectory_subsetter_env[harmony_environment]\n", @@ -179,19 +192,27 @@ "source": [ "if trajectory_subsetter_info is not None:\n", " ts_variable_file_name = 'trajectory_subsetter_variable.h5'\n", - " ts_variable_request = Request(collection=trajectory_subsetter_info['collection'],\n", - " granule_id=[trajectory_subsetter_info['granule_id']],\n", - " variables=trajectory_subsetter_info['requested_variables'])\n", + " ts_variable_request = Request(\n", + " collection=trajectory_subsetter_info['collection'],\n", + " granule_id=[trajectory_subsetter_info['granule_id']],\n", + " variables=trajectory_subsetter_info['requested_variables'],\n", + " )\n", "\n", " submit_and_download(harmony_client, ts_variable_request, ts_variable_file_name)\n", - " assert exists(ts_variable_file_name), 'Unsuccessful Trajectory Subsetter variable subset request.'\n", + " assert exists(\n", + " ts_variable_file_name\n", + " ), 'Unsuccessful Trajectory Subsetter variable subset request.'\n", "\n", - " compare_results_to_reference_file(ts_variable_file_name,\n", - " 'reference_files/trajectory_subsetter_variable_reference.h5')\n", + " compare_results_to_reference_file(\n", + " ts_variable_file_name,\n", + " 'reference_files/trajectory_subsetter_variable_reference.h5',\n", + " )\n", "\n", " print_success('Trajectory Subsetter variable subset request.')\n", "else:\n", - " print(f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { @@ -215,21 +236,31 @@ "source": [ "if trajectory_subsetter_info is not None:\n", " ts_temporal_file_name = 'trajectory_subsetter_temporal.h5'\n", - " ts_temporal_request = Request(collection=trajectory_subsetter_info['collection'],\n", - " granule_id=[trajectory_subsetter_info['granule_id']],\n", - " variables=trajectory_subsetter_info['requested_variables'],\n", - " temporal={'start': datetime(2020, 7, 8, 1, 0, 0),\n", - " 'stop': datetime(2020, 7, 8, 2, 0, 0)})\n", + " ts_temporal_request = Request(\n", + " collection=trajectory_subsetter_info['collection'],\n", + " granule_id=[trajectory_subsetter_info['granule_id']],\n", + " variables=trajectory_subsetter_info['requested_variables'],\n", + " temporal={\n", + " 'start': datetime(2020, 7, 8, 1, 0, 0),\n", + " 'stop': datetime(2020, 7, 8, 2, 0, 0),\n", + " },\n", + " )\n", "\n", " submit_and_download(harmony_client, ts_temporal_request, ts_temporal_file_name)\n", - " assert exists(ts_temporal_file_name), 'Unsuccessful Trajectory Subsetter temporal subset request.'\n", + " assert exists(\n", + " ts_temporal_file_name\n", + " ), 'Unsuccessful Trajectory Subsetter temporal subset request.'\n", "\n", - " compare_results_to_reference_file(ts_temporal_file_name,\n", - " 'reference_files/trajectory_subsetter_temporal_reference.h5')\n", + " compare_results_to_reference_file(\n", + " ts_temporal_file_name,\n", + " 'reference_files/trajectory_subsetter_temporal_reference.h5',\n", + " )\n", "\n", " print_success('Trajectory Subsetter temporal subset request.')\n", "else:\n", - " print(f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { @@ -257,20 +288,27 @@ "if trajectory_subsetter_info is not None:\n", " ts_bbox_file_name = 'trajectory_subsetter_bbox.h5'\n", " ts_bbox_bbox = BBox(w=-74, s=-34, e=-35, n=5)\n", - " ts_bbox_request = Request(collection=trajectory_subsetter_info['collection'],\n", - " granule_id=[trajectory_subsetter_info['granule_id']],\n", - " variables=trajectory_subsetter_info['requested_variables'],\n", - " spatial=ts_bbox_bbox)\n", + " ts_bbox_request = Request(\n", + " collection=trajectory_subsetter_info['collection'],\n", + " granule_id=[trajectory_subsetter_info['granule_id']],\n", + " variables=trajectory_subsetter_info['requested_variables'],\n", + " spatial=ts_bbox_bbox,\n", + " )\n", "\n", " submit_and_download(harmony_client, ts_bbox_request, ts_bbox_file_name)\n", - " assert exists(ts_bbox_file_name), 'Unsuccessful Trajectory Subsetter bounding box subset request.'\n", + " assert exists(\n", + " ts_bbox_file_name\n", + " ), 'Unsuccessful Trajectory Subsetter bounding box subset request.'\n", "\n", - " compare_results_to_reference_file(ts_bbox_file_name,\n", - " 'reference_files/trajectory_subsetter_bbox_reference.h5')\n", + " compare_results_to_reference_file(\n", + " ts_bbox_file_name, 'reference_files/trajectory_subsetter_bbox_reference.h5'\n", + " )\n", "\n", " print_success('Trajectory Subsetter bounding box spatial subset request.')\n", "else:\n", - " print(f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { @@ -294,20 +332,28 @@ "source": [ "if trajectory_subsetter_info is not None:\n", " ts_polygon_file_name = 'trajectory_subsetter_polygon.h5'\n", - " ts_polygon_request = Request(collection=trajectory_subsetter_info['collection'],\n", - " granule_id=[trajectory_subsetter_info['granule_id']],\n", - " variables=trajectory_subsetter_info['requested_variables'],\n", - " shape=trajectory_subsetter_info['shape_file_path'])\n", + " ts_polygon_request = Request(\n", + " collection=trajectory_subsetter_info['collection'],\n", + " granule_id=[trajectory_subsetter_info['granule_id']],\n", + " variables=trajectory_subsetter_info['requested_variables'],\n", + " shape=trajectory_subsetter_info['shape_file_path'],\n", + " )\n", "\n", " submit_and_download(harmony_client, ts_polygon_request, ts_polygon_file_name)\n", - " assert exists(ts_polygon_file_name), 'Unsuccessful Trajectory Subsetter polygon spatial subset request.'\n", + " assert exists(\n", + " ts_polygon_file_name\n", + " ), 'Unsuccessful Trajectory Subsetter polygon spatial subset request.'\n", "\n", - " compare_results_to_reference_file(ts_polygon_file_name,\n", - " 'reference_files/trajectory_subsetter_polygon_reference.h5')\n", + " compare_results_to_reference_file(\n", + " ts_polygon_file_name,\n", + " 'reference_files/trajectory_subsetter_polygon_reference.h5',\n", + " )\n", "\n", " print_success('Trajectory Subsetter polygon spatial subset request.')\n", "else:\n", - " print(f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'Trajectory Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { diff --git a/test/trajectory-subsetter/utilities.py b/test/trajectory-subsetter/utilities.py index a8be3d70..d682d97f 100644 --- a/test/trajectory-subsetter/utilities.py +++ b/test/trajectory-subsetter/utilities.py @@ -3,6 +3,7 @@ increase the readability of the regression test suite. """ + from os import listdir, remove, replace from harmony import Client, Request @@ -10,28 +11,31 @@ from datatree import open_datatree -def compare_results_to_reference_file(results_file_name: str, - reference_file_name: str) -> None: - """ Use `DataTree` functionality to compare data values, variables, - coordinates, metadata, and all their corresponding attributes of - downloaded results to a reference file. +def compare_results_to_reference_file( + results_file_name: str, reference_file_name: str +) -> None: + """Use `DataTree` functionality to compare data values, variables, + coordinates, metadata, and all their corresponding attributes of + downloaded results to a reference file. """ - reference_data = open_datatree(reference_file_name) + reference_data = open_datatree(reference_file_name) results_data = open_datatree(results_file_name) - assert results_data.identical(reference_data), ('Output and reference files ' - 'do not match.') + assert results_data.identical(reference_data), ( + 'Output and reference files ' 'do not match.' + ) reference_data = None results_data = None -def submit_and_download(harmony_client: Client, request: Request, - output_file_name: str): - """ Submit a Harmony request via a `harmony-py` client. Wait for the - Harmony job to finish, then download the results to the specified file - path. +def submit_and_download( + harmony_client: Client, request: Request, output_file_name: str +): + """Submit a Harmony request via a `harmony-py` client. Wait for the + Harmony job to finish, then download the results to the specified file + path. """ downloaded_filename = None @@ -39,10 +43,10 @@ def submit_and_download(harmony_client: Client, request: Request, try: job_id = harmony_client.submit(request) - for filename in [file_future.result() - for file_future - in harmony_client.download_all(job_id, - overwrite=True)]: + for filename in [ + file_future.result() + for file_future in harmony_client.download_all(job_id, overwrite=True) + ]: print(f'Downloaded: {filename}') downloaded_filename = filename @@ -57,8 +61,8 @@ def submit_and_download(harmony_client: Client, request: Request, def remove_results_files() -> None: - """ Remove all NetCDF-4 files downloaded during the Swath Projector - regression tests. + """Remove all NetCDF-4 files downloaded during the Swath Projector + regression tests. """ directory_files = listdir() @@ -69,10 +73,10 @@ def remove_results_files() -> None: def print_error(error_string: str) -> str: - """Print an error, with formatting for red text. """ + """Print an error, with formatting for red text.""" print(f'\033[91m{error_string}\033[0m') def print_success(success_string: str) -> str: - """ Print a success message, with formatting for green text. """ + """Print a success message, with formatting for green text.""" print(f'\033[92mSuccess: {success_string}\033[0m') diff --git a/test/trajectory-subsetter/version.txt b/test/trajectory-subsetter/version.txt index def9a015..c946ee61 100644 --- a/test/trajectory-subsetter/version.txt +++ b/test/trajectory-subsetter/version.txt @@ -1 +1 @@ -0.1.5 \ No newline at end of file +0.1.6 diff --git a/test/variable-subsetter/VariableSubsetter_Regression.ipynb b/test/variable-subsetter/VariableSubsetter_Regression.ipynb index 6c968cbf..ea58dad6 100644 --- a/test/variable-subsetter/VariableSubsetter_Regression.ipynb +++ b/test/variable-subsetter/VariableSubsetter_Regression.ipynb @@ -47,8 +47,12 @@ "from harmony import Client, Collection, Environment, Request\n", "import numpy as np\n", "\n", - "from utilities import (compare_results_to_reference_file, print_success,\n", - " remove_results_files, submit_and_download)" + "from utilities import (\n", + " compare_results_to_reference_file,\n", + " print_success,\n", + " remove_results_files,\n", + " submit_and_download,\n", + ")" ] }, { @@ -90,10 +94,12 @@ "metadata": {}, "outputs": [], "source": [ - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", "\n", @@ -129,8 +135,10 @@ "metadata": {}, "outputs": [], "source": [ - "var_subsetter_non_prod_information = {'collection': Collection(id=''),\n", - " 'granule_id': ''}\n", + "var_subsetter_non_prod_information = {\n", + " 'collection': Collection(id=''),\n", + " 'granule_id': '',\n", + "}\n", "\n", "var_subsetter_env = {}\n", "# var_subsetter_env = {Environment.LOCAL: var_subsetter_non_prod_information,\n", @@ -164,21 +172,26 @@ "source": [ "if var_subsetter_info is not None:\n", " single_var_file_name = 'var_subsetter.nc4'\n", - " single_var_request = Request(collection=var_subsetter_info['collection'],\n", - " granule_id=[var_subsetter_info['granule_id']],\n", - " variables=[''])\n", + " single_var_request = Request(\n", + " collection=var_subsetter_info['collection'],\n", + " granule_id=[var_subsetter_info['granule_id']],\n", + " variables=[''],\n", + " )\n", "\n", " submit_and_download(harmony_client, single_var_request, single_var_file_name)\n", - " assert exists(single_var_file_name), 'Unsuccessful single-variable Variable Subsetter request.'\n", + " assert exists(\n", + " single_var_file_name\n", + " ), 'Unsuccessful single-variable Variable Subsetter request.'\n", "\n", " compare_results_to_reference_file(\n", - " single_var_file_name,\n", - " 'reference_files/var_subsetter_single_var_reference.nc4'\n", + " single_var_file_name, 'reference_files/var_subsetter_single_var_reference.nc4'\n", " )\n", "\n", " print_success('Variable subsetter single variable request.')\n", "else:\n", - " print(f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { @@ -202,20 +215,25 @@ "source": [ "if var_subsetter_info is not None:\n", " all_variables_file_name = 'var_subsetter_all_vars.nc4'\n", - " all_variables_request = Request(collection=var_subsetter_info['collection'],\n", - " granule_id=[var_subsetter_info['granule_id']])\n", + " all_variables_request = Request(\n", + " collection=var_subsetter_info['collection'],\n", + " granule_id=[var_subsetter_info['granule_id']],\n", + " )\n", "\n", " submit_and_download(harmony_client, all_variables_request, all_variables_file_name)\n", - " assert exists(all_variables_file_name), 'Unsuccessful Variable Subsetter all-variable request.'\n", + " assert exists(\n", + " all_variables_file_name\n", + " ), 'Unsuccessful Variable Subsetter all-variable request.'\n", "\n", " compare_results_to_reference_file(\n", - " all_variables_file_name,\n", - " 'reference_files/var_subsetter_all_vars_reference.nc4'\n", + " all_variables_file_name, 'reference_files/var_subsetter_all_vars_reference.nc4'\n", " )\n", "\n", " print_success('Variable Subsetter all variable request.')\n", "else:\n", - " print(f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')" + " print(\n", + " f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.'\n", + " )" ] }, { diff --git a/test/variable-subsetter/utilities.py b/test/variable-subsetter/utilities.py index db8d20f8..801e549e 100644 --- a/test/variable-subsetter/utilities.py +++ b/test/variable-subsetter/utilities.py @@ -3,6 +3,7 @@ increase the readability of the regression test suite. """ + from os import listdir, remove, replace from typing import Union @@ -15,11 +16,12 @@ GroupOrVariable = Union[Group, Variable] -def compare_attributes_to_reference(results_object: GroupOrVariable, - ref_object: GroupOrVariable): - """ Ensure the metadata attributes of two NetCDF-4 objects (groups or - variables) are the same, with the exception of `history` and - `history_json`, which will include references to the request time. +def compare_attributes_to_reference( + results_object: GroupOrVariable, ref_object: GroupOrVariable +): + """Ensure the metadata attributes of two NetCDF-4 objects (groups or + variables) are the same, with the exception of `history` and + `history_json`, which will include references to the request time. """ assert results_object.ncattrs() == ref_object.ncattrs() @@ -28,17 +30,15 @@ def compare_attributes_to_reference(results_object: GroupOrVariable, if attribute_name not in ['history', 'history_json']: if isinstance(ref_attribute_value, np.ndarray): np.testing.assert_array_equal( - results_object.getncattr(attribute_name), - ref_attribute_value + results_object.getncattr(attribute_name), ref_attribute_value ) else: assert results_object.getncattr(attribute_name) == ref_attribute_value -def compare_variable_to_reference(results_variable: Variable, - ref_variable: Variable): - """ Compare two NetCDF-4 variables, ensuring they have the same data in - their arrays and the same metadata attribute. +def compare_variable_to_reference(results_variable: Variable, ref_variable: Variable): + """Compare two NetCDF-4 variables, ensuring they have the same data in + their arrays and the same metadata attribute. """ np.testing.assert_array_equal(results_variable[:], ref_variable[:]) @@ -46,9 +46,9 @@ def compare_variable_to_reference(results_variable: Variable, def compare_group_to_reference(results_group: Group, ref_group: Group): - """ Compare two NetCDF-4 file groups, ensuring they have the same metadata - attributes (excluding provenance), child variables and child groups. - Child variables and groups are then compared recursively. + """Compare two NetCDF-4 file groups, ensuring they have the same metadata + attributes (excluding provenance), child variables and child groups. + Child variables and groups are then compared recursively. """ compare_attributes_to_reference(results_group, ref_group) @@ -60,24 +60,24 @@ def compare_group_to_reference(results_group: Group, ref_group: Group): compare_variable_to_reference(results_group[variable_name], ref_variable) for child_group_name, ref_child_group in ref_group.groups.items(): - compare_group_to_reference(results_group[child_group_name], - ref_child_group) + compare_group_to_reference(results_group[child_group_name], ref_child_group) def compare_results_to_reference_file(results_file: str, ref_file: str): - """ Compare two NetCDF-4 files recursively, checking that the both have the - same group structure, variables and metadata attributes. + """Compare two NetCDF-4 files recursively, checking that the both have the + same group structure, variables and metadata attributes. """ with Dataset(results_file) as results_ds, Dataset(ref_file) as ref_ds: compare_group_to_reference(results_ds, ref_ds) -def submit_and_download(harmony_client: Client, request: Request, - output_file_name: str): - """ Submit a Harmony request via a `harmony-py` client. Wait for the - Harmony job to finish, then download the results to the specified file - path. +def submit_and_download( + harmony_client: Client, request: Request, output_file_name: str +): + """Submit a Harmony request via a `harmony-py` client. Wait for the + Harmony job to finish, then download the results to the specified file + path. """ downloaded_filename = None @@ -85,10 +85,10 @@ def submit_and_download(harmony_client: Client, request: Request, try: job_id = harmony_client.submit(request) - for filename in [file_future.result() - for file_future - in harmony_client.download_all(job_id, - overwrite=True)]: + for filename in [ + file_future.result() + for file_future in harmony_client.download_all(job_id, overwrite=True) + ]: print(f'Downloaded: {filename}') downloaded_filename = filename @@ -103,8 +103,8 @@ def submit_and_download(harmony_client: Client, request: Request, def remove_results_files() -> None: - """ Remove all NetCDF-4 files downloaded during the Swath Projector - regression tests. + """Remove all NetCDF-4 files downloaded during the Swath Projector + regression tests. """ directory_files = listdir() @@ -115,10 +115,10 @@ def remove_results_files() -> None: def print_error(error_string: str) -> str: - """Print an error, with formatting for red text. """ + """Print an error, with formatting for red text.""" print(f'\033[91m{error_string}\033[0m') def print_success(success_string: str) -> str: - """ Print a success message, with formatting for green text. """ + """Print a success message, with formatting for green text.""" print(f'\033[92mSuccess: {success_string}\033[0m') diff --git a/test/variable-subsetter/version.txt b/test/variable-subsetter/version.txt index 845639ee..9faa1b7a 100644 --- a/test/variable-subsetter/version.txt +++ b/test/variable-subsetter/version.txt @@ -1 +1 @@ -0.1.4 +0.1.5