Simplified the CAMELS data download (#824)

.github/workflows/python-app.yml

@@ -47,6 +47,7 @@ jobs:
         run: |
           # Download test grid data
           synthesizer-download --test-grids -d tests/test_grid/ --dust-grid
+          synthesizer-download -d tests/data --camels-data
       - name: Test with pytest
         run: |
           pytest

.github/workflows/static.yml

@@ -39,6 +39,7 @@ jobs:
       - name: Download test data
         run: |
           synthesizer-download --test-grids -d tests/test_grid/ --dust-grid
+          synthesizer-download -d tests/data --camels-data
       - name: Sphinx Build
         run: |
           cd docs
@@ -51,7 +52,7 @@ jobs:
         with:
           name: github-pages
           path: docs/build/html/
-          
+
   deploy:
     environment:
       name: github-pages

.gitignore

@@ -10,6 +10,7 @@ docs/source/gen_modules/
 src/synthesizer/_version.py
 docs/source/sg_execution_times.rst
 docs/test_filter_collection.hdf5
+tests/data/*.hdf5
 *.hdf5
 
 # Byte-compiled / optimized / DLL files

README.md

@@ -9,8 +9,6 @@
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![License: GPLv3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
-
-
 Synthesizer is a python package for generating synthetic astrophysical observables. It is modular, flexible and fast.
 
 Read the documentation [here](https://flaresimulations.github.io/synthesizer/).
@@ -25,8 +23,8 @@ To install, enter the `synthesizer` directory and install with pip.
 
     cd synthesizer
     pip install .
-    
-We also provide optional dependency sets for development (``dev``), testing (``test``), and building the documentation (``docs``) should you ever needed them. To install all dependancies simply run the following (or delete as appropriate to get a specific subset):
+
+We also provide optional dependency sets for development (`dev`), testing (`test`), and building the documentation (`docs`) should you ever needed them. To install all dependancies simply run the following (or delete as appropriate to get a specific subset):
 
     pip install .[dev,test,docs]
 
@@ -41,8 +39,17 @@ If you wish to run the examples, or need some data for development purposes, we
 ```bash
 synthesizer-download --test-grids --dust-grid -d tests/test_grid
 ```
+
 This command will store the SPS, AGN, and dust grids in the `tests` directory at the root of the repo; all examples expect this data to reside in this location.
 
+You will also want to download the preprocessed CAMELS Illustris-TNG data,
+
+```bash
+synthesizer-download --camels-data -d tests/data/
+```
+
+this is a very small set of galaxies taken from the [CAMELS suite](https://camels.readthedocs.io/en/latest/) of simulations. We use this in some particle based examples.
+
 ## Contributing
 
 Please see [here](docs/CONTRIBUTING.md) for contribution guidelines.

docs/source/getting_started/downloading_grids.rst

@@ -14,3 +14,9 @@ To download the test grid, ``cd`` into the root synthesizer directory and run th
 
     cd synthesizer
     synthesizer-download --test-grids -d tests/test_grid/ --dust-grid
+
+You'll also want to download some test data if you want to run any particle based examples. We provided some preprocessed TNG data from CAMELS. To download it run 
+
+.. code-block:: bash
+
+    synthesizer-download --camels-data -d tests/data/

src/synthesizer/download_data.py

@@ -44,6 +44,16 @@
         "id": "jidw4cgtf95x3gjvw4hj6",
         "rlkey": "z7sbb7z5253dt90ootr5hm5jv",
     },
+    "camels_snap.hdf5": {
+        "file": "camels_snap.hdf5",
+        "id": "c44wvkjm5pqsxpsl54oq0",
+        "rlkey": "j14smjen4osffhlyif1kz00bu",
+    },
+    "camels_subhalo.hdf5": {
+        "file": "camels_subhalo.hdf5",
+        "id": "srjaltgac4e2tsrxmxrdb",
+        "rlkey": "ov0icvv7znw9ybfr31h133jiq",
+    },
 }
 
 
@@ -206,23 +216,16 @@ def download_dust_grid(destination):
     _download("MW3.1.hdf5", destination)
 
 
-def download_camels_data(snap, lh, destination):
+def download_camels_data(destination):
     """
-    Download a CAMELs dataset.
+    Download the CAMELS data.
 
     Args:
-        snap (str)
-            The snapshot tag to download.
-        lh (str)
-            The LH variant tag of the sim to download.
         destination (str)
             The path to the destination directory.
     """
-    # Convert lh
-    lh = str(lh)
-    raise exceptions.UnimplementedFunctionality(
-        "CAMELS data is not yet available for download."
-    )
+    _download("camels_snap.hdf5", destination)
+    _download("camels_subhalo.hdf5", destination)
 
 
 def download():
@@ -260,27 +263,12 @@ def download():
         help="Download the dust grid for the Drain & Li (2007) model",
     )
 
-    # Add the flag for camels data (this requires related arguments to define
-    # exactly which dataset to download)
+    # Add the flag for processed camels data
     parser.add_argument(
         "--camels-data",
         "-c",
         action="store_true",
-        help="Download the CAMELS dataset",
-    )
-
-    # Add the CAMELs arguments
-    parser.add_argument(
-        "--camels-snap",
-        type=str,
-        help="Which snapshot should be downloaded? (Default: 031)",
-        default="031",
-    )
-    parser.add_argument(
-        "--camels-lh",
-        type=int,
-        help="Which LH variant should be downloaded? (Default: 1)",
-        default=1,
+        help="Download the CAMELS TNG dataset for testing",
     )
 
     # Add a flag to go ham and download everything
@@ -316,7 +304,7 @@ def download():
     if everything:
         download_test_grids(dest)
         download_dust_grid(dest)
-        download_camels_data(args.camels_snap, args.camels_lh, dest)
+        download_camels_data(dest)
         return
 
     # Test data?
@@ -334,7 +322,7 @@ def download():
 
     # Camels data?
     if camels:
-        download_camels_data(args.camels_snap, args.camels_lh, dest)
+        download_camels_data(dest)
 
 
 if __name__ == "__main__":