From 69a958581ecd2d32ee9cb1c38bcd3847b8b920bf Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 28 Jun 2024 06:27:40 -0600 Subject: [PATCH 1/8] fix cmake for github workflow --- machines/cmake_macros/ubuntu-latest.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/cmake_macros/ubuntu-latest.cmake b/machines/cmake_macros/ubuntu-latest.cmake index 2a5718bb..3be39559 100644 --- a/machines/cmake_macros/ubuntu-latest.cmake +++ b/machines/cmake_macros/ubuntu-latest.cmake @@ -1 +1 @@ -set(PFUNIT_PATH "$GITHUB_WORKSPACE/pfunit/build") \ No newline at end of file +set(PFUNIT_PATH "$GITHUB_WORKSPACE/pfunit/build") From 323d35eb6cc5d4287b690a0f2953d48b1e1b5abc Mon Sep 17 00:00:00 2001 From: alperaltuntas Date: Tue, 23 Jul 2024 11:52:19 -0600 Subject: [PATCH 2/8] remove old mom, ww3 grids, add wtx2_32, change _POP to _MOM, remove T.g1. aliases --- component_grids_nuopc.xml | 16 +-- modelgrid_aliases_nuopc.xml | 259 ++++++++++++++---------------------- 2 files changed, 105 insertions(+), 170 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 0eb63959..21a89a1f 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -367,12 +367,6 @@ $DIN_LOC_ROOT/share/meshes/gx3v7_120309_ESMFmesh.nc gx3v7 is displaced Greenland pole v7 3-deg grid: - - 540 458 - $DIN_LOC_ROOT/share/meshes/tx0.66v1_190314_ESMFmesh.nc - tx0.66v1 is tripole v1 0.66-deg MOM6 grid: - Experimental for MOM6 experiments - 540 480 $DIN_LOC_ROOT/share/meshes/tx2_3v2_230415_ESMFmesh.nc @@ -538,11 +532,11 @@ gx3v7 global grid For testing of the WAV model - - 540 458 - $DIN_LOC_ROOT/share/meshes/wtx0.66v1_210917_ESMFmesh.nc - wtx0.66v1 is tripole v1 0.66-deg WW3 grid based on MOM6 grid - MOM6 tx0.66v1-based WW3 grid where cells above 88N are masked + + 540 480 + $DIN_LOC_ROOT/share/meshes/wtx2_3v2_231005b_ESMFmesh.nc + wtx2_3v2 is tripole v2 2/3-deg WW3 grid based on MOM6 grid + MOM6 tx2_3v2-based WW3 grid where cells above 88N are masked diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 751ca50d..23019a6b 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -154,14 +154,14 @@ 0.125nldas2 - + 360x720cru 360x720cru tx0.1v3 tx0.1v3 - + 360x720cru 360x720cru tx0.1v3 @@ -169,7 +169,7 @@ tx0.1v3 - + 360x720cru 360x720cru tx0.1v3 @@ -177,7 +177,7 @@ tx0.1v3 - + 360x720cru 360x720cru tx0.1v3 @@ -226,42 +226,42 @@ gx3v7 - + T42 T42 T42 gx1v7 - + T42 T42 T42 gx1v6 - + T42 T42 T42 gx1v7 - + T5 T5 T5 gx3v7 - + T85 T85 T85 gx1v6 - + T85 T85 T85 @@ -324,38 +324,6 @@ tx0.1v3 - - TL319 - TL319 - gx1v7 - JRA025v2 - gx1v7 - - - - TL319 - TL319 - gx1v7 - JRA025v2 - gx1v7 - gx1v7 - - - - TL319 - TL319 - gx1v7 - JRA025 - gx1v7 - - - - TL319 - TL319 - tx0.66v1 - JRA025 - - TL319 TL319 @@ -371,12 +339,12 @@ wgx3v7 - + TL319 TL319 - tx0.66v1 + tx2_3v2 JRA025 - wtx0.66v1 + wtx2_3v2 @@ -408,24 +376,24 @@ gx1v7 - - TL639 - TL639 - tx0.66v1 - + + T62 + T62 + tx2_3v2 - + T62 T62 - tx0.66v1 + tx2_3v2 + wgx3v7 - + T62 T62 - tx0.66v1 - tx0.66v1 + tx2_3v2 + wtx2_3v2 @@ -433,11 +401,6 @@ T62 tx0.25v1 - - 0.9x1.25 - 0.9x1.25 - tx0.66v1 - 0.9x1.25 @@ -459,28 +422,6 @@ ais8:gris4 - - T62 - T62 - gx1v6 - gx1v6 - - - - T62 - T62 - gx1v7 - gx1v7 - - - - T62 - T62 - gx1v7 - gx1v7 - gx1v7 - - T62 T62 @@ -629,21 +570,21 @@ null - + 0.9x1.25 0.9x1.25 0.9x1.25 gx1v6 - + 0.9x1.25 0.9x1.25 0.9x1.25 gx1v7 - + 0.9x1.25 0.9x1.25 0.9x1.25 @@ -651,7 +592,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 0.9x1.25 @@ -659,7 +600,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 0.9x1.25 @@ -668,7 +609,7 @@ - + 0.47x0.63 0.47x0.63 0.47x0.63 @@ -736,14 +677,14 @@ gx1v7 - + 1.9x2.5 1.9x2.5 1.9x2.5 gx1v6 - + 1.9x2.5 1.9x2.5 1.9x2.5 @@ -757,14 +698,14 @@ null - + 1.9x2.5 1.9x2.5 1.9x2.5 gx1v7 - + 1.9x2.5 1.9x2.5 1.9x2.5 @@ -772,7 +713,7 @@ gx1v7 - + 1.9x2.5 1.9x2.5 1.9x2.5 @@ -780,7 +721,7 @@ gx1v7 - + 1.9x2.5 1.9x2.5 1.9x2.5 @@ -795,49 +736,49 @@ gx3v7 - + 0.23x0.31 0.23x0.31 0.23x0.31 gx1v6 - + 0.23x0.31 0.23x0.31 0.23x0.31 gx1v7 - + 2.5x3.33 2.5x3.33 2.5x3.33 gx1v6 - + 2.5x3.33 2.5x3.33 2.5x3.33 gx1v7 - + 4x5 4x5 4x5 gx3v7 - + 10x15 10x15 10x15 gx3v7 - + 10x15 10x15 10x15 @@ -845,7 +786,7 @@ gx3v7 - + 10x15 10x15 10x15 @@ -853,7 +794,7 @@ gx3v7 - + 10x15 10x15 10x15 @@ -868,21 +809,21 @@ - + ne3np4.pg3 ne3np4.pg3 ne3np4.pg3 gx3v7 - + ne3np4 ne3np4 ne3np4 gx3v7 - + ne5np4 ne5np4 ne5np4 @@ -896,7 +837,7 @@ gx1v7 - + ne16np4 ne16np4 ne16np4 @@ -940,11 +881,11 @@ gx1v7 - + ne30np4.pg3 ne30np4.pg3 - tx0.66v1 - tx0.66v1 + tx2_3v2 + tx2_3v2 @@ -1010,14 +951,14 @@ gx1v7 - + ne30np4 ne30np4 ne30np4 gx1v6 - + ne30np4 ne30np4 ne30np4 @@ -1038,7 +979,7 @@ gx1v7 - + ne60np4 ne60np4 ne60np4 @@ -1066,14 +1007,14 @@ tx0.1v2 - + ne120np4 ne120np4 ne120np4 gx1v6 - + ne120np4 ne120np4 ne120np4 @@ -1103,14 +1044,14 @@ tx0.1v2 - + ne240np4 ne240np4 ne240np4 gx1v6 - + ne240np4 ne240np4 ne240np4 @@ -1119,7 +1060,7 @@ - + ne5np4.pg2 ne5np4.pg2 ne5np4.pg2 @@ -1133,14 +1074,14 @@ gx1v7 - + ne60np4.pg2 ne60np4.pg2 ne60np4.pg2 gx1v7 - + ne120np4.pg2 ne120np4.pg2 ne120np4.pg2 @@ -1154,7 +1095,7 @@ tx0.1v2 - + ne240np4.pg2 ne240np4.pg2 ne240np4.pg2 @@ -1163,49 +1104,49 @@ - + ne5np4.pg3 ne5np4.pg3 ne5np4.pg3 gx3v7 - + ne16np4.pg3 ne16np4.pg3 ne16np4.pg3 gx1v7 - + ne30np4.pg3 ne30np4.pg3 ne30np4.pg3 gx1v7 - + ne60np4.pg3 ne60np4.pg3 ne60np4.pg3 gx1v7 - + ne120np4.pg3 ne120np4.pg3 ne120np4.pg3 gx1v7 - + ne120np4.pg3 ne120np4.pg3 ne120np4.pg3 tx0.1v3 - + ne240np4.pg3 ne240np4.pg3 ne240np4.pg3 @@ -1228,28 +1169,28 @@ - + ne5np4.pg4 ne5np4.pg4 ne5np4.pg4 gx3v7 - + ne30np4.pg4 ne30np4.pg4 ne30np4.pg4 gx1v7 - + ne60np4.pg4 ne60np4.pg4 ne60np4.pg4 gx1v7 - + ne120np4.pg4 ne120np4.pg4 ne120np4.pg4 @@ -1265,63 +1206,63 @@ gx1v7 - + ne0np4CONUS.ne30x8 ne0np4CONUS.ne30x8 ne0np4CONUS.ne30x8 gx1v7 - + ne0np4TESTONLY.ne5x4 ne0np4TESTONLY.ne5x4 ne0np4TESTONLY.ne5x4 gx3v7 - + ne0np4CONUS.ne30x8 ne0np4CONUS.ne30x8 ne0np4CONUS.ne30x8 tx0.1v2 - + ne0np4.ARCTIC.ne30x4 ne0np4.ARCTIC.ne30x4 ne0np4.ARCTIC.ne30x4 tx0.1v2 - + ne0np4.ARCTIC.ne30x4 ne0np4.ARCTIC.ne30x4 tx0.1v2 tx0.1v2 - + ne0np4.ARCTIC.ne30x4 ne0np4.ARCTIC.ne30x4 ne0np4.ARCTIC.ne30x4 gx1v7 - + ne0np4.ARCTIC.ne30x4 ne0np4.ARCTIC.ne30x4 gx1v7 gx1v7 - + ne0np4.ARCTICGRIS.ne30x8 ne0np4.ARCTICGRIS.ne30x8 ne0np4.ARCTICGRIS.ne30x8 tx0.1v2 - + ne0np4.POLARCAP.ne30x4 ne0np4.POLARCAP.ne30x4 ne0np4.POLARCAP.ne30x4 @@ -1330,77 +1271,77 @@ - + mpasa480 mpasa480 mpasa480 gx1v7 - + mpasa240 mpasa240 mpasa240 gx1v7 - + mpasa120 mpasa120 mpasa120 gx1v7 - + mpasa60 mpasa60 mpasa60 gx1v7 - + mpasa30 mpasa30 mpasa30 gx1v7 - + mpasa15 mpasa15 mpasa15 gx1v7 - + mpasa12 mpasa12 mpasa12 gx1v7 - + mpasa15-3conus mpasa15-3conus mpasa15-3conus tx0.1v3 - + mpasa7p5 mpasa7p5 mpasa7p5 gx1v7 - + mpasa3p75 mpasa3p75 mpasa3p75 gx1v7 - + mpasa3p75 mpasa3p75 mpasa3p75 @@ -1474,21 +1415,21 @@ gx1v7 - + C96 C96 C96 - tx0.66v1 + tx2_3v2 - + C96 C96 - tx0.66v1 - tx0.66v1 + tx2_3v2 + tx2_3v2 - + C96 C96 tx0.25v1 @@ -1510,7 +1451,7 @@ gx1v7 - + C384 C384 tx0.25v1 From 12a08f7435788bdc131b322985f86abb8aaa47f1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 23 Jul 2024 13:32:58 -0600 Subject: [PATCH 3/8] remove obsolete specs --- config_archive.xml | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/config_archive.xml b/config_archive.xml index 4a677134..3a433ae7 100644 --- a/config_archive.xml +++ b/config_archive.xml @@ -1,48 +1,5 @@ - - r - r[ho] - rh.* - h\d*.*\.nc$ - d[dovt] - unset - - rpointer.ocn$NINST_STRING.restart - ./$CASE.pop$NINST_STRING.r.$DATENAME.nc,RESTART_FMT=nc - - - rpointer.ocn$NINST_STRING.ovf - ./$CASE.pop$NINST_STRING.ro.$DATENAME - - - rpointer.ocn$NINST_STRING.tavg - ./$CASE.pop$NINST_STRING.rh.$DATENAME.nc - - - rpointer.pop - casename.pop_0001.r.1976-01-01-00000.nc - casename.pop.r.1976-01-01-00000.nc - casename.pop.h.1976-01-01-00000.nc - casename.pop.h.1975-02-01-00000.nc - casename.pop.h0.1976-01-01-00000.nc - casename.pop.dd.1976-01-01-00000 - casename.pop.rh.ecosys.nyear1.1976-01-01-00000.nc - casename.pop.r.1975-01-01-00000.nc - anothercasename.pop.r.1976-01-01-00000.nc - - - - - r - hi.*\.nc$ - unset - - rpointer.wav$NINST_STRING - unset - - - r rh\d? From e0c4c82a011a7fc4bd9c94f950dac75047557ac1 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Thu, 1 Aug 2024 21:35:01 +0200 Subject: [PATCH 4/8] add first oneapi toolchain betzy --- machines/betzy/config_machines.xml | 11 +++++-- .../cmake_macros/intel-oneapi_betzy.cmake | 32 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 machines/cmake_macros/intel-oneapi_betzy.cmake diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index d96cb97c..37229fe3 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -1,7 +1,7 @@ BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM LINUX - intel + intel,intel-oneapi openmpi,impi /cluster/work/users/$USER/noresm /cluster/shared/noresm/inputdata @@ -30,6 +30,14 @@ $ENV{LMOD_PKG}/libexec/lmod python module module + + + StdEnv + /cluster/shared/noresm/eb_mods/modules/all + ESMF/8.4.2-iomkl-2022a-ParallelIO-2.5.10 + Python/3.11.3-GCCcore-12.3.0 + CMake/3.26.3-GCCcore-12.3.0 + StdEnv @@ -37,7 +45,6 @@ ESMF/8.4.1-iomkl-2021b-ParallelIO-2.5.10 CMake/3.21.1-GCCcore-11.2.0 Python/3.9.6-GCCcore-11.2.0 - ParMETIS/4.0.3-iompi-2021b diff --git a/machines/cmake_macros/intel-oneapi_betzy.cmake b/machines/cmake_macros/intel-oneapi_betzy.cmake new file mode 100644 index 00000000..3a9aab19 --- /dev/null +++ b/machines/cmake_macros/intel-oneapi_betzy.cmake @@ -0,0 +1,32 @@ +set(PIO_FILESYSTEM_HINTS "lustre") +string(APPEND CFLAGS " -qopt-report -march=core-avx2") +string(APPEND FFLAGS " -qopt-report -march=core-avx2") +set(NETCDF_C_PATH $ENV{EBROOTNETCDF}) +set(NETCDF_FORTRAN_PATH $ENV{EBROOTNETCDFMINFORTRAN}) +set(PNETCDF_PATH $ENV{PNETCDF}) + +if(DEFINED ENV{PIO}) + set(PIO_LIBDIR "$ENV{PIO}/lib") + set(PIO_INCDIR "$ENV{PIO}/include") +endif() + +string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") +string(APPEND CPPDEFS " -DHAVE_GETTID") + +if (COMP_WAV STREQUAL ww3dev) + if (MPILIB STREQUAL openmpi) + string(APPEND SLIBS " -l:libparmetis.a -lmetis ") + endif() +endif() + +if (MPILIB STREQUAL impi) + set(MPICC "mpiicc") + set(MPICXX "mpiicpc") + set(MPIFC "mpiifort") +endif() + +if (MPILIB STREQUAL openmpi) + set(MPICC "mpicc") + set(MPICXX "mpicpc") + set(MPIFC "mpifort") +endif() From f2f0a859bdaa2a4eff2a82b2299e12c433d4ad40 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Mon, 12 Aug 2024 15:47:37 +0200 Subject: [PATCH 5/8] allow error for old intel warning and change order for compilers --- machines/betzy/config_machines.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 37229fe3..99d1e913 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -1,8 +1,8 @@ BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM LINUX - intel,intel-oneapi - openmpi,impi + intel-oneapi,intel + openmpi,impi /cluster/work/users/$USER/noresm /cluster/shared/noresm/inputdata /cluster/shared/noresm/inputdata/atm/datm7 @@ -21,7 +21,7 @@ srun - + $ENV{LMOD_PKG}/init/perl $ENV{LMOD_PKG}/init/env_modules_python.py $ENV{LMOD_PKG}/init/csh From 96289a64a0fd3a355bd2388afdfd87ffa1027169 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Mon, 12 Aug 2024 18:42:43 +0200 Subject: [PATCH 6/8] Add ParMETIS --- machines/betzy/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 99d1e913..fe636fad 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -37,6 +37,7 @@ ESMF/8.4.2-iomkl-2022a-ParallelIO-2.5.10 Python/3.11.3-GCCcore-12.3.0 CMake/3.26.3-GCCcore-12.3.0 + ParMETIS/4.0.3-iompi-2022a From 80397452fe472a5c9fdaccdfb1e7ab8813741be5 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Mon, 12 Aug 2024 18:58:02 +0200 Subject: [PATCH 7/8] revert component_grid_nuopc to 0.031 --- component_grids_nuopc.xml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 53d27d2d..869e5a01 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -328,6 +328,12 @@ $DIN_LOC_ROOT/share/meshes/gx3v7_120309_ESMFmesh.nc gx3v7 is displaced Greenland pole v7 3-deg grid: + + 540 458 + $DIN_LOC_ROOT/share/meshes/tx0.66v1_190314_ESMFmesh.nc + tx0.66v1 is tripole v1 0.66-deg MOM6 grid: + Experimental for MOM6 experiments + 540 480 $DIN_LOC_ROOT/share/meshes/tx2_3v2_230415_ESMFmesh.nc @@ -517,11 +523,11 @@ gx3v7 global grid For testing of the WAV model - - 540 480 - $DIN_LOC_ROOT/share/meshes/wtx2_3v2_231005b_ESMFmesh.nc - wtx2_3v2 is tripole v2 2/3-deg WW3 grid based on MOM6 grid - MOM6 tx2_3v2-based WW3 grid where cells above 88N are masked + + 540 458 + $DIN_LOC_ROOT/share/meshes/wtx0.66v1_210917_ESMFmesh.nc + wtx0.66v1 is tripole v1 0.66-deg WW3 grid based on MOM6 grid + MOM6 tx0.66v1-based WW3 grid where cells above 88N are masked 360 384 From 9290e2269859579f4e6f70a70b2f35217e8d7587 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Mon, 12 Aug 2024 19:12:37 +0200 Subject: [PATCH 8/8] fix ww3 in cmake macro for oneapi --- machines/cmake_macros/intel-oneapi_betzy.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/cmake_macros/intel-oneapi_betzy.cmake b/machines/cmake_macros/intel-oneapi_betzy.cmake index 3a9aab19..fb169c4f 100644 --- a/machines/cmake_macros/intel-oneapi_betzy.cmake +++ b/machines/cmake_macros/intel-oneapi_betzy.cmake @@ -13,7 +13,7 @@ endif() string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") string(APPEND CPPDEFS " -DHAVE_GETTID") -if (COMP_WAV STREQUAL ww3dev) +if (COMP_WAV STREQUAL ww3) if (MPILIB STREQUAL openmpi) string(APPEND SLIBS " -l:libparmetis.a -lmetis ") endif()