Fixes for VirtualBox (#755)

Changes to the VirtualBox/* files [ci skip] * call docker scripts in nearly all INSTALL*sh * don't install pip etc via apt, but get latest version (as in docker). This avoids problems with apt/distutils packages * uninstall python3-pexpect as it creates conflicts * install python dependencies with requirements*.txt * removed numpy <1.20 requirement * set max version for numba * update to FindCython * use python3/cython in the PATH as we no longer install cython via apt Co-authored-by: Kris Thielemans <[email protected]>
SyneRBI · Aug 18, 2022 · 57ae12e · 57ae12e
1 parent fef2163
commit 57ae12e
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 45 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,15 @@
 # ChangeLog
-# v3.3.0
+## v3.3.1
+- VM: 
+   - Updates to run using docker scripts
+   - installs custom pip and all python prerequisites with pip
+   - Bugfix in finding cython and python in UPDATE.sh
+- CMake:
+   - FindCython allows hints
+- updated versions:
+   - numba < 0.51, due to incompatibilities with python 3.6 (which is installed in the GHA and VM)
+
+## v3.3.0
 - known problems:
    - VM and jupyterhub scripts need merging various fixes
 - gemeric *.cmake fixes:

diff --git a/CMake/FindCython.cmake b/CMake/FindCython.cmake
@@ -11,40 +11,57 @@
 SET(CYTHON_EXECUTABLE cython CACHE STRING "Cython executable name")
 SET(CYTHON_FLAGS --cplus --fast-fail)
 
-SET(Cython_FOUND FALSE)
-IF (CYTHON_EXECUTABLE)
-    # Try to run Cython, to make sure it works:
-    execute_process(
-        COMMAND ${CYTHON_EXECUTABLE} "--version"
-        RESULT_VARIABLE CYTHON_RESULT
-        OUTPUT_QUIET
-        ERROR_QUIET
-        )
-    if (CYTHON_RESULT EQUAL 0)
-        # Only if cython exits with the return code 0, we know that all is ok:
-        SET(Cython_FOUND TRUE)
-        SET(Cython_Compilation_Failed FALSE)
-    else (CYTHON_RESULT EQUAL 0)
-        SET(Cython_Compilation_Failed TRUE)
-    endif (CYTHON_RESULT EQUAL 0)
-ENDIF (CYTHON_EXECUTABLE)
+if (CYTHON_ROOT)
+  set(CYTHON_EXECUTABLES "${CYTHON_EXECUTABLE};${CYTHON_ROOT}/cython")
+else()
+  set(CYTHON_EXECUTABLES "${CYTHON_EXECUTABLE}")
+endif()
+
+message(STATUS "Looking for Cython among ${CYTHON_EXECUTABLES}")
+
+set(Cython_FOUND FALSE)
+foreach(cy_exe IN LISTS CYTHON_EXECUTABLES)
+    message(status "Trying with ${cy_exe}")
+    IF (cy_exe)
+        # Try to run Cython, to make sure it works:
+        execute_process(
+            COMMAND ${cy_exe} "--version"
+            RESULT_VARIABLE CYTHON_RESULT
+            OUTPUT_QUIET 
+            ERROR_VARIABLE cython_version
+            )
+        if (CYTHON_RESULT EQUAL 0)
+            # Only if cython exits with the return code 0, we know that all is ok:
+            SET(Cython_FOUND TRUE)
+            SET(Cython_Compilation_Failed FALSE)
+            set(CYTHON_EXECUTABLE ${cy_exe})
+            message(status "YAS! ${cy_exe} ${CYTHON_EXECUTABLE} ${cython_version}")
+            break()
+        else (CYTHON_RESULT EQUAL 0)
+            SET(Cython_Compilation_Failed TRUE)
+        endif (CYTHON_RESULT EQUAL 0)
+    endif (cy_exe)
+endforeach()
+
+
 
 
 IF (Cython_FOUND)
 	IF (NOT Cython_FIND_QUIETLY)
-		MESSAGE(STATUS "Found CYTHON: ${CYTHON_EXECUTABLE}")
+		MESSAGE(STATUS "Found CYTHON version ${cython_version}: ${CYTHON_EXECUTABLE}")
 	ENDIF (NOT Cython_FIND_QUIETLY)
 ELSE (Cython_FOUND)
-	IF (Cython_FIND_REQUIRED)
+    message(WARNING "Cython not found!")
+	if (Cython_FIND_REQUIRED)
         if(Cython_Compilation_Failed)
-            MESSAGE(STATUS "Found CYTHON: ${CYTHON_EXECUTABLE}")
+            MESSAGE(STATUS "Found CYTHON: ${CYTHON_EXECUTABLE} But compilation failed")
 			# On Win the testing of Cython does not return any accessible value, so the test is not carried out. Fresh Cython install was tested and works.
 			IF(NOT MSVC)
 				MESSAGE(FATAL_ERROR "Your Cython version is too old. Please upgrade Cython.")
 			ENDIF(NOT MSVC)
         else(Cython_Compilation_Failed)
             MESSAGE(FATAL_ERROR "Could not find Cython. Please install Cython.")
         endif(Cython_Compilation_Failed)
-	ENDIF (Cython_FIND_REQUIRED)
-ENDIF (Cython_FOUND)
+	endif (Cython_FIND_REQUIRED)
+endif (Cython_FOUND)
 
diff --git a/VirtualBox/scripts/INSTALL_prerequisites_with_apt-get.sh b/VirtualBox/scripts/INSTALL_prerequisites_with_apt-get.sh
@@ -14,6 +14,9 @@ SUDO=sudo
 
 # TODO would be better to guarantee absolute path for SCRIPTS
 SCRIPTS="$(dirname $0)/../../docker"
+# uninstall pexpect which is preinstalled in the VM
+# https://github.com/SyneRBI/SIRF-SuperBuild/issues/742#issuecomment-1205090681
+$SUDO apt remove -y python3-pexpect
 
 $SUDO "$SCRIPTS/build_essential-ubuntu.sh"
 
@@ -36,11 +39,11 @@ ${APT_GET_INSTALL} expect
 echo "Installing python APT packages"
 # we will use pip for most
 # some extra package needed for jupyter
-qt=pyqt5
-${APT_GET_INSTALL} python3-dev python3-pip python3-tk python3-${qt} python3-${qt}.qtsvg python3-${qt}.qtwebkit
+# qt=pyqt5
+# ${APT_GET_INSTALL} python3-dev python3-pip python3-tk python3-${qt} python3-${qt}.qtsvg python3-${qt}.qtwebkit
 
-echo "Installing CIL pre-requisites..."
-${APT_GET_INSTALL} cython3 python3-h5py python3-wget
+# echo "Installing CIL pre-requisites..."
+# ${APT_GET_INSTALL} cython3 python3-h5py python3-wget
 # dependency for ASTRA-toolbox autotools-dev automake autogen autoconf libtool
 
 echo "Run INSTALL_python_packages.sh after this."
diff --git a/VirtualBox/scripts/INSTALL_python_packages.sh b/VirtualBox/scripts/INSTALL_python_packages.sh
@@ -65,12 +65,25 @@ while (( "$#" )); do
   esac
 done
 
-$PYTHON -m pip install $PIPOPTIONS --upgrade pip wheel setuptools
-$PYTHON -m pip install $PIPOPTIONS --only-binary=numpy,scipy,matplotlib numpy scipy matplotlib nose coverage docopt deprecation nibabel pytest tqdm
-$PYTHON -m pip install $PIPOPTIONS jupyter spyder
-$PYTHON -m pip uninstall $PIPOPTIONS -y spyder-kernels
+export DEBIAN_FRONTEND=noninteractive
+SUDO=sudo
+APT_GET_INSTALL="$SUDO apt-get install -y --no-install-recommends"
+${APT_GET_INSTALL} ${PYTHON}-dev  
+# TODO would be better to guarantee absolute path for SCRIPTS
+SCRIPTS="$(dirname $0)/../../docker"
+# installs pip
+curl $($PYTHON ${SCRIPTS}/get_pip_download_link.py) > get-pip.py
+${PYTHON} get-pip.py
+rm get-pip.py
+
+${PYTHON} -m pip install $PIPOPTIONS -U setuptools wheel
+${PYTHON} -m pip install $PIPOPTIONS -U -r ${SCRIPTS}/requirements.txt
+${PYTHON} -m pip install $PIPOPTIONS -U -r ${SCRIPTS}/requirements-service.txt
+# $PYTHON -m pip install $PIPOPTIONS --upgrade pip wheel setuptools
+# $PYTHON -m pip install $PIPOPTIONS --only-binary=numpy,scipy,matplotlib numpy scipy matplotlib nose coverage docopt deprecation nibabel pytest tqdm
+# $PYTHON -m pip install $PIPOPTIONS jupyter spyder
+# $PYTHON -m pip uninstall $PIPOPTIONS -y spyder-kernels
 
 # otherwise Jupyter uses py 2 even when you choose py3: https://github.com/jupyter/jupyter/issues/270
 $PYTHON -m ipykernel install --user  
-# CIL
-$PYTHON -m pip install $PIPOPTIONS pillow olefile
+
diff --git a/VirtualBox/scripts/UPDATE.sh b/VirtualBox/scripts/UPDATE.sh
@@ -141,9 +141,21 @@ if [ -d $SIRF_SRC_PATH/CCPPETMR_VM ]; then
 fi
 
 SIRF_INSTALL_PATH=$SIRF_SRC_PATH/install
-# best to use full path
+
+# best to use full path for python3/cython
+PYTHON_EXECUTABLE=$(which python3)
+if which python3; then
+  PYTHON_EXECUTABLE=$(which python3)
+else
+  PYTHON_EXECUTABLE=$(which python)
+fi
 PYTHON_EXECUTABLE=/usr/bin/python3
-CYTHON_EXECUTABLE=/usr/bin/cython3
+
+if which cython3; then
+    CYTHON_EXECUTABLE=$(which cython3)
+else
+  CYTHON_EXECUTABLE=$(which cython)
+fi
 
 # ignore notebook keys, https://github.com/CCPPETMR/SIRF-Exercises/issues/20
 "$PYTHON_EXECUTABLE" -m pip install -U --user nbstripout
@@ -203,7 +215,8 @@ SuperBuild(){
         -DBUILD_CIL=ON\
         -DCYTHON_EXECUTABLE="$CYTHON_EXECUTABLE"\
         -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE"\
-        -DBUILD_pet_rd_tools=ON
+        -DBUILD_pet_rd_tools=ON\
+        -DCYTHON_ROOT="${HOME}/.local/bin"
   cmake --build . -j${num_parallel}
 
   if [ ! -f ${SIRF_INSTALL_PATH}/share/gadgetron/config/gadgetron.xml ]

diff --git a/VirtualBox/scripts/bootstrap_ubuntu.sh b/VirtualBox/scripts/bootstrap_ubuntu.sh
@@ -79,4 +79,5 @@ bash $userHOME/devel/SIRF-SuperBuild/VirtualBox/scripts/INSTALL_python_packages.
 # Therefore no browser is installed on the VM
 
 chown -R $SIRFUSERNAME:users $userHOME
+export PATH=${userHOME}/.local/bin:${PATH}
 sudo -u $SIRFUSERNAME -H bash $userHOME/devel/SIRF-SuperBuild/VirtualBox/scripts/UPDATE.sh
diff --git a/docker/build_essential-ubuntu.sh b/docker/build_essential-ubuntu.sh
@@ -22,17 +22,17 @@ pushd $INSTALL_DIR
 # CMake
 curl -o cmake.tgz -L https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz
 tar xzf cmake.tgz && rm cmake.tgz
-ln -s cmake-*x86_64 cmake
+ln -s cmake-*x86_64 cmake || true
 export PATH="$PWD/cmake/bin:$PATH"
 
 # ccache
 mkdir -p bin
 pushd bin
 # ccache compiler override
-ln -s "$(which ccache)" g++
-ln -s "$(which ccache)" g++-8
-ln -s "$(which ccache)" gcc
-ln -s "$(which ccache)" gcc-8
+ln -s "$(which ccache)" g++ || true
+ln -s "$(which ccache)" g++-8 || true
+ln -s "$(which ccache)" gcc || true
+ln -s "$(which ccache)" gcc-8 || true
 export PATH="$PWD:$PATH"
 popd
 

diff --git a/docker/requirements.txt b/docker/requirements.txt
@@ -1,18 +1,22 @@
+setuptools
+wheel
+numpy        # CIL
+scipy        # CIL
 docopt
 matplotlib
 git+https://github.com/ismrmrd/ismrmrd-python-tools.git@master#egg=ismrmrd-python-tools
 Cython       # CIL
-numpy<=1.20  # CIL
-scipy        # CIL
 h5py         # CIL
 Pillow       # CIL
 wget         # CIL
 six          # CIL
 olefile      # CIL
 git+https://github.com/data-exchange/dxchange.git  # CIL
-numba        # CIL
+# https://numba.discourse.group/t/numba-0-50-0-and-llvmlite-0-33-0-released/19
+# https://github.com/numba/llvmlite/issues/621
+numba<0.51        # CIL
 pandas
 tifffile
 nibabel
 deprecation
-nose
+nose