Finalize segmentation

.gitignore

@@ -165,3 +165,8 @@ cython_debug/
 data/*
 1.segment_images/examples/raw_z_input/*
 1.segment_images/processed_data/*
+1.segment_images/scripts/job_ids.txt
+1.segment_images/*.out
+
+# SLURM output_files
+*.out

.pre-commit-config.yaml

@@ -30,7 +30,7 @@ repos:
 
   # remove unused imports
 -   repo: https://github.com/hadialqattan/pycln.git
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
     -   id: pycln
 

0.preprocessing_data/HPC_preprocessing.sh

@@ -17,6 +17,7 @@ cd scripts/ || exit
 
 python 0.update_file_structure.py --HPC True
 python 1.make_z-stack_images.py
+python 3.data_validation.py
 
 cd .. || exit
 

0.preprocessing_data/local_preprocessing.sh

@@ -8,6 +8,8 @@ cd scripts/ || exit
 
 python 0.update_file_structure.py --HPC False
 python 1.make_z-stack_images.py
+python 3.data_validation.py
+
 
 cd .. || exit
 

0.preprocessing_data/notebooks/3.data_validation.ipynb

@@ -0,0 +1,123 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": []
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import pathlib\n",
+                "\n",
+                "import numpy as np\n",
+                "import skimage.io as io"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def image_directory_check(\n",
+                "    image_directory: pathlib.Path, expected_number_of_files: int = 5\n",
+                ") -> int:\n",
+                "    counter = 0\n",
+                "    files = [x for x in image_directory.iterdir() if x.is_file()]\n",
+                "    if len(files) < expected_number_of_files or len(files) > expected_number_of_files:\n",
+                "        print(f\"{image_directory} has {len(files)} files\")\n",
+                "        counter += 1\n",
+                "    for file in files:\n",
+                "        image = io.imread(file)\n",
+                "        # check that image contains non-zero values\n",
+                "        if np.count_nonzero(image) == 0:\n",
+                "            print(f\"{file} contains only zeros\")\n",
+                "            counter += 1\n",
+                "    return counter"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "z_stack_images_dir = pathlib.Path(\"../../data/z-stack_images/\").resolve(strict=True)\n",
+                "normalized_z_stack_images_dir = pathlib.Path(\"../../data/normalized_z/\").resolve(\n",
+                "    strict=True\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 4,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "/home/lippincm/Documents/GFF_3D_organoid_profiling_pipeline/data/z-stack_images/F3-1/F3-1_555.tif contains only zeros\n",
+                        "/home/lippincm/Documents/GFF_3D_organoid_profiling_pipeline/data/z-stack_images/F3-1/F3-1_488.tif contains only zeros\n",
+                        "/home/lippincm/Documents/GFF_3D_organoid_profiling_pipeline/data/z-stack_images/F3-1/F3-1_405.tif contains only zeros\n",
+                        "/home/lippincm/Documents/GFF_3D_organoid_profiling_pipeline/data/z-stack_images/F3-1/F3-1_TRANS.tif contains only zeros\n",
+                        "/home/lippincm/Documents/GFF_3D_organoid_profiling_pipeline/data/z-stack_images/F3-1/F3-1_640.tif contains only zeros\n",
+                        "/home/lippincm/Documents/GFF_3D_organoid_profiling_pipeline/data/normalized_z/F3-1 has 0 files\n",
+                        "Total number of issues in z-stack images: 5\n",
+                        "Total number of issues in normalized z-stack images: 1\n"
+                    ]
+                }
+            ],
+            "source": [
+                "z_stack_images_dir_directories = [x for x in z_stack_images_dir.iterdir() if x.is_dir()]\n",
+                "\n",
+                "z_stack_counter = 0\n",
+                "for dir in z_stack_images_dir_directories:\n",
+                "    directory_counter = image_directory_check(\n",
+                "        image_directory=dir, expected_number_of_files=5\n",
+                "    )\n",
+                "    z_stack_counter += directory_counter\n",
+                "\n",
+                "normalized_z_stack_images_dir_directories = [\n",
+                "    x for x in normalized_z_stack_images_dir.iterdir() if x.is_dir()\n",
+                "]\n",
+                "normalized_z_stack_counter = 0\n",
+                "for dir in normalized_z_stack_images_dir_directories:\n",
+                "    directory_counter = image_directory_check(\n",
+                "        image_directory=dir, expected_number_of_files=5\n",
+                "    )\n",
+                "    normalized_z_stack_counter += directory_counter\n",
+                "\n",
+                "print(f\"Total number of issues in z-stack images: {z_stack_counter}\")\n",
+                "print(\n",
+                "    f\"Total number of issues in normalized z-stack images: {normalized_z_stack_counter}\"\n",
+                ")"
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "gff_preprocessing_env",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.10.15"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 2
+}

1.segment_images/HPC_segmentation.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --partition=amilan
+#SBATCH --qos=long
+#SBATCH --account=amc-general
+#SBATCH --time=7-00:00:00
+#SBATCH --output=main_job_submit_output-%j.out
+
+cd slurm_scripts || exit
+
+job1=$(sbatch 0.inital_segmentation_HPC.sh | awk '{print $4}')
+
+job2=$(sbatch --dependency=afterok:$job1 1a.process_segmentation_parent.sh | awk '{print $4}')
+
+job3=$(sbatch --dependency=afterok:$job2 2a.process_segmentation_parent.sh | awk '{print $4}')
+
+job4=$(sbatch --dependency=afterok:$job3 3a.process_segmentation_parent.sh | awk '{print $4}')
+
+job5=$(sbatch --dependency=afterok:$job4 4.check_job_status.sh | awk '{print $4}')
+
+echo "All jobs submitted"
+
+exit 0

1.segment_images/README.md

@@ -12,3 +12,4 @@ For nuclei and cells, we take a sliding window of n z-slices, where we perform m
 We then decouple the max-projection to collect a segmentation mask per z-slice.
 We then output the masks across all z-slices per image set as a stacked-TIFF file.
 
+Well F4-2 for the NF0014 group throws an error in cellpose - I will exclude these images from the segmentation process.

1.segment_images/local_segmentation.sh

@@ -1,45 +1,43 @@
 #!/bin/bash
 
-NOTEBOOK=False
-
 # activate  cellprofiler environment
 conda activate GFF_segmentation
 
 jupyter nbconvert --to=script --FilesWriter.build_directory=scripts/ notebooks/*.ipynb
 
-# run the notebooks with papermill
-if [ "$NOTEBOOK" = True ]; then
-    cd notebooks/ || exit
-    papermill 0.segment_nuclei_organoids.ipynb 0.segment_nuclei_organoids.ipynb
-    papermill 1.segment_cells_organoids.ipynb 1.segment_cells_organoids.ipynb
-    papermill 2.make_nuclei_segmentation_videos.ipynb 2.make_nuclei_segmentation_videos.ipynb
-    papermill 3.make_cell_segmentation_videos.ipynb 3.make_cell_segmentation_videos.ipynb
-    cd ../ || exit
-fi
-
-if [ "$NOTEBOOK" = False ]; then
-    cd scripts/ || exit
-    # get all input directories in specified directory
-    z_stack_dir="../../data/z-stack_images/"
-    input_dirs=$(ls -d $z_stack_dir*)
-
-
-    total_dirs=$(echo "$input_dirs" | wc -l)
-    current_dir=0
-
-    # loop through all input directories
-    for dir in $input_dirs; do
-        dir=${dir%*/}
-        current_dir=$((current_dir + 1))
-        echo -ne "Processing directory $current_dir of $total_dirs\r"
-        python 0.segment_nuclei_organoids.py --input_dir "$dir" --window_size 3 --clip_limit 0.05
-        python 1.segment_cells_organoids.py --input_dir "$dir" --window_size 3 --clip_limit 0.1
-        python 2.make_nuclei_segmentation_videos.py --input_dir "$dir"
-        python 3.make_cell_segmentation_videos.py --input_dir "$dir"
+
+cd scripts/ || exit
+# get all input directories in specified directory
+# z_stack_dir="../../data/z-stack_images"
+z_stack_dir="../../data/test_dir/"
+mapfile -t input_dirs < <(ls -d "$z_stack_dir"/*)
+
+total_dirs=$(echo "${input_dirs[@]}" | wc -w)
+echo "Total directories: $total_dirs"
+current_dir=0
+compartments=( "nuclei" "cell" )
+
+# loop through all input directories
+for dir in "${input_dirs[@]}"; do
+    dir=${dir%*/}
+    current_dir=$((current_dir + 1))
+    echo -ne "Processing directory $current_dir of $total_dirs\r"
+    python 0.segment_nuclei_organoids.py --input_dir "$dir" --window_size 3 --clip_limit 0.05
+    python 1.segment_cells_organoids.py --input_dir "$dir" --window_size 3 --clip_limit 0.1
+    python 2.segment_whole_organoids.py --input_dir "$dir" --window_size 3 --clip_limit 0.1
+    for compartment in "${compartments[@]}"; do
+        python 3.segmentation_decoupling.py --input_dir "$dir" --compartment "$compartment"
+        python 4.reconstruct_3D_masks.py --input_dir "$dir" --compartment "$compartment" --radius_constraint 10
+        python 6.make_cell_segmentation_videos.py --input_dir "$dir" --compartment "$compartment"
     done
-    echo -ne "\n"
-    cd ../ || exit
-fi
+    python 5.create_cytoplasm_masks.py --input_dir "$dir"
+    python 6.make_cell_segmentation_videos.py --input_dir "$dir" --compartment "cytoplasm"
+    python 7.clean_up_segmentation.py
+
+done
+echo -ne "\n"
+cd ../ || exit
+
 
 # deactivate cellprofiler environment
 conda deactivate