Fix Bug in sample Parameter in feature_select(): Prevent Unintended Row Removal and Dataset Modification

.gitignore

@@ -168,3 +168,6 @@ cython_debug/
 
 # Miscellaneous
 .DS_Store
+
+# removes files that are meant for prototyping
+_*

pycytominer/cyto_utils/features.py

@@ -175,7 +175,7 @@ def drop_outlier_features(
 
     # Subset dataframe
     if samples != "all":
-        population_df.query(samples, inplace=True)
+        population_df = population_df.query(samples)
 
     if features == "infer":
         features = infer_cp_features(population_df)

pycytominer/operations/correlation_threshold.py

@@ -48,7 +48,7 @@ def correlation_threshold(
 
     # Subset dataframe and calculate correlation matrix across subset features
     if samples != "all":
-        population_df.query(samples, inplace=True)
+        population_df = population_df.query(samples)
 
     if features == "infer":
         features = infer_cp_features(population_df)

pycytominer/operations/get_na_columns.py

@@ -33,7 +33,7 @@ def get_na_columns(population_df, features="infer", samples="all", cutoff=0.05):
     """
 
     if samples != "all":
-        population_df.query(samples, inplace=True)
+        population_df = population_df.query(samples)
 
     if features == "infer":
         features = infer_cp_features(population_df)

pycytominer/operations/noise_removal.py

@@ -43,7 +43,7 @@ def noise_removal(
     """
     # Subset dataframe
     if samples != "all":
-        population_df.query(samples, inplace=True)
+        population_df = population_df.query(samples)
 
     if features == "infer":
         features = infer_cp_features(population_df)

pycytominer/operations/variance_threshold.py

@@ -51,7 +51,7 @@ def variance_threshold(
 
     # Subset dataframe
     if samples != "all":
-        population_df.query(samples, inplace=True)
+        population_df = population_df.query(samples)
 
     if features == "infer":
         features = infer_cp_features(population_df)

tests/test_feature_select.py

@@ -169,6 +169,58 @@ def test_feature_select_noise_removal():
     pd.testing.assert_frame_equal(result4b, expected_result4b)
     pd.testing.assert_frame_equal(result5b, expected_result5b)
 
+    # testing samples query leveraging metadata
+    data_unique_test_df3 = data_unique_test_df.copy()
+    data_unique_test_df3_features = data_unique_test_df3.columns.tolist()
+
+    # adding metadata column in order to create a samples query
+    data_unique_test_df3["Metadata_sample"] = ["A", "B"] * 50
+    sample_query = "Metadata_sample == 'A'"
+
+    # adding perturb group metadata column for noise_removal operation
+    data_unique_test_df3["perturb_group"] = data_unique_test_df_groups
+
+    # establishing all operations that use "samples" parameter
+    # note that blocklist does not use samples parameter
+    all_operations = [
+        "noise_removal",
+        "drop_na_columns",
+        "variance_threshold",
+        "correlation_threshold",
+        "drop_outliers",
+    ]
+    for operation_idx, operation in enumerate(all_operations):
+        # testing single operation
+        results6a = feature_select(
+            profiles=data_unique_test_df3,
+            features=data_unique_test_df3_features,
+            operation=operation,
+            samples=sample_query,
+            noise_removal_perturb_groups="perturb_group",
+            noise_removal_stdev_cutoff=500,
+        )
+
+        # checking if no rows were not removed
+        assert (
+            results6a.shape[0] == data_unique_test_df3.shape[0]
+        ), f"Row counts do not match: {results6a[0]} != {data_unique_test_df3.shape[0]} in operation: {operation}"
+
+        # testing multiple operations (continually appends operations)
+        concat_operations = all_operations[: operation_idx + 1]
+        results6b = feature_select(
+            profiles=data_unique_test_df3,
+            features=data_unique_test_df3_features,
+            operation=concat_operations,
+            samples=sample_query,
+            noise_removal_perturb_groups="perturb_group",
+            noise_removal_stdev_cutoff=500,
+        )
+
+        # checking if no rows were not removed
+        assert (
+            results6b.shape[0] == data_unique_test_df3.shape[0]
+        ), f"Row counts do not match: {results6a[0]} != {data_unique_test_df3.shape[0]} in operation: {concat_operations}"
+
     # Test assertion errors for the user inputting the perturbation groupings
     bad_perturb_list = ["a", "a", "b", "b", "a", "a", "b"]