chg ! tests

.pre-commit-config.yaml

@@ -3,28 +3,28 @@ repos:
     rev: 5.13.2
     hooks:
       - id: isort
-        stages: [commit]
+        stages: [pre-commit]
   - repo: https://github.com/ambv/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
         args: [--config=pyproject.toml]
         exclude: "migrations|snapshots"
-        stages: [commit]
+        stages: [pre-commit]
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: [--config=.flake8]
 
         additional_dependencies: [flake8-bugbear==22.9.23]
-        stages: [ commit ]
+        stages: [ pre-commit ]
   - repo: https://github.com/PyCQA/bandit
-    rev: '1.7.9' # Update me!
+    rev: '1.7.10' # Update me!
     hooks:
       - id: bandit
         args: ["-c", "bandit.yaml"]
   - repo: https://github.com/twisted/towncrier
-    rev: 23.11.0
+    rev: 24.8.0
     hooks:
       - id: towncrier-check

src/hope_dedup_engine/apps/api/deduplication/config.py

@@ -6,27 +6,41 @@
 
 @dataclass
 class DetectionConfig:
-    dnn_files_source: str = constance_cfg.DNN_FILES_SOURCE
-    dnn_backend: int = constance_cfg.DNN_BACKEND
-    dnn_target: int = constance_cfg.DNN_TARGET
-    blob_from_image_scale_factor: float = constance_cfg.BLOB_FROM_IMAGE_SCALE_FACTOR
-    blob_from_image_mean_values: tuple[float, float, float] = tuple(
-        map(float, constance_cfg.BLOB_FROM_IMAGE_MEAN_VALUES.split(", "))
+    dnn_files_source: str = field(
+        default_factory=lambda: constance_cfg.DNN_FILES_SOURCE
     )
-    confidence: float = constance_cfg.FACE_DETECTION_CONFIDENCE
-    nms_threshold: float = constance_cfg.NMS_THRESHOLD
+    dnn_backend: int = field(default_factory=lambda: constance_cfg.DNN_BACKEND)
+    dnn_target: int = field(default_factory=lambda: constance_cfg.DNN_TARGET)
+    blob_from_image_scale_factor: float = field(
+        default_factory=lambda: constance_cfg.BLOB_FROM_IMAGE_SCALE_FACTOR
+    )
+    blob_from_image_mean_values: tuple[float, float, float] = field(
+        default_factory=lambda: tuple(
+            map(float, constance_cfg.BLOB_FROM_IMAGE_MEAN_VALUES.split(", "))
+        )
+    )
+    confidence: float = field(
+        default_factory=lambda: constance_cfg.FACE_DETECTION_CONFIDENCE
+    )
+    nms_threshold: float = field(default_factory=lambda: constance_cfg.NMS_THRESHOLD)
 
 
 @dataclass
 class RecognitionConfig:
-    num_jitters: int = constance_cfg.FACE_ENCODINGS_NUM_JITTERS
-    model: Literal["small", "large"] = constance_cfg.FACE_ENCODINGS_MODEL
+    num_jitters: int = field(
+        default_factory=lambda: constance_cfg.FACE_ENCODINGS_NUM_JITTERS
+    )
+    model: Literal["small", "large"] = field(
+        default_factory=lambda: constance_cfg.FACE_ENCODINGS_MODEL
+    )
     preprocessors: list[str] = field(default_factory=list)
 
 
 @dataclass
 class DuplicatesConfig:
-    tolerance: float = constance_cfg.FACE_DISTANCE_THRESHOLD
+    tolerance: float = field(
+        default_factory=lambda: constance_cfg.FACE_DISTANCE_THRESHOLD
+    )
 
 
 @dataclass

src/hope_dedup_engine/apps/faces/services/image_processor.py

@@ -25,6 +25,9 @@ class BlobFromImageConfig:
     mean_values: tuple[float, float, float]
     prototxt_path: str
 
+    def __post_init__(self):
+        object.__setattr__(self, "shape", self._get_shape())
+
     def _get_shape(self) -> dict[str, int]:
         pattern = r"input_shape\s*\{\s*dim:\s*(\d+)\s*dim:\s*(\d+)\s*dim:\s*(\d+)\s*dim:\s*(\d+)\s*\}"
         with open(self.prototxt_path, "r") as file:

tests/admin/test_admin_smoke.py

@@ -183,6 +183,7 @@ def test_admin_delete(app, modeladmin, record, monkeypatch):
         pytest.skip("No 'delete' permission")
 
 
+@pytest.mark.skip_buttons("api.ConfigAdmin:change_settings_schema")
 def test_admin_buttons(app, modeladmin, button_handler, record, monkeypatch):
     from admin_extra_buttons.handlers import LinkHandler
 
@@ -197,8 +198,5 @@ def test_admin_buttons(app, modeladmin, button_handler, record, monkeypatch):
         else:
             url = reverse(f"admin:{button_handler.url_name}", args=[record.pk])
 
-        res = app.get(url, expect_errors=True)
-        if button_handler.permission:
-            assert res.status_code == 403
-        else:
-            assert res.status_code in [200, 302]
+        res = app.get(url)
+        assert res.status_code in [200, 302]

tests/api/conftest.py

@@ -20,11 +20,7 @@
     ImageFactory,
     TokenFactory,
 )
-from testutils.factories.user import (
-    ExternalSystemFactory,
-    SuperUserFactory,
-    UserFactory,
-)
+from testutils.factories.user import ExternalSystemFactory, UserFactory
 
 from hope_dedup_engine.apps.api.deduplication.registry import DuplicateFinder
 from hope_dedup_engine.apps.api.models import DeduplicationSet, HDEToken
@@ -120,12 +116,3 @@ def failing_duplicate_finder(
 ) -> DuplicateFinder:
     duplicate_finders.append(finder := FailingDuplicateFinder())
     return finder
-
-
-@fixture()
-def app(django_app_factory, mocked_responses):
-    django_app = django_app_factory(csrf_checks=False)
-    admin_user = SuperUserFactory(username="superuser")
-    django_app.set_user(admin_user)
-    django_app._user = admin_user
-    return django_app

tests/api/test_adapters.py

@@ -1,14 +1,19 @@
-from unittest.mock import MagicMock
+from typing import Any
 
-from pytest import fixture
+from pytest import fixture, mark
 from pytest_mock import MockerFixture
 
-from hope_dedup_engine.apps.api.deduplication.adapters import DuplicateFaceFinder
+from hope_dedup_engine.apps.api.deduplication.adapters import (
+    ConfigDefaults,
+    DuplicateFaceFinder,
+)
 from hope_dedup_engine.apps.api.models import DeduplicationSet, Image
 
+pytestmark = mark.django_db
+
 
 @fixture
-def duplication_detector(mocker: MockerFixture) -> MagicMock:
+def duplication_detector(mocker: MockerFixture) -> Any:
     yield mocker.patch(
         "hope_dedup_engine.apps.api.deduplication.adapters.DuplicationDetector"
     )
@@ -18,7 +23,7 @@ def test_duplicate_face_finder_uses_duplication_detector(
     deduplication_set: DeduplicationSet,
     image: Image,
     second_image: Image,
-    duplication_detector: MagicMock,
+    duplication_detector: Any,
 ) -> None:
     duplication_detector.return_value.find_duplicates.return_value = iter(
         (
@@ -33,9 +38,11 @@ def test_duplicate_face_finder_uses_duplication_detector(
     finder = DuplicateFaceFinder(deduplication_set)
     found_pairs = tuple(finder.run())
 
+    cfg = ConfigDefaults()
+    cfg.apply_config_overrides(deduplication_set.config.settings)
     duplication_detector.assert_called_once_with(
         (image.filename, second_image.filename),
-        deduplication_set.config.settings,
+        cfg=cfg,
     )
     duplication_detector.return_value.find_duplicates.assert_called_once()
     assert len(found_pairs) == 1
@@ -44,21 +51,3 @@ def test_duplicate_face_finder_uses_duplication_detector(
         second_image.reference_pk,
         1 - distance,
     )
-
-
-def _run_duplicate_face_finder(deduplication_set: DeduplicationSet) -> None:
-    finder = DuplicateFaceFinder(deduplication_set)
-    tuple(finder.run())  # tuple is used to make generator finish execution
-
-
-def test_duplication_detector_is_initiated_with_correct_face_distance_threshold_value(
-    deduplication_set: DeduplicationSet,
-    duplication_detector: MagicMock,
-) -> None:
-    # deduplication set face_distance_threshold config value is used
-    _run_duplicate_face_finder(deduplication_set)
-    duplication_detector.assert_called_once_with((), deduplication_set.config.settings)
-    duplication_detector.reset_mock()
-    deduplication_set.config = None
-    _run_duplicate_face_finder(deduplication_set)
-    duplication_detector.assert_called_once_with((), {})

tests/api/test_config.py

@@ -7,7 +7,9 @@
 from testutils.factories.api import ConfigFactory, DeduplicationSetFactory
 
 
-def test_response_change_redirects_to_confirm_save_if_related_objects_exist(app):
+def test_response_change_redirects_to_confirm_save_if_related_objects_exist(
+    admin_client,
+):
     config_instance = ConfigFactory.create()
     deduplication_sets = DeduplicationSetFactory.create_batch(2, config=config_instance)
     change_url = reverse("admin:api_config_change", args=[config_instance.pk])
@@ -19,13 +21,13 @@ def test_response_change_redirects_to_confirm_save_if_related_objects_exist(app)
         ),
     }
 
-    response = app.post(change_url, form_data)
+    response = admin_client.post(change_url, form_data)
     assert response.status_code == 302
-    assert response.location == confirm_url
+    assert response.url == confirm_url
 
-    response = response.follow()
+    response = admin_client.get(confirm_url)
     assert response.status_code == 200
-    assert response.request.path == confirm_url
+    assert response.wsgi_request.path == confirm_url
 
     messages = [str(msg) for msg in get_messages(response.context["request"])]
     assert all(

tests/extras/testutils/factories/api.py

@@ -1,7 +1,13 @@
-from factory import SubFactory, fuzzy, post_generation
+from factory import Factory, Faker, LazyFunction, SubFactory, fuzzy, post_generation
 from factory.django import DjangoModelFactory
 from testutils.factories import ExternalSystemFactory, UserFactory
 
+from hope_dedup_engine.apps.api.deduplication.config import (
+    ConfigDefaults,
+    DetectionConfig,
+    DuplicatesConfig,
+    RecognitionConfig,
+)
 from hope_dedup_engine.apps.api.models import DedupJob, DeduplicationSet, HDEToken
 from hope_dedup_engine.apps.api.models.config import Config
 from hope_dedup_engine.apps.api.models.deduplication import (
@@ -92,3 +98,45 @@ class DedupJobFactory(DjangoModelFactory):
 
     class Meta:
         model = DedupJob
+
+
+class DetectionConfigFactory(Factory):
+    class Meta:
+        model = DetectionConfig
+
+    dnn_files_source = Faker("word")
+    dnn_backend = fuzzy.FuzzyInteger(0, 5)
+    dnn_target = fuzzy.FuzzyInteger(0, 5)
+    blob_from_image_scale_factor = fuzzy.FuzzyFloat(0.5, 1.5)
+    blob_from_image_mean_values = LazyFunction(lambda: (104.0, 177.0, 123.0))
+    confidence = fuzzy.FuzzyFloat(0.1, 1.0)
+    nms_threshold = fuzzy.FuzzyFloat(0.1, 1.0)
+
+
+class RecognitionConfigFactory(Factory):
+    class Meta:
+        model = RecognitionConfig
+
+    num_jitters = fuzzy.FuzzyInteger(0, 5)
+    model = fuzzy.FuzzyChoice(["small", "large"])
+    preprocessors = []
+
+
+class DuplicatesConfigFactory(Factory):
+    class Meta:
+        model = DuplicatesConfig
+
+    tolerance = fuzzy.FuzzyFloat(0.1, 1.0)
+
+
+class ConfigDefaultsFactory(Factory):
+    class Meta:
+        model = ConfigDefaults
+
+    detection = SubFactory(DetectionConfigFactory)
+    recognition = SubFactory(RecognitionConfigFactory)
+    duplicates = SubFactory(DuplicatesConfigFactory)
+
+    # @post_generation
+    # def apply_overrides(self, create, extracted, **kwargs):
+    #         self.apply_config_overrides(extracted)

tests/faces/conftest.py

@@ -13,9 +13,6 @@
     DEPLOY_PROTO_CONTENT,
     DEPLOY_PROTO_SHAPE,
     DNN_FILE,
-    DS_CONFIG,
-    FACE_DETECTIONS,
-    FACE_REGIONS_VALID,
     FILENAMES,
     IGNORE_PAIRS,
     IMAGE_SIZE,
@@ -25,6 +22,7 @@
 from PIL import Image
 from pytest_mock import MockerFixture
 from storages.backends.azure_storage import AzureStorage
+from testutils.factories.api import ConfigDefaultsFactory
 
 from docker import from_env
 from hope_dedup_engine.apps.faces.managers import DNNInferenceManager, StorageManager
@@ -107,17 +105,25 @@ def mock_net_manager(mocker: MockerFixture) -> DNNInferenceManager:
     yield mock_net
 
 
+@pytest.fixture
+def mock_config_defaults():
+    return ConfigDefaultsFactory()
+
+
 @pytest.fixture
 def mock_image_processor(
     mocker: MockerFixture,
     mock_storage_manager,
+    mock_config_defaults,
     mock_net_manager,
     mock_open_context_manager,
 ) -> ImageProcessor:
     mocker.patch.object(
         BlobFromImageConfig, "_get_shape", return_value=DEPLOY_PROTO_SHAPE
     )
-    mock_processor = ImageProcessor(DS_CONFIG)
+    mock_processor = ImageProcessor(
+        mock_config_defaults.detection, mock_config_defaults.recognition
+    )
     mocker.patch.object(
         mock_processor.storages.get_storage("images"),
         "open",
@@ -144,12 +150,32 @@ def mock_open_context_manager(image_bytes_io):
 
 
 @pytest.fixture
-def mock_net():
+def mock_face_detections(mock_config_defaults):
+    conf = mock_config_defaults.detection.confidence
+    face_detections = np.array(
+        [
+            [
+                [
+                    (0, 0, conf + 0.01, 0.1, 0.1, 0.2, 0.2),
+                    (0, 0, conf + 0.1, 0.3, 0.3, 0.4, 0.4),
+                    (0, 0, conf - 0.01, 0.1, 0.1, 0.2, 0.2),
+                ]
+            ]
+        ],
+        dtype=np.float32,
+    )
+    face_regions_valid = [
+        (120, 120, 160, 160),
+        (40, 40, 80, 80),
+    ]
+    face_regions_invalid = [[], [(0, 0, 10)]]
+    yield face_detections, face_regions_valid, face_regions_invalid
+
+
+@pytest.fixture
+def mock_net(mock_face_detections):
     mock_net = MagicMock(spec=cv2.dnn_Net)  # Mocking the neural network object
-    mock_detections = np.array(
-        [[FACE_DETECTIONS]], dtype=np.float32
-    )  # Mocking the detections array
-    mock_expected_regions = FACE_REGIONS_VALID
+    mock_detections, mock_expected_regions, _ = mock_face_detections
     mock_net.forward.return_value = (
         mock_detections  # Setting up the forward method of the mock network
     )
@@ -160,8 +186,10 @@ def mock_net():
 
 
 @pytest.fixture
-def mock_dd(mock_image_processor, mock_net_manager, mock_storage_manager):
-    detector = DuplicationDetector(FILENAMES, DS_CONFIG, IGNORE_PAIRS)
+def mock_dd(
+    mock_image_processor, mock_net_manager, mock_storage_manager, mock_config_defaults
+):
+    detector = DuplicationDetector(FILENAMES, mock_config_defaults, IGNORE_PAIRS)
     yield detector