From b83c048c7ce2fbc3c18a157af4cd5bbad1c8c6e9 Mon Sep 17 00:00:00 2001 From: Marcos Prieto Date: Thu, 23 Jan 2025 11:53:22 +0100 Subject: [PATCH] Allow to add annotations to the job queue by ID --- h/services/job_queue.py | 15 ++++++++++- h/tasks/job_queue.py | 9 +++++++ h/templates/admin/search.html.jinja2 | 8 ++++++ h/views/admin/search.py | 36 +++++++++++++++++++++++++ tests/unit/h/services/job_queue_test.py | 21 +++++++++++++++ tests/unit/h/tasks/job_queue_test.py | 19 +++++++++++++ tests/unit/h/views/admin/search_test.py | 22 +++++++++++++++ 7 files changed, 129 insertions(+), 1 deletion(-) diff --git a/h/services/job_queue.py b/h/services/job_queue.py index 956907970a7..2851ee54db1 100644 --- a/h/services/job_queue.py +++ b/h/services/job_queue.py @@ -10,7 +10,8 @@ class Priority: SINGLE_ITEM = 1 SINGLE_USER = 100 SINGLE_GROUP = 100 - BETWEEN_TIMES = 1000 + BETWEEN_TIMES = 1_000 + BY_IDS = 1_000 class JobQueueService: @@ -60,6 +61,18 @@ def add_by_id(self, name, annotation_id, tag, force=False, schedule_in=None): where = [Annotation.id == annotation_id] self.add_where(name, where, tag, Priority.SINGLE_ITEM, force, schedule_in) + def add_by_ids( + self, name, annotation_ids: list[str], tag, force=False, schedule_in=None + ): + """ + Queue annotations by ID. + + :param annotation_ids: List of annotation IDs to be queued, in the + application-level URL-safe format + """ + where = [Annotation.id.in_(annotation_ids)] + self.add_where(name, where, tag, Priority.BY_IDS, force, schedule_in) + def add_by_user(self, name, userid: str, tag, force=False, schedule_in=None): """ Queue all a user's annotations. diff --git a/h/tasks/job_queue.py b/h/tasks/job_queue.py index adce83ef553..ff788efbb3f 100644 --- a/h/tasks/job_queue.py +++ b/h/tasks/job_queue.py @@ -25,3 +25,12 @@ def add_annotations_from_group(name, groupid, tag, force=False, schedule_in=None celery.request.find_service(name="queue_service").add_by_group( name, groupid, tag, force=force, schedule_in=schedule_in ) + + +@celery.task +def add_annotations_by_ids( + name, annotation_ids: list[str], tag, force=False, schedule_in=None +): + celery.request.find_service(name="queue_service").add_by_ids( + name, annotation_ids, tag, force=force, schedule_in=schedule_in + ) diff --git a/h/templates/admin/search.html.jinja2 b/h/templates/admin/search.html.jinja2 index 854ee1381ee..d4d9d29c3f5 100644 --- a/h/templates/admin/search.html.jinja2 +++ b/h/templates/admin/search.html.jinja2 @@ -64,5 +64,13 @@ {{ force_checkbox("reindex_group_force") }} {% endcall %} + + {% call reindex_form(heading="Process all by ID", action="reindex_ids") %} +
+ + +
+ {{ force_checkbox("reindex_ids_force") }} + {% endcall %} {% endblock %} diff --git a/h/views/admin/search.py b/h/views/admin/search.py index 4889d56c67e..197623eecd8 100644 --- a/h/views/admin/search.py +++ b/h/views/admin/search.py @@ -3,6 +3,7 @@ from pyramid.view import view_config, view_defaults from h import models, tasks +from h.db.types import URLSafeUUID from h.security import Permission @@ -96,6 +97,41 @@ def reindex_group(self): f"Began reindexing annotations in group {groupid} ({group.name})" ) + @view_config( + request_method="POST", + request_param="reindex_ids", + require_csrf=True, + renderer="h:templates/admin/search.html.jinja2", + ) + def queue_annotations_by_id(self): + annotation_ids = self._annotation_ids_from_text_area( + self.request.params["annotation_ids"] + ) + force = bool(self.request.params.get("reindex_ids_force")) + + tasks.job_queue.add_annotations_by_ids.delay( + self.request.params["name"], annotation_ids, tag="reindex_ids", force=force + ) + return self._notify_reindexing_started("Began reindexing annotations by ID.") + + def _annotation_ids_from_text_area(self, textarea: str) -> list[str]: + ids = [ + annotation_id.strip() + for annotation_id in textarea.split("\n") + if annotation_id.strip() + ] + annotation_ids = [] + for annotation_id in ids: + # If the ID looks like an hex UUID, convert it to URL-safe + if len(annotation_id) == 36: + annotation_ids.append(URLSafeUUID.hex_to_url_safe(annotation_id)) + continue + + # Otherwise assume it's already URL-safe + annotation_ids.append(annotation_id) + + return annotation_ids + def _notify_reindexing_started(self, message): self.request.session.flash(message, "success") return HTTPFound(self.request.route_url("admin.search")) diff --git a/tests/unit/h/services/job_queue_test.py b/tests/unit/h/services/job_queue_test.py index 035ca4b8349..9a7f42f34e7 100644 --- a/tests/unit/h/services/job_queue_test.py +++ b/tests/unit/h/services/job_queue_test.py @@ -121,6 +121,27 @@ def test_add_by_id(self, svc, add_where): where = add_where.call_args[0][1] assert where[0].compare(Annotation.id == sentinel.annotation_id) + def test_add_annotations_by_ids(self, svc, add_where): + svc.add_by_ids( + sentinel.name, + [sentinel.id_1, sentinel.id_2], + sentinel.tag, + schedule_in=sentinel.schedule_in, + force=sentinel.force, + ) + + add_where.assert_called_once_with( + sentinel.name, + [Any.instance_of(BinaryExpression)], + sentinel.tag, + Priority.BY_IDS, + sentinel.force, + sentinel.schedule_in, + ) + + where = add_where.call_args[0][1] + assert where[0].compare(Annotation.id.in_([sentinel.id_1, sentinel.id_2])) + def test_add_annotations_between_times(self, svc, add_where): svc.add_between_times( sentinel.name, diff --git a/tests/unit/h/tasks/job_queue_test.py b/tests/unit/h/tasks/job_queue_test.py index b3227102ac9..526af500370 100644 --- a/tests/unit/h/tasks/job_queue_test.py +++ b/tests/unit/h/tasks/job_queue_test.py @@ -55,6 +55,25 @@ def test_it(self, queue_service): ) +class TestAddAnnotationsByIDs: + def test_it(self, queue_service): + job_queue.add_annotations_by_ids( + sentinel.name, + sentinel.annotation_ids, + sentinel.tag, + force=sentinel.force, + schedule_in=sentinel.schedule_in, + ) + + queue_service.add_by_ids.assert_called_once_with( + sentinel.name, + sentinel.annotation_ids, + sentinel.tag, + force=sentinel.force, + schedule_in=sentinel.schedule_in, + ) + + @pytest.fixture(autouse=True) def celery(patch, pyramid_request): cel = patch("h.tasks.job_queue.celery") diff --git a/tests/unit/h/views/admin/search_test.py b/tests/unit/h/views/admin/search_test.py index 08f6ab9be44..b1c50ae334b 100644 --- a/tests/unit/h/views/admin/search_test.py +++ b/tests/unit/h/views/admin/search_test.py @@ -94,6 +94,28 @@ def test_reindex_group_errors_if_group_not_found( with pytest.raises(NotFoundError, match="Group def456 not found"): views.reindex_group() + def test_queue_annotaions_by_id(self, views, tasks, pyramid_request): + pyramid_request.params = { + "annotation_ids": """ + cdff42be-2fc0-11ef-ae06-37653ab647c1 + zf9Cvi_AEe-uBjdlOrZHwQ + + """, + "name": "jobname", + } + + views.queue_annotations_by_id() + + tasks.job_queue.add_annotations_by_ids.delay.assert_called_once_with( + "jobname", + ["zf9Cvi_AEe-uBjdlOrZHwQ", "zf9Cvi_AEe-uBjdlOrZHwQ"], + tag="reindex_ids", + force=False, + ) + assert pyramid_request.session.peek_flash("success") == [ + "Began reindexing annotations by ID." + ] + @pytest.fixture def views(self, pyramid_request, queue_service): # pylint:disable=unused-argument return SearchAdminViews(pyramid_request)