From 88ec94b6ec8427931414bebfb302c1f038899320 Mon Sep 17 00:00:00 2001 From: Hakan Date: Tue, 20 Aug 2024 18:12:51 +0300 Subject: [PATCH 1/5] create extended version of special audit --- .../api/endpoints/datamart/__init__.py | 1 + .../api/endpoints/datamart/audit_special.py | 58 +++++++++++++++++++ src/etools_datamart/api/urls.py | 3 + ...1_auditspecial_amount_refunded_and_more.py | 49 ++++++++++++++++ .../apps/mart/data/models/audit_special.py | 28 +++++++++ 5 files changed, 139 insertions(+) create mode 100644 src/etools_datamart/apps/mart/data/migrations/0031_auditspecial_amount_refunded_and_more.py diff --git a/src/etools_datamart/api/endpoints/datamart/__init__.py b/src/etools_datamart/api/endpoints/datamart/__init__.py index 4c931888..f596fbb8 100644 --- a/src/etools_datamart/api/endpoints/datamart/__init__.py +++ b/src/etools_datamart/api/endpoints/datamart/__init__.py @@ -7,6 +7,7 @@ from .audit_micro_assessment import MicroAssessmentViewSet from .audit_result import AuditResultViewSet from .audit_special import AuditSpecialViewSet +from .audit_special import AuditExtendedSpecialViewSet from .audit_spotcheck import SpotCheckFindingViewSet from .famindicator import FAMIndicatorViewSet from .fm_questions import FMOntrackViewSet, FMOptionsViewSet, FMQuestionViewSet diff --git a/src/etools_datamart/api/endpoints/datamart/audit_special.py b/src/etools_datamart/api/endpoints/datamart/audit_special.py index 081a5f25..b85f011f 100644 --- a/src/etools_datamart/api/endpoints/datamart/audit_special.py +++ b/src/etools_datamart/api/endpoints/datamart/audit_special.py @@ -15,6 +15,39 @@ class AuditSpecialSerializer(DataMartSerializer): partner_name = serializers.SerializerMethodField() url = serializers.SerializerMethodField() + class Meta(DataMartSerializer.Meta): + model = models.AuditSpecial + exclude = ( + "seen", + "source_id", + "write_off_required", + "justification_provided_and_accepted", + "amount_refunded", + "pending_unsupported_amount", + "audited_expenditure", + "financial_findings", + ) + + def get_url(self, obj): + try: + return "{}/ap/special-audits/{}/overview".format( + config.ETOOLS_ADDRESS, + obj.source_id, + ) + except KeyError: + return "" + + def get_partner_name(self, obj): + try: + return obj.partner["name"] + except KeyError: + return "N/A" + + +class AuditExtendedSpecialSerializer(DataMartSerializer): + partner_name = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + class Meta(DataMartSerializer.Meta): model = models.AuditSpecial exclude = ( @@ -61,3 +94,28 @@ class AuditSpecialViewSet(DataMartViewSet): def get_querystringfilter_form(self, request, filter): return AuditSpecialFilterForm(request.GET, filter.form_prefix) + + +class AuditExtendedSpecialFilterForm(forms.Form): + date_of_field_visit = DateRangePickerField( + label="Date of Field Visit", + required=False, + ) + date_of_final_report = DateRangePickerField( + label="Date of Final Report", + required=False, + ) + + +class AuditExtendedSpecialViewSet(DataMartViewSet): + querystringfilter_form_base_class = AuditExtendedSpecialFilterForm + + serializer_class = AuditExtendedSpecialSerializer + queryset = models.AuditSpecial.objects.all() + filter_fields = ( + "date_of_field_visit", + "date_of_final_report", + ) + + def get_querystringfilter_form(self, request, filter): + return AuditExtendedSpecialFilterForm(request.GET, filter.form_prefix) diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index f2bb4c3b..bac1da40 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -27,6 +27,9 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r"datamart/audit/micro-assessment", endpoints.MicroAssessmentViewSet) router.register(r"datamart/audit/results", endpoints.AuditResultViewSet) router.register(r"datamart/audit/special-audit", endpoints.AuditSpecialViewSet) +router.register( + r"datamart/audit/extended-special-audit", endpoints.AuditExtendedSpecialViewSet, basename="extended-special-audit" +) router.register(r"datamart/audit/spot-check-findings", endpoints.SpotCheckFindingViewSet) router.register(r"datamart/actionpoints", endpoints.ActionPointViewSet) router.register(r"datamart/locations", endpoints.LocationViewSet) diff --git a/src/etools_datamart/apps/mart/data/migrations/0031_auditspecial_amount_refunded_and_more.py b/src/etools_datamart/apps/mart/data/migrations/0031_auditspecial_amount_refunded_and_more.py new file mode 100644 index 00000000..199896f9 --- /dev/null +++ b/src/etools_datamart/apps/mart/data/migrations/0031_auditspecial_amount_refunded_and_more.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.13 on 2024-08-20 15:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("data", "0030_alter_pdindicator_baseline_denominator_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="auditspecial", + name="amount_refunded", + field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), + ), + migrations.AddField( + model_name="auditspecial", + name="audited_expenditure", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=20, null=True, verbose_name="Audited Expenditure $" + ), + ), + migrations.AddField( + model_name="auditspecial", + name="financial_findings", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=20, null=True, verbose_name="Financial Findings $" + ), + ), + migrations.AddField( + model_name="auditspecial", + name="justification_provided_and_accepted", + field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), + ), + migrations.AddField( + model_name="auditspecial", + name="pending_unsupported_amount", + field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), + ), + migrations.AddField( + model_name="auditspecial", + name="write_off_required", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=20, null=True, verbose_name="Impairment" + ), + ), + ] diff --git a/src/etools_datamart/apps/mart/data/models/audit_special.py b/src/etools_datamart/apps/mart/data/models/audit_special.py index e0cd3fe4..7c1eff87 100644 --- a/src/etools_datamart/apps/mart/data/models/audit_special.py +++ b/src/etools_datamart/apps/mart/data/models/audit_special.py @@ -25,8 +25,10 @@ def process_country(self): batch_size = settings.RESULTSET_BATCH_SIZE logger.debug(f"Batch size:{batch_size}") + # TODO: Include engagement in qs qs = AuditSpecialaudit.objects.select_related( "engagement_ptr", + "engagement_ptr__", "engagement_ptr__agreement", "engagement_ptr__agreement__auditor_firm__organization", ) @@ -44,6 +46,14 @@ def process_country(self): self.increment_counter(op) def get_special_procedures_count(self, record, values, field_name): + values["pending_unsupported_amount"] = ( + record._impl.financial_findings + - record.amount_refunded + - record.additional_supporting_documentation_provided + - record.justification_provided_and_accepted + - record.write_off_required + ) + return AuditSpecialauditrecommendation.objects.filter(audit=record._impl).count() @@ -98,6 +108,18 @@ class AuditSpecial(EtoolsDataMartModel): action_points = JSONField(blank=True, null=True, default=dict) action_points_data = JSONField(blank=True, null=True, default=dict) + # Extension after amendment + write_off_required = models.DecimalField("Impairment", max_digits=20, decimal_places=2, blank=True, null=True) + justification_provided_and_accepted = models.DecimalField(max_digits=20, decimal_places=2, blank=True, null=True) + amount_refunded = models.DecimalField(max_digits=20, decimal_places=2, blank=True, null=True) + pending_unsupported_amount = models.DecimalField(max_digits=20, decimal_places=2, blank=True, null=True) + audited_expenditure = models.DecimalField( + verbose_name=_("Audited Expenditure $"), blank=True, null=True, decimal_places=2, max_digits=20 + ) + financial_findings = models.DecimalField( + verbose_name=_("Financial Findings $"), blank=True, null=True, decimal_places=2, max_digits=20 + ) + loader = AuditSpecialLoader() class Meta: @@ -117,4 +139,10 @@ class Options: special_procedures_count="-", action_points="-", action_points_data="i", + # TODO mapping for added fields + financial_findings="_impl.financial_findings", + audited_expenditure="_impl.audited_expenditure", + # amount_refunded="_impl.amount_refunded", + # write_off_required="_impl.write_off_required", + # justification_provided_and_accepted = "_impl.justification_provided_and_accepted", ) From d1fcd802334858bf57bd5af3eaad8f57c784cb89 Mon Sep 17 00:00:00 2001 From: Hakan Date: Tue, 27 Aug 2024 16:29:59 +0300 Subject: [PATCH 2/5] map new fields for audit special --- .../api/endpoints/datamart/__init__.py | 3 +- .../apps/mart/data/models/audit_special.py | 31 +++++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/etools_datamart/api/endpoints/datamart/__init__.py b/src/etools_datamart/api/endpoints/datamart/__init__.py index f596fbb8..97d4f80e 100644 --- a/src/etools_datamart/api/endpoints/datamart/__init__.py +++ b/src/etools_datamart/api/endpoints/datamart/__init__.py @@ -6,8 +6,7 @@ from .audit_financial_finding import AuditFinancialFindingViewSet from .audit_micro_assessment import MicroAssessmentViewSet from .audit_result import AuditResultViewSet -from .audit_special import AuditSpecialViewSet -from .audit_special import AuditExtendedSpecialViewSet +from .audit_special import AuditExtendedSpecialViewSet, AuditSpecialViewSet from .audit_spotcheck import SpotCheckFindingViewSet from .famindicator import FAMIndicatorViewSet from .fm_questions import FMOntrackViewSet, FMOptionsViewSet, FMQuestionViewSet diff --git a/src/etools_datamart/apps/mart/data/models/audit_special.py b/src/etools_datamart/apps/mart/data/models/audit_special.py index 7c1eff87..7c98dadc 100644 --- a/src/etools_datamart/apps/mart/data/models/audit_special.py +++ b/src/etools_datamart/apps/mart/data/models/audit_special.py @@ -2,7 +2,7 @@ from django.contrib.postgres.fields import ArrayField from django.core.paginator import Paginator from django.db import models -from django.db.models import JSONField +from django.db.models import JSONField, Prefetch from django.utils.translation import gettext as _ from celery.utils.log import get_task_logger @@ -13,7 +13,7 @@ from etools_datamart.apps.mart.data.models.audit_engagement import EngagementMixin from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel from etools_datamart.apps.sources.etools.enrichment.consts import AuditEngagementConsts -from etools_datamart.apps.sources.etools.models import AuditSpecialaudit, AuditSpecialauditrecommendation +from etools_datamart.apps.sources.etools.models import AuditAudit, AuditSpecialaudit, AuditSpecialauditrecommendation from .partner import Partner @@ -28,9 +28,15 @@ def process_country(self): # TODO: Include engagement in qs qs = AuditSpecialaudit.objects.select_related( "engagement_ptr", - "engagement_ptr__", "engagement_ptr__agreement", "engagement_ptr__agreement__auditor_firm__organization", + # "engagement_ptr__AuditAudit_engagement_ptr", + ).prefetch_related( + Prefetch( + "engagement_ptr__AuditAudit_engagement_ptr", + queryset=AuditAudit.objects.all(), + to_attr="prefetched_AuditAudits", + ), ) paginator = DatamartPaginator(qs, batch_size) @@ -46,8 +52,13 @@ def process_country(self): self.increment_counter(op) def get_special_procedures_count(self, record, values, field_name): + financial_findings = 0 + audit = AuditAudit.objects.all().filter(engagement_ptr_id=record._impl.engagement_ptr_id) + + # TODO: Retrieve financial findings from matching AuditAudit record + values["pending_unsupported_amount"] = ( - record._impl.financial_findings + financial_findings - record.amount_refunded - record.additional_supporting_documentation_provided - record.justification_provided_and_accepted @@ -139,10 +150,10 @@ class Options: special_procedures_count="-", action_points="-", action_points_data="i", - # TODO mapping for added fields - financial_findings="_impl.financial_findings", - audited_expenditure="_impl.audited_expenditure", - # amount_refunded="_impl.amount_refunded", - # write_off_required="_impl.write_off_required", - # justification_provided_and_accepted = "_impl.justification_provided_and_accepted", + # TODO mapping for added fields that will be fetched from AuditAudit + ##financial_findings="_impl.financial_findings", + ##audited_expenditure="_impl.audited_expenditure", + amount_refunded="amount_refunded", + write_off_required="write_off_required", + justification_provided_and_accepted="justification_provided_and_accepted", ) From 2620044f9561e27e42a3226e1380d1ce54a6b34d Mon Sep 17 00:00:00 2001 From: Hakan Date: Wed, 28 Aug 2024 16:39:07 +0300 Subject: [PATCH 3/5] bump up version and add change details --- CHANGES | 11 +++++++++++ src/etools_datamart/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 552ddb37..7db903f8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,14 @@ +4.8.0 +---- +* New endpoint has been addded to be used for slighly modified version special audit export + The new end point is 'datamart/audit/extended-special-audit' + + Upon deployment of this feature export 395 (https://datamart.unicef.io/admin/unicef_rest_framework/export/395/change/) + needs to be updated for Url value '/api/latest/datamart/audit/extended-special-audit/' + + This will make sure export 395 will have more fields in it while keeping /api/latest/datamart/audit/special-audit/ until + it is dropped. + 4.7.0 ---- * add feature that logs timestamp of access to exports diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 06cc3512..e1fb2e0f 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = "etools-datamart" -VERSION = __version__ = "4.7.0" +VERSION = __version__ = "4.8.0" __author__ = "" From b0641994d907043986751473192e97274fee0797 Mon Sep 17 00:00:00 2001 From: Hakan Date: Wed, 28 Aug 2024 17:30:03 +0300 Subject: [PATCH 4/5] extended AuditSpecial to simplify --- CHANGES | 9 +-- .../api/endpoints/datamart/__init__.py | 2 +- .../api/endpoints/datamart/audit_special.py | 58 ------------------- src/etools_datamart/api/urls.py | 3 - 4 files changed, 2 insertions(+), 70 deletions(-) diff --git a/CHANGES b/CHANGES index 7db903f8..d53100f4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,14 +1,7 @@ 4.8.0 ---- -* New endpoint has been addded to be used for slighly modified version special audit export - The new end point is 'datamart/audit/extended-special-audit' - - Upon deployment of this feature export 395 (https://datamart.unicef.io/admin/unicef_rest_framework/export/395/change/) - needs to be updated for Url value '/api/latest/datamart/audit/extended-special-audit/' + AuditSpecial model has been extended to include the addional fields to allow export to include these changes. - This will make sure export 395 will have more fields in it while keeping /api/latest/datamart/audit/special-audit/ until - it is dropped. - 4.7.0 ---- * add feature that logs timestamp of access to exports diff --git a/src/etools_datamart/api/endpoints/datamart/__init__.py b/src/etools_datamart/api/endpoints/datamart/__init__.py index 97d4f80e..4c931888 100644 --- a/src/etools_datamart/api/endpoints/datamart/__init__.py +++ b/src/etools_datamart/api/endpoints/datamart/__init__.py @@ -6,7 +6,7 @@ from .audit_financial_finding import AuditFinancialFindingViewSet from .audit_micro_assessment import MicroAssessmentViewSet from .audit_result import AuditResultViewSet -from .audit_special import AuditExtendedSpecialViewSet, AuditSpecialViewSet +from .audit_special import AuditSpecialViewSet from .audit_spotcheck import SpotCheckFindingViewSet from .famindicator import FAMIndicatorViewSet from .fm_questions import FMOntrackViewSet, FMOptionsViewSet, FMQuestionViewSet diff --git a/src/etools_datamart/api/endpoints/datamart/audit_special.py b/src/etools_datamart/api/endpoints/datamart/audit_special.py index b85f011f..081a5f25 100644 --- a/src/etools_datamart/api/endpoints/datamart/audit_special.py +++ b/src/etools_datamart/api/endpoints/datamart/audit_special.py @@ -15,39 +15,6 @@ class AuditSpecialSerializer(DataMartSerializer): partner_name = serializers.SerializerMethodField() url = serializers.SerializerMethodField() - class Meta(DataMartSerializer.Meta): - model = models.AuditSpecial - exclude = ( - "seen", - "source_id", - "write_off_required", - "justification_provided_and_accepted", - "amount_refunded", - "pending_unsupported_amount", - "audited_expenditure", - "financial_findings", - ) - - def get_url(self, obj): - try: - return "{}/ap/special-audits/{}/overview".format( - config.ETOOLS_ADDRESS, - obj.source_id, - ) - except KeyError: - return "" - - def get_partner_name(self, obj): - try: - return obj.partner["name"] - except KeyError: - return "N/A" - - -class AuditExtendedSpecialSerializer(DataMartSerializer): - partner_name = serializers.SerializerMethodField() - url = serializers.SerializerMethodField() - class Meta(DataMartSerializer.Meta): model = models.AuditSpecial exclude = ( @@ -94,28 +61,3 @@ class AuditSpecialViewSet(DataMartViewSet): def get_querystringfilter_form(self, request, filter): return AuditSpecialFilterForm(request.GET, filter.form_prefix) - - -class AuditExtendedSpecialFilterForm(forms.Form): - date_of_field_visit = DateRangePickerField( - label="Date of Field Visit", - required=False, - ) - date_of_final_report = DateRangePickerField( - label="Date of Final Report", - required=False, - ) - - -class AuditExtendedSpecialViewSet(DataMartViewSet): - querystringfilter_form_base_class = AuditExtendedSpecialFilterForm - - serializer_class = AuditExtendedSpecialSerializer - queryset = models.AuditSpecial.objects.all() - filter_fields = ( - "date_of_field_visit", - "date_of_final_report", - ) - - def get_querystringfilter_form(self, request, filter): - return AuditExtendedSpecialFilterForm(request.GET, filter.form_prefix) diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index bac1da40..f2bb4c3b 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -27,9 +27,6 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r"datamart/audit/micro-assessment", endpoints.MicroAssessmentViewSet) router.register(r"datamart/audit/results", endpoints.AuditResultViewSet) router.register(r"datamart/audit/special-audit", endpoints.AuditSpecialViewSet) -router.register( - r"datamart/audit/extended-special-audit", endpoints.AuditExtendedSpecialViewSet, basename="extended-special-audit" -) router.register(r"datamart/audit/spot-check-findings", endpoints.SpotCheckFindingViewSet) router.register(r"datamart/actionpoints", endpoints.ActionPointViewSet) router.register(r"datamart/locations", endpoints.LocationViewSet) From 9e2b57692d4b9edb6d269796a4773ed40c344f5b Mon Sep 17 00:00:00 2001 From: Hakan Date: Tue, 3 Sep 2024 16:26:19 +0300 Subject: [PATCH 5/5] version adjustments --- CHANGES | 2 +- src/etools_datamart/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index d53100f4..de9d9ab2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ -4.8.0 +4.7.1 ---- AuditSpecial model has been extended to include the addional fields to allow export to include these changes. diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index e1fb2e0f..236f19de 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = "etools-datamart" -VERSION = __version__ = "4.8.0" +VERSION = __version__ = "4.7.1" __author__ = ""