Audit logs

hawc/apps/animal/admin.py

@@ -1,10 +1,11 @@
 from django.contrib import admin
+from reversion.admin import VersionAdmin
 
 from . import models
 
 
 @admin.register(models.Experiment)
-class ExperimentAdmin(admin.ModelAdmin):
+class ExperimentAdmin(VersionAdmin, admin.ModelAdmin):
     raw_id_fields = ("study", "dtxsid")
     list_display = (
         "id",
@@ -24,7 +25,7 @@ class ExperimentAdmin(admin.ModelAdmin):
 
 
 @admin.register(models.AnimalGroup)
-class AnimalGroupAdmin(admin.ModelAdmin):
+class AnimalGroupAdmin(VersionAdmin, admin.ModelAdmin):
     list_display = (
         "id",
         "experiment",
@@ -46,7 +47,7 @@ class DoseGroupInline(admin.TabularInline):
 
 
 @admin.register(models.DosingRegime)
-class DosingRegimeAdmin(admin.ModelAdmin):
+class DosingRegimeAdmin(VersionAdmin, admin.ModelAdmin):
     list_display = (
         "id",
         "dosed_animals",
@@ -72,7 +73,7 @@ class EndpointGroupInline(admin.TabularInline):
 
 
 @admin.register(models.Endpoint)
-class EndpointAdmin(admin.ModelAdmin):
+class EndpointAdmin(VersionAdmin, admin.ModelAdmin):
     list_display = (
         "id",
         "assessment_id",

hawc/apps/animal/templates/animal/animalgroup_detail.html

@@ -1,18 +1,22 @@
 {% extends 'assessment-rooted.html' %}
 
+{% load hawc %}
+
 {% block content %}
     {% if obj_perms.edit %}
       <div class="dropdown btn-group float-right">
         <a class="btn btn-primary dropdown-toggle" data-toggle="dropdown">Actions</a>
         <div class="dropdown-menu dropdown-menu-right">
             <span class="dropdown-header">Animal Group Editing</span>
             <a class="dropdown-item" href="{% url 'animal:animal_group_update' object.pk %}">Update</a>
+            <a class="dropdown-item" href="{% audit_url object %}">Change logs</a>
             <a class="dropdown-item" href="{% url 'animal:animal_group_delete' object.pk %}">Delete</a>
 
             {% if object.dosing_regime and object == object.dosing_regime.dosed_animals %}
               <div class="dropdown-divider"></div>
               <span class="dropdown-header">Dose Regime Editing</span>
               <a class="dropdown-item" href="{% url 'animal:dosing_regime_update' object.dosing_regime.pk %}">Update</a>
+              <a class="dropdown-item" href="{% audit_url object.dosing_regime %}">Change logs</a>
             {% endif %}
 
             <div class="dropdown-divider"></div>

hawc/apps/animal/templates/animal/endpoint_detail.html

@@ -1,5 +1,7 @@
 {% extends 'assessment-rooted.html' %}
 
+{% load hawc %}
+
 {% block content %}
 
   {% if crud == "Read" %}
@@ -9,6 +11,7 @@
       <div class="dropdown-menu dropdown-menu-right">
         <span class="dropdown-header">Endpoint Editing</span>
         <a class="dropdown-item" href="{{ object.get_update_url }}">Update endpoint</a>
+        <a class="dropdown-item" href="{% audit_url object %}">Change logs</a>
         <a class="dropdown-item" href="{% url 'animal:endpoint_delete' object.pk %}">Delete endpoint</a>
         {% if assessment.enable_bmd and object.bmd_modeling_possible %}
           <div class="dropdown-divider"></div>

hawc/apps/animal/templates/animal/experiment_detail.html

@@ -1,5 +1,7 @@
 {% extends 'assessment-rooted.html' %}
 
+{% load hawc %}
+
 {% block content %}
     <h2 class="d-inline-block">{{object}}</h2>
     {% if obj_perms.edit %}
@@ -8,6 +10,7 @@ <h2 class="d-inline-block">{{object}}</h2>
         <div class="dropdown-menu dropdown-menu-right">
             <span class="dropdown-header">Experiment Editing</span>
             <a class="dropdown-item" href="{% url 'animal:experiment_update' object.pk %}">Update</a>
+            <a class="dropdown-item" href="{% audit_url object %}">Change logs</a>
             <a class="dropdown-item" href="{% url 'animal:experiment_delete' object.pk %}">Delete</a>
             <div class="dropdown-divider"></div>
             <span class="dropdown-header">Animal Group Editing</span>

hawc/apps/assessment/actions/audit.py

@@ -0,0 +1,263 @@
+from enum import Enum
+from typing import Optional
+
+import pandas as pd
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import QuerySet
+from reversion.models import Version
+
+from ...common.helper import FlatExport
+from ...common.serializers import PydanticDrfSerializer
+from ..constants import EpiVersion
+from ..models import Assessment
+
+
+def versions_by_content_type(app_label: str, model: str, qs: Optional[QuerySet] = None) -> QuerySet:
+    if qs is None:
+        qs = Version.objects.all()
+    ct = ContentType.objects.get(app_label=app_label, model=model)
+    return qs.filter(content_type=ct)
+
+
+def versions_by_related_field(
+    related_field: str, related_values: list, qs: Optional[QuerySet] = None
+) -> QuerySet:
+    qs = Version.objects.all() if qs is None else qs
+    ored_values = "|".join([str(id) for id in related_values])
+    data_regex = (
+        rf"[\"\']fields[\"\']\s*:\s*{{[^}}]*?[\"\']{related_field}[\"\']\s*:\s*({ored_values})\s*,"
+    )
+    return qs.filter(serialized_data__iregex=data_regex)
+
+
+class AuditType(str, Enum):
+    ASSESSMENT = "assessment"
+    ANIMAL = "animal"
+    EPI = "epi"
+    ROB = "riskofbias"
+
+
+class AssessmentAuditSerializer(PydanticDrfSerializer):
+    assessment: Assessment
+    type: AuditType
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def get_assessment_queryset(self):
+        # assessments
+        assess_qs = Version.objects.get_for_model(Assessment).filter(object_id=self.assessment.pk)
+
+        # get assessment attachments
+        attach_qs = versions_by_content_type("assessment", "attachment")
+        attach_qs = versions_by_related_field(
+            "content_type", [ContentType.objects.get_for_model(Assessment).id], attach_qs
+        )
+        attach_qs = versions_by_related_field("object_id", [self.assessment.pk], attach_qs)
+
+        # get assessment datasets
+        dataset_qs = versions_by_content_type("assessment", "dataset")
+        dataset_qs = versions_by_related_field("assessment", [self.assessment.pk], dataset_qs)
+
+        # get assessment dataset revisions
+        dataset_revision_qs = versions_by_content_type("assessment", "datasetrevision")
+        dataset_revision_qs = versions_by_related_field(
+            "dataset", set(dataset_qs.values_list("object_id", flat=True)), dataset_revision_qs
+        )
+
+        # get assessment summary tables
+        summary_table_qs = versions_by_content_type("summary", "summarytable")
+        summary_table_qs = versions_by_related_field(
+            "assessment", [self.assessment.pk], summary_table_qs
+        )
+        # get assessment visuals
+        visual_qs = versions_by_content_type("summary", "visual")
+        visual_qs = versions_by_related_field("assessment", [self.assessment.pk], visual_qs)
+        # get assessment data pivots
+        data_pivot_qs = versions_by_content_type("summary", "datapivot")
+        data_pivot_qs = versions_by_related_field("assessment", [self.assessment.pk], data_pivot_qs)
+        # get data pivot uploads
+        data_pivot_upload_qs = versions_by_content_type("summary", "datapivotupload")
+        data_pivot_upload_qs = data_pivot_upload_qs.filter(
+            object_id__in=set(data_pivot_qs.values_list("object_id", flat=True))
+        )
+        # get data pivot queries
+        data_pivot_query_qs = versions_by_content_type("summary", "datapivotquery")
+        data_pivot_query_qs = data_pivot_query_qs.filter(
+            object_id__in=set(data_pivot_qs.values_list("object_id", flat=True))
+        )
+
+        return (
+            assess_qs
+            | attach_qs
+            | dataset_qs
+            | dataset_revision_qs
+            | summary_table_qs
+            | visual_qs
+            | data_pivot_qs
+            | data_pivot_upload_qs
+            | data_pivot_query_qs
+        )
+
+    def get_animal_queryset(self):
+        # get assessment references
+        reference_qs = versions_by_content_type("lit", "reference")
+        reference_qs = versions_by_related_field("assessment", [self.assessment.pk], reference_qs)
+        # get reference studies
+        study_qs = versions_by_content_type("study", "study")
+        study_qs = study_qs.filter(
+            object_id__in=set(reference_qs.values_list("object_id", flat=True))
+        )
+        # get study experiments
+        experiment_qs = versions_by_content_type("animal", "experiment")
+        experiment_qs = versions_by_related_field(
+            "study", set(study_qs.values_list("object_id", flat=True)), experiment_qs
+        )
+        # get experiment animal groups
+        animal_group_qs = versions_by_content_type("animal", "animalgroup")
+        animal_group_qs = versions_by_related_field(
+            "experiment", set(experiment_qs.values_list("object_id", flat=True)), animal_group_qs
+        )
+        # get animal group endpoints
+        endpoint_qs = versions_by_content_type("animal", "endpoint")
+        endpoint_qs = versions_by_related_field(
+            "animal_group", set(animal_group_qs.values_list("object_id", flat=True)), endpoint_qs
+        )
+        # get base endpoints
+        base_endpoint_qs = versions_by_content_type("assessment", "baseendpoint")
+        base_endpoint_qs = base_endpoint_qs.filter(
+            object_id__in=set(endpoint_qs.values_list("object_id", flat=True))
+        )
+
+        return experiment_qs | animal_group_qs | endpoint_qs | base_endpoint_qs
+
+    def get_epiv1_queryset(self):
+        # get assessment references
+        reference_qs = versions_by_content_type("lit", "reference")
+        reference_qs = versions_by_related_field("assessment", [self.assessment.pk], reference_qs)
+        # get reference studies
+        study_qs = versions_by_content_type("study", "study")
+        study_qs = study_qs.filter(
+            object_id__in=set(reference_qs.values_list("object_id", flat=True))
+        )
+        # get study populations
+        study_population_qs = versions_by_content_type("epi", "studypopulation")
+        study_population_qs = versions_by_related_field(
+            "study", set(study_qs.values_list("object_id", flat=True)), study_population_qs
+        )
+        # get study population outcomes
+        outcome_qs = versions_by_content_type("epi", "outcome")
+        outcome_qs = versions_by_related_field(
+            "study_population",
+            set(study_population_qs.values_list("object_id", flat=True)),
+            outcome_qs,
+        )
+        # get study population exposures
+        exposure_qs = versions_by_content_type("epi", "exposure")
+        exposure_qs = versions_by_related_field(
+            "study_population",
+            set(study_population_qs.values_list("object_id", flat=True)),
+            exposure_qs,
+        )
+        # get outcome results
+        result_qs = versions_by_content_type("epi", "result")
+        result_qs = versions_by_related_field(
+            "outcome", set(outcome_qs.values_list("object_id", flat=True)), result_qs
+        )
+
+        return study_population_qs | outcome_qs | exposure_qs | result_qs
+
+    def get_epiv2_queryset(self):
+        # get assessment references
+        reference_qs = versions_by_content_type("lit", "reference")
+        reference_qs = versions_by_related_field("assessment", [self.assessment.pk], reference_qs)
+        # get reference studies
+        study_qs = versions_by_content_type("study", "study")
+        study_qs = study_qs.filter(
+            object_id__in=set(reference_qs.values_list("object_id", flat=True))
+        )
+        # get study designs
+        design_qs = versions_by_content_type("epiv2", "design")
+        design_qs = versions_by_related_field(
+            "study", set(study_qs.values_list("object_id", flat=True)), design_qs
+        )
+        # get design chemicals
+        chemical_qs = versions_by_content_type("epiv2", "chemical")
+        chemical_qs = versions_by_related_field(
+            "study", set(design_qs.values_list("object_id", flat=True)), chemical_qs
+        )
+        # get design exposures
+        exposure_qs = versions_by_content_type("epiv2", "exposure")
+        exposure_qs = versions_by_related_field(
+            "design", set(design_qs.values_list("object_id", flat=True)), exposure_qs
+        )
+        # get design exposure levels
+        exposure_level_qs = versions_by_content_type("epiv2", "exposurelevel")
+        exposure_level_qs = versions_by_related_field(
+            "design", set(design_qs.values_list("object_id", flat=True)), exposure_level_qs
+        )
+        # get design outcomes
+        outcome_qs = versions_by_content_type("epiv2", "outcome")
+        outcome_qs = versions_by_related_field(
+            "design", set(design_qs.values_list("object_id", flat=True)), outcome_qs
+        )
+        # get design adjustment factors
+        adjustment_factor_qs = versions_by_content_type("epiv2", "adjustmentfactor")
+        adjustment_factor_qs = versions_by_related_field(
+            "design", set(design_qs.values_list("object_id", flat=True)), adjustment_factor_qs
+        )
+        # get design data extractions
+        data_extraction_qs = versions_by_content_type("epiv2", "dataextraction")
+        data_extraction_qs = versions_by_related_field(
+            "design", set(design_qs.values_list("object_id", flat=True)), data_extraction_qs
+        )
+
+        return (
+            design_qs
+            | chemical_qs
+            | exposure_qs
+            | exposure_level_qs
+            | outcome_qs
+            | adjustment_factor_qs
+            | data_extraction_qs
+        )
+
+    def get_riskofbias_queryset(self):
+        # get assessment domains
+        domain_qs = versions_by_content_type("riskofbias", "riskofbiasdomain")
+        domain_qs = versions_by_related_field("assessment", [self.assessment.pk], domain_qs)
+        # get domain metrics
+        metric_qs = versions_by_content_type("riskofbias", "riskofbiasmetric")
+        metric_qs = versions_by_related_field(
+            "domain", set(domain_qs.values_list("object_id", flat=True)), metric_qs
+        )
+        # get metric scores
+        score_qs = versions_by_content_type("riskofbias", "riskofbiasscore")
+        score_qs = versions_by_related_field(
+            "metric", set(metric_qs.values_list("object_id", flat=True)), score_qs
+        )
+
+        return domain_qs | metric_qs | score_qs
+
+    def get_queryset(self):
+        audit_type = self.type
+        if audit_type == AuditType.EPI:
+            audit_type = "epiv1" if self.assessment.epi_version == EpiVersion.V1 else "epiv2"
+        qs = getattr(self, f"get_{audit_type}_queryset")()
+        return qs.select_related("content_type", "revision")
+
+    def export(self) -> FlatExport:
+        qs = self.get_queryset()
+        df = pd.DataFrame(
+            qs.values_list(
+                "content_type__app_label",
+                "content_type__model",
+                "object_id",
+                "serialized_data",
+                "revision__user",
+                "revision__date_created",
+            ),
+            columns=["app", "model", "pk", "serialized_data", "user", "date_revised"],
+        )
+        export = FlatExport(df=df, filename=f"{self.assessment}-{self.type}-audit-logs")
+        return export

hawc/apps/assessment/api.py

@@ -13,6 +13,7 @@
 from rest_framework.decorators import action
 from rest_framework.exceptions import APIException, PermissionDenied
 from rest_framework.pagination import PageNumberPagination
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from hawc.services.epa import dsstox
@@ -21,6 +22,7 @@
 from ..common.renderers import PandasRenderers
 from ..common.views import create_object_log
 from . import models, serializers
+from .actions.audit import AssessmentAuditSerializer
 
 
 class DisabledPagination(PageNumberPagination):
@@ -301,7 +303,7 @@ def public(self, request):
         return Response(serializer.data)
 
     @action(detail=True)
-    def endpoints(self, request, pk: int = None):
+    def endpoints(self, request, pk: int):
         """
         Optimized for queryset speed; some counts in get_queryset
         and others in the list here; depends on if a "select distinct" is
@@ -458,6 +460,15 @@ def endpoints(self, request, pk: int = None):
 
         return Response({"name": instance.name, "id": instance.id, "items": items})
 
+    @action(detail=True, url_path=r"logs/(?P<type>[\w]+)", renderer_classes=PandasRenderers)
+    def logs(self, request: Request, pk: int, type: str):
+        instance = self.get_object()
+        if not instance.user_is_team_member_or_higher(self.request.user):
+            raise PermissionDenied()
+        serializer = AssessmentAuditSerializer.from_drf(data=dict(assessment=instance, type=type))
+        export = serializer.export()
+        return Response(export)
+
 
 class DatasetViewset(AssessmentViewset):
     model = models.Dataset