Wake County

eatsmart/locations/wake/api.py

@@ -0,0 +1,159 @@
+import csv
+import os
+import io
+import logging
+import requests
+import zipfile
+import tempfile
+import datetime
+import time
+import pprint
+
+
+from eatsmart.locations.base import Importer
+from eatsmart.locations.wake import forms
+from inspections.models import Establishment, Inspection, Violation
+
+
+logger = logging.getLogger(__name__)
+
+
+class WakeCounty(object):
+
+    url = "http://www.wakegov.com/data/Documents/WCRestaurantInspections.zip"
+
+    def download_and_unzip_data(self, destination):
+        logger.debug("Requesting {}".format(self.url))
+        response = requests.get(self.url)
+        archive = zipfile.ZipFile(io.BytesIO(response.content))
+        logger.debug("Extracting archive into {}".format(str(destination)))
+        archive.extractall(path=destination)
+
+    def import_lives(self):
+        with tempfile.TemporaryDirectory(prefix='wake') as destination:
+            logger.debug("Created temp directory {}".format(destination))
+            self.download_and_unzip_data(destination)
+            businesses = os.path.join(destination, 'businesses.csv')
+            # if os.path.exists(businesses):
+            #     BusinessImporter().run(path=businesses)
+            inspections = os.path.join(destination, 'inspections.csv')
+            # Inspection.objects.filter(establishment__county='Wake').delete()
+            # if os.path.exists(inspections):
+            #     InspectionImporter().run(path=inspections)
+            violations = os.path.join(destination, 'violations.csv')
+            if os.path.exists(violations):
+                ViolationImporter().run(path=violations)
+
+
+class WakeCSVImporter(Importer):
+    "Special importer to open CSV files using the Windows encoding"
+
+    def run(self, path):
+        logger.debug("Importing {}".format(path))
+        with open(path, 'r', encoding='ISO-8859-1') as csv_file:
+            reader = csv.DictReader(csv_file)
+            self.fetch(reader)
+
+    def fetch(self, data, **kwargs):
+        "Primay import workflow with error handling"
+        objects = []
+        start_time = time.time()
+        for index, api in enumerate(data):
+            row = self.map_fields(api=api, **kwargs)
+            form = self.Form(dict(row))
+            if not form.is_valid():
+                errors = {'model': self.Model._meta.object_name,
+                          'errors': dict(form.errors.items()),
+                          'cleaned_data': form.cleaned_data,
+                          'api': api,
+                          'row': row}
+                logger.error(pprint.pformat(errors, indent=4))
+                continue
+            try:
+                instance = self.get_instance(data=form.cleaned_data, **kwargs)
+            except self.Model.DoesNotExist:
+                # Instance doesn't exist, must be new
+                instance = None
+            if instance:
+                form.instance = instance
+            objects.append(form.save())
+            if index % 20 == 0:
+                elapsed_time = time.time() - start_time
+                values = {'model': self.Model._meta.object_name,
+                          'id': row.get('external_id', 'n/a'),
+                          's': len(objects)/elapsed_time}
+                msg = "{model} ID: {id} ({s:.2f} records/sec)".format(**values)
+                logger.debug(msg)
+                start_time = time.time()
+                objects = []
+
+
+class BusinessImporter(WakeCSVImporter):
+    "Import Wake County, NC restaurants"
+
+    Model = Establishment
+    Form = forms.BusinessForm
+
+    def get_instance(self, data):
+        "Instance exists if we have external_id and it's within Wake County"
+        return self.Model.objects.get(external_id=data['external_id'],
+                                      county=data['county'])
+
+    def map_fields(self, api):
+        "Map CSV field names from Wake's data to our database schema"
+        return {'external_id': api['business_id'],
+                'name': api['name'],
+                'type': 1,  # Restaurant
+                'address': api['address'],
+                'city': api['city'],
+                'county': 'Wake',
+                'state': 'NC',
+                'postal_code': api['postal_code'],
+                'phone_number': api['phone_number'],
+                'lat': api['latitude'],
+                'lon': api['longitude'],
+                'status': 'active'}
+
+
+class InspectionImporter(WakeCSVImporter):
+    "Import Wake inspections"
+
+    Model = Inspection
+    Form = forms.InspectionForm
+
+    def get_instance(self, data):
+        "Inspections with same establishment, date, and type is existing"
+        query = {
+            'date': data['date'],
+            'type': data['type'],
+            'establishment': data['establishment'],
+        }
+        return self.Model.objects.get(**query)
+
+    def map_fields(self, api):
+        "Map CSV field names from Wake's data to our database schema"
+        return {'establishment': api['business_id'],
+                'date': api['date'],
+                'type': api['type'],
+                'score': api['score'],
+                'description': api['description']}
+
+
+class ViolationImporter(WakeCSVImporter):
+    "Import Wake violations"
+
+    Model = Violation
+    Form = forms.ViolationForm
+
+    def get_instance(self, data):
+        "Instance exists if we have external_id for the given inspection"
+        return self.Model.objects.get(date=data['date'],
+                                      code=data['code'],
+                                      inspection=data['inspection'])
+
+    def map_fields(self, api):
+        "Map CSV field names from Wake's data to our database schema"
+        return {'establishment': api['business_id'],
+                'date': api['date'],
+                'code': api['code'],
+                'description': api['description']}

eatsmart/locations/wake/forms.py

@@ -0,0 +1,102 @@
+import logging
+
+from django.contrib.gis import forms
+from django.contrib.gis.geos import Point
+
+from inspections.models import Establishment, Inspection, Violation
+
+
+DATE_FORMATS = ['%Y%m%d']
+INSPECTION_TYPE_MAP = {
+    'initial': 5,  # Permit
+    'routine': 1,  # Routine Inspection
+    'followup': 9,  # Verification
+    'complaint': 31,  # Critical Violation Followup
+}
+LIVES_INSPECTION_TYPES = [(x, x) for x in INSPECTION_TYPE_MAP.keys()]
+
+logger = logging.getLogger(__name__)
+
+
+class BusinessForm(forms.ModelForm):
+    "Validate and clean Wake's bussiness data"
+
+    lat = forms.FloatField(required=False)
+    lon = forms.FloatField(required=False)
+
+    class Meta:
+        model = Establishment
+        exclude = ('location',)
+
+    def clean_city(self):
+        city = self.cleaned_data['city']
+        return city.title()
+
+    def clean(self):
+        lat = self.cleaned_data.get('lat', None)
+        lon = self.cleaned_data.get('lon', None)
+        if lat and lon:
+            self.cleaned_data['location'] = Point(lon, lat)
+        return self.cleaned_data
+
+    def save(self, commit=True):
+        instance = super().save(commit=False)
+        if 'location' in self.cleaned_data:
+            instance.location = self.cleaned_data['location']
+        instance.save()
+        return instance
+
+
+class InspectionForm(forms.ModelForm):
+    "Validate and clean Wake's inspection data"
+
+    establishment = forms.CharField()
+    score = forms.FloatField(required=False)
+    date = forms.DateTimeField(input_formats=DATE_FORMATS)
+    type = forms.ChoiceField(choices=LIVES_INSPECTION_TYPES)
+
+    class Meta:
+        model = Inspection
+
+    def clean_type(self):
+        type_ = self.cleaned_data['type']
+        return INSPECTION_TYPE_MAP[type_]
+
+    def clean_establishment(self):
+        query = {'county': 'Wake',
+                 'external_id': self.cleaned_data['establishment']}
+        try:
+            return Establishment.objects.get(**query)
+        except Establishment.DoesNotExist:
+            raise forms.ValidationError("Establishment doesn't exist")
+
+
+class ViolationForm(forms.ModelForm):
+    "Validate and clean Wake's violation data"
+
+    establishment = forms.CharField()
+    inspection = forms.CharField(required=False)
+    date = forms.DateTimeField(input_formats=DATE_FORMATS)
+
+    class Meta:
+        model = Violation
+
+    def clean(self):
+        cleaned_data = self.cleaned_data
+        query = {'county': 'Wake',
+                 'external_id': cleaned_data['establishment']}
+        try:
+            establishment = Establishment.objects.get(**query)
+        except Establishment.DoesNotExist:
+            raise forms.ValidationError("Establishment doesn't exist")
+        query = {'date': cleaned_data['date'],
+                 'establishment': establishment}
+        try:
+            inspection = Inspection.objects.get(**query)
+        except Inspection.DoesNotExist:
+            raise forms.ValidationError("Inspection doesn't exist")
+        except Inspection.MultipleObjectsReturned:
+            raise forms.ValidationError("Multiple inspections found: {}".format(str(query)))
+        cleaned_data['inspection'] = inspection
+        cleaned_data['establishment'] = establishment
+        return cleaned_data

eatsmart/locations/wake/management/commands/import_wake.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand
+
+from eatsmart.locations.wake import api
+
+
+class Command(BaseCommand):
+    """Import saniation data from Durham County API"""
+
+    def handle(self, *args, **options):
+        api.WakeCounty().import_lives()

eatsmart/settings/base.py

@@ -151,6 +151,7 @@
     'inspections',
     'users',
     'eatsmart.locations.durham',
+    'eatsmart.locations.wake',
 )
 
 # A sample logging configuration. The only tangible logging

inspections/admin.py

@@ -5,9 +5,8 @@
 
 class EstablishmentAdmin(LeafletGeoAdmin):
     search_fields = ('name', 'address')
-    list_display = ('id', 'name', 'type',
-                    'county', 'state_id', 'point', 'update_date')
-    list_filter = ('county', 'postal_code')
+    list_display = ('id', 'name', 'type', 'county', 'point', 'update_date')
+    list_filter = ('county', 'update_date')
     ordering = ('-update_date',)
 
     def point(self, obj):
@@ -16,23 +15,53 @@ def point(self, obj):
         return None
 
 
+class InspectionCountyFilter(admin.SimpleListFilter):
+    title = 'County'
+    parameter_name = 'county'
+
+    def lookups(self, request, model_admin):
+        counties = Establishment.objects.values_list('county', flat=True)
+        return [(name, name) for name in counties.distinct()]
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(establishment__county=self.value())
+        else:
+            return queryset
+
+
 class InspectionAdmin(admin.ModelAdmin):
     search_fields = ('id', 'establishment__external_id', 'external_id',
                      'establishment__name')
-    list_display = ('id', 'external_id', 'establishment', 'type',
-                    'date', 'update_date')
-    list_filter = ('update_date', 'type')
+    list_display = ('id', 'establishment', 'type',
+                    'date', 'external_id', 'update_date')
+    list_filter = (InspectionCountyFilter, 'type', 'update_date')
     ordering = ('-date',)
     raw_id_fields = ('establishment',)
     date_hierarchy = 'date'
 
 
+class ViolationCountyFilter(admin.SimpleListFilter):
+    title = 'County'
+    parameter_name = 'county'
+
+    def lookups(self, request, model_admin):
+        counties = Establishment.objects.values_list('county', flat=True)
+        return [(name, name) for name in counties.distinct()]
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(inspection__establishment__county=self.value())
+        else:
+            return queryset
+
+
 class ViolationAdmin(admin.ModelAdmin):
     search_fields = ('id', 'external_id', 'code', 'description',
                      'establishment__name')
-    list_display = ('id', 'external_id', 'establishment', 'code',
-                    'date', 'comments')
-    list_filter = ('code',)
+    list_display = ('id', 'establishment', 'code',
+                    'date', 'comments', 'external_id')
+    list_filter = (ViolationCountyFilter, 'date')
     raw_id_fields = ('establishment', 'inspection')
     ordering = ('-date',)
     date_hierarchy = 'date'