Updates from develop branch (kartoza#395)

* Fix menu (kartoza#357)

* Add custom render dashboard (kartoza#359)

* Add int and float types in csv upload (kartoza#360)

* Fix bugs in layer selector order (kartoza#363)

* Fix unsaved selected layer not displayed (kartoza#364)

* Add api to get non validated records for sidepanel validation later (kartoza#365)

* Api to get non validated records

* Add tests for unvalidated records api

* Generate permission by taxon class (kartoza#366)

* Get allowed collections based on permission (kartoza#368)

* Fix selector bugs (kartoza#370)

* Update validate records page (kartoza#371)

* Fix send email to validator (kartoza#373)

* Fix legend is covered (kartoza#375)

* Fix biodiversity legend bug (kartoza#376)

* Spatial filter (kartoza#374)

* Add spatial filter panel button

* User boundary model

* Add upload functionality in modal

* Backend for upload user boundary shapefile

* Show boundary list and add click event

* Search by user boundary

* Add boundary name input

* Select multiple boundary

* Add clear and apply filter

* Update search

* Add search mode flag

* Make marker still clickable

* Remove console log

* Update production requirements (kartoza#377)

* Fix spatial search for location site (kartoza#381)

* Fix search user_boundaries

* Fix reset search (kartoza#383)

* Add coming soon menu (kartoza#382)

* Fix spatial search if site outside boundary (kartoza#384)

* Additional filters (kartoza#385)

* Optional filters

* Fix tests

* Add new fields to collection models (kartoza#386)

* Add new fields to collection models

* Add migration file

* Add site code to location site

* Update csv upload to accomodate new fields

* Add missing migration

* Fix tests

* Fix flake8

* Fix filter not working (kartoza#387)

* Add reference category filter (kartoza#388)

* Add api to get the reference category

* Show reference category lists in filter panel

* Filter function with reference category

* Fix travis

* Fix missing variable (kartoza#389)

* Fix csv uploads (kartoza#390)

* Update location site model (kartoza#391)

* Validation ui (kartoza#392)

* Show validation list in front end

* Show point on map

* Show and hide detail

* Add validate function

* Return pagination information

* Update layout

* Fix layout

* Clear point in fetch collection

* Fix travis

* Fix side panel behavior

* Only show validate button to user who has permission

* Add compiled js

* Fix get key function (kartoza#393)

* Set search panel title in settings (kartoza#394)

bims/admin.py

@@ -11,6 +11,7 @@
 from django.contrib import admin as django_admin
 from django.core.mail import send_mail
 
+from django.contrib.auth.models import Permission
 from django.contrib.flatpages.admin import FlatPageAdmin
 from django.contrib.flatpages.models import FlatPage
 from django.db import models
@@ -40,6 +41,7 @@
     ShapefileUploadSession,
     Shapefile,
     NonBiodiversityLayer,
+    UserBoundary,
 )
 
 from bims.conf import TRACK_PAGEVIEWS
@@ -191,6 +193,11 @@ class ClusterAdmin(admin.ModelAdmin):
     list_filter = ('boundary', 'module')
 
 
+class PermissionAdmin(admin.ModelAdmin):
+    list_display = ('name', 'codename')
+    list_filter = ('name', 'codename')
+
+
 class CarouselHeaderAdmin(OrderedModelAdmin):
     list_display = ('order', 'description', 'banner', 'move_up_down_links')
 
@@ -238,6 +245,13 @@ class ShapefileAdmin(admin.ModelAdmin):
     )
 
 
+class UserBoundaryAdmin(admin.ModelAdmin):
+    list_display = (
+        'name',
+        'user'
+    )
+
+
 class LinkAdmin(admin.ModelAdmin):
     list_display = ('name', 'category')
     list_filter = ('category',)
@@ -377,5 +391,9 @@ class PageviewAdmin(admin.ModelAdmin):
 admin.site.register(FlatPage, FlatPageCustomAdmin)
 
 admin.site.register(Visitor, VisitorAdmin)
+admin.site.register(Permission, PermissionAdmin)
+
+admin.site.register(UserBoundary, UserBoundaryAdmin)
+
 if TRACK_PAGEVIEWS:
     admin.site.register(Pageview, PageviewAdmin)

bims/api_views/collection.py

@@ -26,6 +26,7 @@
     update_min_bbox,
     geo_serializer
 )
+from bims.models.user_boundary import UserBoundary
 
 
 class GetCollectionAbstract(APIView):
@@ -55,10 +56,30 @@ def apply_filter(query_value, filters, ignore_bbox=False):
         """
 
         fuzzy_search = False
+        user_boundaries = None
+        filter_mode = False
 
         sqs = SearchQuerySet()
         settings.ELASTIC_MIN_SCORE = 0
 
+        # All filters
+        taxon = filters.get('taxon', None)
+        bbox = filters.get('bbox', None)
+        query_collector = filters.get('collector', None)
+        boundary = filters.get('boundary', None)
+        user_boundary = filters.get('userBoundary', None)
+        query_category = filters.get('category', None)
+        reference_category = filters.get('referenceCategory', None)
+        year_from = filters.get('yearFrom', None)
+        year_to = filters.get('yearTo', None)
+        months = filters.get('months', None)
+
+        if taxon or query_collector or \
+                boundary or user_boundary or \
+                query_category or reference_category or \
+                year_from or year_to or months:
+            filter_mode = True
+
         if query_value:
             clean_query = sqs.query.clean(query_value)
             results = SearchQuerySet().filter(
@@ -82,18 +103,20 @@ def apply_filter(query_value, filters, ignore_bbox=False):
                 ).models(BiologicalCollectionRecord)
                 settings.ELASTIC_MIN_SCORE = 0
         else:
-            results = SearchQuerySet().all().models(BiologicalCollectionRecord)
-            results = results.filter(validated=True)
+            if filter_mode:
+                results = SearchQuerySet().all().models(
+                        BiologicalCollectionRecord)
+                results = results.filter(validated=True)
+            else:
+                results = []
 
-        taxon = filters.get('taxon', None)
         if taxon:
             results = sqs.filter(
                 taxon_gbif=taxon
             ).models(BiologicalCollectionRecord)
 
         # get by bbox
         if not ignore_bbox:
-            bbox = filters.get('bbox', None)
             if bbox:
                 bbox_array = bbox.split(',')
                 downtown_bottom_left = Point(
@@ -111,47 +134,61 @@ def apply_filter(query_value, filters, ignore_bbox=False):
 
         # additional filters
         # query by collectors
-        query_collector = filters.get('collector')
         if query_collector:
             qs_collector = SQ()
             qs = json.loads(query_collector)
             for query in qs:
                 qs_collector.add(SQ(collector=query), SQ.OR)
             results = results.filter(qs_collector)
 
-        boundary = filters.get('boundary')
         if boundary:
             qs_collector = SQ()
             qs = json.loads(boundary)
             for query in qs:
                 qs_collector.add(SQ(boundary=query), SQ.OR)
             results = results.filter(qs_collector)
 
+        if user_boundary:
+            qs = json.loads(user_boundary)
+            user_boundaries = UserBoundary.objects.filter(
+                    pk__in=qs
+            )
+            for user_boundary in user_boundaries:
+                for geom in user_boundary.geometry:
+                    results = results.polygon(
+                            'location_center',
+                            geom
+                    )
+
         # query by category
-        query_category = filters.get('category')
         if query_category:
             qs_category = SQ()
             qs = json.loads(query_category)
             for query in qs:
                 qs_category.add(SQ(category=query), SQ.OR)
             results = results.filter(qs_category)
 
+        # query by reference category
+        if reference_category:
+            qs_reference_category = SQ()
+            qs = json.loads(reference_category)
+            for query in qs:
+                qs_reference_category.add(SQ(reference_category=query), SQ.OR)
+            results = results.filter(qs_reference_category)
+
         # query by year from
-        year_from = filters.get('yearFrom')
         if year_from:
             clean_query_year_from = sqs.query.clean(year_from)
             results = results.filter(
                 collection_date_year__gte=clean_query_year_from)
 
         # query by year to
-        year_to = filters.get('yearTo')
         if year_to:
             clean_query_year_to = sqs.query.clean(year_to)
             results = results.filter(
                 collection_date_year__lte=clean_query_year_to)
 
         # query by months
-        months = filters.get('months')
         if months:
             qs = months.split(',')
             qs_month = SQ()
@@ -170,24 +207,48 @@ def apply_filter(query_value, filters, ignore_bbox=False):
                     site_name__contains=query_value
             ).models(LocationSite)
 
+        location_site_results = location_site_search
+        location_site_user_boundary = None
+
         if boundary:
             qs_collector = SQ()
             qs = json.loads(boundary)
             for query in qs:
                 qs_collector.add(SQ(boundary=query), SQ.OR)
-            if isinstance(location_site_search, SearchQuerySet):
-                location_site_search = location_site_search.filter(
+            if isinstance(location_site_results, SearchQuerySet):
+                location_site_results = location_site_results.filter(
                         qs_collector)
 
-        if len(location_site_search) > 0:
+        if user_boundaries and isinstance(location_site_search,
+                                          SearchQuerySet):
+            location_site_user_boundary = location_site_search
+            for user_boundary in user_boundaries:
+                for geom in user_boundary.geometry:
+                    location_site_user_boundary = \
+                        location_site_user_boundary.polygon(
+                            'location_site_point',
+                            geom)
+
+        site_results = []
+
+        if user_boundaries:
+            if boundary:
+                site_results = \
+                    location_site_results | location_site_user_boundary
+            elif location_site_user_boundary:
+                site_results = location_site_user_boundary
+        else:
+            site_results = location_site_results
+
+        if len(site_results) > 0 or isinstance(
+                location_site_user_boundary, SearchQuerySet):
             # If there are fuzzy results from collection search but we
             # got non fuzzy results from location site, then remove
             # all the fuzzy results from collection
             if fuzzy_search and \
                     len(collection_results) > 0:
                 collection_results = []
             fuzzy_search = False
-        site_results = location_site_search
 
         return collection_results, site_results, fuzzy_search
 

bims/api_views/non_validated_record.py

@@ -0,0 +1,58 @@
+# coding=utf-8
+from django.http.response import HttpResponse
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from rest_framework.views import APIView, Response
+from rest_framework import status
+from bims.models.biological_collection_record import BiologicalCollectionRecord
+from bims.serializers.bio_collection_serializer import (
+    BioCollectionSerializer,
+)
+from bims.permissions.api_permission import IsValidator, AllowedTaxon
+
+
+class GetNonValidatedRecords(APIView):
+
+    permission_classes = [IsValidator, ]
+    page_limit = 5
+
+    def get(self, request):
+        try:
+            allowed_taxon = AllowedTaxon()
+            taxon_list = allowed_taxon.get(request.user)
+            records = BiologicalCollectionRecord.objects.filter(
+                    validated=False,
+                    ready_for_validation=True,
+                    taxon_gbif_id__in=taxon_list
+            )
+
+            paginator = Paginator(records, self.page_limit)
+            page = request.GET.get('page')
+
+            try:
+                records = paginator.page(page)
+                current_page = int(page)
+            except PageNotAnInteger:
+                records = paginator.page(1)
+                current_page = 1
+            except EmptyPage:
+                records = paginator.page(paginator.num_pages)
+                current_page = paginator.num_pages
+
+            serializer = BioCollectionSerializer(
+                    records,
+                    many=True
+            )
+            response_data = {
+                'data': serializer.data,
+                'pagination': {
+                    'current_page': current_page,
+                    'max_page': paginator.num_pages,
+                    'page_limit': self.page_limit
+                }
+            }
+            return Response(response_data)
+        except BiologicalCollectionRecord.DoesNotExist:
+            return HttpResponse(
+                'Object Does Not Exist',
+                status=status.HTTP_400_BAD_REQUEST
+            )

bims/api_views/reference_category.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+from django.db.models import Q
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from bims.models.biological_collection_record import BiologicalCollectionRecord
+
+
+class ReferenceCategoryList(APIView):
+    """Return list of reference category"""
+    def get(self, request, *args):
+        reference_category = \
+            BiologicalCollectionRecord.objects.filter(
+                    ~Q(reference_category='') & Q(validated=True)).\
+            values_list(
+                    'reference_category', flat=True).\
+            distinct()
+        results = []
+        for reference in reference_category:
+            results.append(
+                {
+                    'category': reference
+                }
+            )
+        return Response(results)

bims/api_views/search.py

@@ -45,8 +45,8 @@ def get(self, request):
 
         if len(collection_results) > 0:
             bio_ids = collection_results.values_list('model_pk', flat=True)
-            taxon_ids = list(set(collection_results.values_list(
-                    'taxon_gbif', flat=True)))
+            taxon_ids = collection_results.values_list(
+                    'taxon_gbif', flat=True)
             taxons = Taxon.objects.filter(
                     id__in=taxon_ids).annotate(
                     num_occurrences=Count(

bims/api_views/send_notification_to_validator.py

@@ -31,6 +31,19 @@ def get(self, request):
         if pk is not None:
             try:
                 bio_record = BiologicalCollectionRecord.objects.get(pk=pk)
+
+                taxon_classname = bio_record.taxon_gbif_id.taxon_class
+                class_permission = Permission.objects.filter(
+                        content_type__app_label='bims',
+                        codename='can_validate_%s' % taxon_classname.lower()
+                )
+                class_validators = Profile.objects.filter(
+                        Q(user_permissions=class_permission) |
+                        Q(groups__permissions=class_permission)
+                )
+                for validator in class_validators:
+                    validator_emails.append(validator.email)
+
                 bio_record.ready_for_validation = True
                 bio_record.save()
                 send_mail(

bims/api_views/user_boundary.py

@@ -0,0 +1,16 @@
+# coding=utf-8
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from bims.models.user_boundary import UserBoundary
+from bims.serializers.boundary_serializer import UserBondarySerializer
+
+
+class UserBoundaryList(APIView):
+    """API for listing boundary."""
+
+    def get(self, request, *args):
+        boundaries = UserBoundary.objects.filter(
+                user=request.user
+        )
+        serializer = UserBondarySerializer(boundaries, many=True)
+        return Response(serializer.data)

bims/indexes/biological_collection_record.py

@@ -102,6 +102,11 @@ class BiologicalCollectionIndex(indexes.SearchIndex, indexes.Indexable):
         indexed=True
     )
 
+    reference_category = indexes.CharField(
+        model_attr='reference_category',
+        indexed=True
+    )
+
     boundary = indexes.IntegerField()
 
     def prepare_taxon_gbif(self, obj):

bims/management/commands/generate_permissions.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+from bims.permissions.generate_permission import generate_permission
+from bims.models.taxon import Taxon
+
+
+class Command(BaseCommand):
+    """Generate permissions for all taxon class
+    """
+
+    def handle(self, *args, **options):
+        taxa = Taxon.objects.all().values('taxon_class').distinct()
+        for taxon in taxa:
+            generate_permission(taxon['taxon_class'])