Merge pull request #26 from experius/feature/PWAI-460

Feature/pwai 460

seosnap/migrations/0008_page_tags.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.12 on 2022-02-08 12:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('seosnap', '0007_page_x_magento_tags'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Tag',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

seosnap/models/__init__.py

@@ -1,3 +1,4 @@
+from .tag import *
 from .website import *
 from .extract_field import *
 from .page import *

seosnap/models/page.py

@@ -2,11 +2,11 @@
 from django_mysql.models import Model
 
 from seosnap.utils.JSONField import JSONField
-from seosnap.models import Website
-
+from seosnap.models import Website, Tag
 
 class Page(Model):
     website = models.ForeignKey(Website, related_name='pages', on_delete=models.CASCADE)
+    tags = models.ManyToManyField(Tag)
 
     address = models.CharField(max_length=255)
     content_type = models.CharField(max_length=255, null=True, default=None)

seosnap/models/tag.py

@@ -0,0 +1,9 @@
+from django.db import models
+from django_mysql.models import Model
+
+
+class Tag(Model):
+    name = models.CharField(max_length=255, unique=True)
+
+    def __str__(self):
+        return f'tag - {self.name}'

seosnap/urls.py

@@ -30,6 +30,11 @@
 website_cache_redo_website = views.RedoPageCache.as_view({'post': 'cache_redo_website'})
 website_cache_redo_pages = views.RedoPageCache.as_view({'post': 'cache_redo_addresses'})
 
+# URLs for multiple sites
+pages = views.Pages.as_view({'get': 'get_pages'})
+queues = views.Queues.as_view({'get': 'get_queues'})
+cache_redo_tags = views.Pages.as_view({'post': 'cache_redo_tag'})
+
 urlpatterns = [
     path('websites', website_list, name='websites-list'),
     path('websites/<int:pk>', website_detail, name='websites-retrieve'),
@@ -60,4 +65,10 @@
     path('websites/<int:website_id>/cache/redo/tags', website_cache_redo_tags, name='websites-cache-redo-tags'),
     path('websites/<int:website_id>/cache/redo/website', website_cache_redo_website, name='websites-cache-redo-website'),
     path('websites/<int:website_id>/pages/redo', website_cache_redo_pages, name='websites-cache-redo-addresses'),
+
+    # TODO Multiple get
+    path('pages', pages, name='pages-list'),
+    path('queues', queues, name='queues-list'),
+    # path('cache/redo/tags', website_pages, name='websites-pages-list'),
+
 ]

seosnap/views/page.py

@@ -8,7 +8,7 @@
 from datetime import datetime, timedelta, timezone
 from django.db.models import Q
 
-from seosnap.models import Page, Website, QueueItem
+from seosnap.models import Page, Website, QueueItem, Tag
 from seosnap.models.sitemap import Sitemap
 from seosnap.serializers import PageSerializer
 from django.core import serializers
@@ -179,18 +179,36 @@ def update_pages(self, request, version, website_id=None):
 
         existing = {page.address: page for page in Page.objects.filter(address__in=addresses, website_id=website_id)}
         allowed_fields = set(PageSerializer.Meta.fields) - set(PageSerializer.Meta.read_only_fields)
+
+        tagsString = ""
         for item in items:
             item = {k: item[k] for k in allowed_fields if k in item}
             if item['address'] in existing:
                 page = existing[item['address']]
                 for k, v in item.items(): setattr(page, k, v)
             else:
                 existing[item['address']] = Page(**item)
+            tagsString = tagsString + " " + item['x_magento_tags'].strip()
+
+        tagsArrayFull = set(tagsString.strip().split(" "))
+
+        createTagObjects = []
+        existingTags = Tag.objects.filter(name__in=tagsArrayFull).values_list('name', flat=True)
+        for tagString in tagsArrayFull:
+            if tagString not in existingTags:
+                tag = Tag(name=tagString)
+                createTagObjects.append(tag)
+        Tag.objects.bulk_create(createTagObjects)
 
         with transaction.atomic():
             cache_updated_at = website.cache_updated_at
             for page in existing.values():
                 page.website_id = website_id
+
+                tagsArray = page.x_magento_tags.strip().split(' ')
+                existingTagObjects = Tag.objects.filter(name__in=tagsArray)
+                page.tags.set(existingTagObjects)
+
                 cache_updated_at = page.cached_at
                 page.save()
             website.cache_updated_at = cache_updated_at
@@ -237,34 +255,20 @@ def cache_redo_tag(self, request, version, website_id=None):
 
         query = Q()
         for tag in tags.split(' '):
-            query |= Q(x_magento_tags__contains=tag)
-
-        queryset = website.pages.all().filter(query)
+            query |= Q(tags__name=tag)
 
-        print(len(queryset))
+        pages = website.pages.filter(query)
 
-        # urlList = []
         createQueueObjects = []
-        # for p in queryset:
-        #     urlList.append(p.address)
-        #
-        # print(len(urlList))
-        itemsFound = QueueItem.objects.filter(page__in=queryset).filter(status="unscheduled").values_list('page_id', flat=True)
-        print(itemsFound)
-
-        for p in queryset:
+
+        itemsFound = QueueItem.objects.filter(page__in=pages).filter(status="unscheduled").values_list('page_id', flat=True)
+        for p in pages:
             if p.id not in itemsFound:
                 queue_item: QueueItem = QueueItem(page=p, website=website, priority=request.data['priority'])
                 createQueueObjects.append(queue_item)
 
         QueueItem.objects.bulk_create(createQueueObjects)
 
-        # Length of queryset
-
-        # All pages
-        print('xx')
-        print(request)
-
         return HttpResponse(status=200)
 
     @decorators.action(detail=True, methods=['post'])
@@ -305,3 +309,52 @@ def cache_redo_addresses(self, request, version, website_id=None):
             return HttpResponse(status=200)
 
         return HttpResponse(status=404)
+
+
+class Pages(viewsets.ViewSet, PageNumberPagination):
+
+    @decorators.action(detail=True, methods=['get'])
+    def get_pages(self, request, version):
+        website_ids = []
+        if request.query_params.getlist('website_ids'):
+            website_ids = request.query_params.getlist('website_ids')
+
+        if request.GET.get('filter'):
+            queryset = list(filter(lambda page: page.address.startswith(request.GET.get('filter')), Page.objects.filter(website_id__in=website_ids).all()))
+        else:
+            queryset = Page.objects.filter(website_id__in=website_ids).all()
+
+        if request.GET.get('limit'):
+            self.page_size = request.GET.get('limit')
+
+        page = self.paginate_queryset(queryset, request)
+        if page is not None:
+            serializer = PageSerializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = PageSerializer(queryset, many=True)
+        return Response(serializer.data)
+
+    @decorators.action(detail=True, methods=['post'])
+    def cache_redo_tag(self, request, version):
+        website_ids = []
+        tags = request.data['tags']
+        if request.query_params.getlist('website_ids'):
+            website_ids = request.query_params.getlist('website_ids')
+
+        query = Q()
+        for tag in tags.split(' '):
+            query |= Q(tags__name=tag)
+
+        pages = Page.objects.filter(website_id__in=website_ids).filter(query)
+        createQueueObjects = []
+        itemsFound = QueueItem.objects.filter(page__in=pages).filter(status="unscheduled").values_list('page_id',
+                                                                                                       flat=True)
+        for p in pages:
+            if p.id not in itemsFound:
+                queue_item: QueueItem = QueueItem(page=p, website=p.website_id, priority=request.data['priority'])
+                createQueueObjects.append(queue_item)
+
+        QueueItem.objects.bulk_create(createQueueObjects)
+
+        return HttpResponse(status=200)

seosnap/views/queue.py

@@ -34,7 +34,7 @@ def queue(self, request, version, website_id=None):
         if not allowed or not website: return Response([])
 
         data = website.queue_items.filter(status='unscheduled') \
-                   .order_by('-priority', '-created_at') \
+                   .order_by('priority', '-created_at') \
                    .all()[:50]
 
         with transaction.atomic():
@@ -169,3 +169,27 @@ def delete_multiple_queue_items(self, request, version, website_id=None):
 
         return HttpResponse(status=200)
 
+
+class Queues(viewsets.ViewSet, PageNumberPagination):
+
+    @decorators.action(detail=True, methods=['get'])
+    def get_queues(self, request, version):
+        website_ids = []
+        if request.query_params.getlist('website_ids'):
+            website_ids = request.query_params.getlist('website_ids')
+
+        if request.GET.get('filter'):
+            queryset = QueueItem.objects.filter(website_id__in=website_ids).filter(status__in=['scheduled', 'unscheduled']).filter(page__address__icontains=request.GET.get('filter')).all()
+        else:
+            queryset = QueueItem.objects.filter(website_id__in=website_ids).filter(status__in=['scheduled', 'unscheduled']).all()
+
+        if request.GET.get('limit'):
+            self.page_size = request.GET.get('limit')
+
+        page = self.paginate_queryset(queryset, request)
+        if page is not None:
+            serializer = QueueSerializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = QueueSerializer(queryset, many=True)
+        return Response(serializer.data)