From 5944e76e217c44a135b610ad4f7e0f8971345ba4 Mon Sep 17 00:00:00 2001 From: Jaroslav Sevcik Date: Thu, 15 Jul 2021 11:30:17 +0200 Subject: [PATCH] chore(entity): change logic of update and delete --- test/entities/test_objects_to_yaml.py | 98 +++++- test/entities/test_yaml_to_objects.py | 317 ++++++++++++++---- zoo/api/query.py | 6 +- zoo/api/types.py | 35 +- zoo/base/settings.py | 1 + zoo/entities/builder.py | 97 +++--- .../migrations/0004_auto_20210713_1003.py | 31 +- .../migrations/0005_auto_20210715_0658.py | 24 ++ .../migrations/0006_auto_20210715_0713.py | 34 ++ .../migrations/0007_auto_20210715_0934.py | 19 ++ .../migrations/0008_auto_20210715_1010.py | 23 ++ .../migrations/0009_auto_20210719_0927.py | 18 + zoo/entities/models.py | 26 +- .../yaml_definitions/component_base.yaml | 8 +- .../yaml_definitions/component_library.yaml | 3 + .../yaml_definitions/component_service.yaml | 4 + zoo/factories.py | 12 +- zoo/repos/entities_yaml.py | 31 +- zoo/repos/tasks.py | 14 +- 19 files changed, 606 insertions(+), 195 deletions(-) create mode 100644 zoo/entities/migrations/0005_auto_20210715_0658.py create mode 100644 zoo/entities/migrations/0006_auto_20210715_0713.py create mode 100644 zoo/entities/migrations/0007_auto_20210715_0934.py create mode 100644 zoo/entities/migrations/0008_auto_20210715_1010.py create mode 100644 zoo/entities/migrations/0009_auto_20210719_0927.py diff --git a/test/entities/test_objects_to_yaml.py b/test/entities/test_objects_to_yaml.py index 4b4c6505..3f07eed9 100644 --- a/test/entities/test_objects_to_yaml.py +++ b/test/entities/test_objects_to_yaml.py @@ -8,27 +8,27 @@ pytestmark = pytest.mark.django_db -@pytest.fixture -def generate_base_component(component_base_factory, link_factory, group_factory): +def test_generate_base_component(group_factory, component_base_factory, link_factory): + group = group_factory( + id=1, product_owner="john", project_owner="doe" + ) component = component_base_factory( id=1, name="base", + label="Base", type="database", description="This is my fancy component", kind="component", owner="platform", service=None, library=None, + group=group, source__id=1, source__remote_id=1, source__owner="jasckson", source__name="thiwer", source__url="https://gitlab.com/thiwer/thiwer", ) - component.save() - - group = group_factory(id=1, product_owner="john", project_owner="doe", entity=component) - group.save() link_factory( id=1, @@ -44,13 +44,12 @@ def generate_base_component(component_base_factory, link_factory, group_factory) entity=component, ) - -def test_generate_base_component(generate_base_component): expected = """ - apiVersion: v1alpha1 kind_: component metadata: name: base + label: Base owner: platform group: product_owner: john @@ -64,7 +63,6 @@ def test_generate_base_component(generate_base_component): icon: poop - name: Sentry url: https://sentry.skypicker.com - icon: null spec: type_: database """ @@ -72,3 +70,85 @@ def test_generate_base_component(generate_base_component): content = generate(repository) assert validate(content) assert expected.strip() == content.strip() + + +def test_generate_component_service(component_base_factory, service_factory, group_factory, + link_factory, repository_factory, environment_factory): + repository = repository_factory( + id=22, + remote_id=22, + name="test_proj1", + owner="john_doe1", + url="https://github.com/john_doe1/test_proj1", + provider="github", + ) + + service = service_factory( + owner="platform", + name="my-service", + lifecycle="production", + impact="employees", + repository=repository, + slack_channel="#platform-software", + docs_url="https://docs.com" + ) + environment_factory( + name="production", + service=service, + health_check_url="https://health.com", + dashboard_url="https://dashboard.datadog.com", + service_urls=["https://service.com"] + ) + group = group_factory( + id=1, product_owner="john", project_owner="doe" + ) + component_service = component_base_factory( + id=1, + name="service", + label="Service", + type="service", + description="This is my fancy service", + kind="component", + owner="platform", + service=service, + library=None, + group=group, + source=repository + ) + link_factory( + id=1, + name="Datadog", + url="https://dashboard.datadog.com", + icon="poop", + entity=component_service, + ) + expected = """ +- apiVersion: v1alpha1 + kind_: component + metadata: + name: service + label: Service + owner: platform + group: + product_owner: john + project_owner: doe + maintainers: [] + description: This is my fancy service + tags: [] + links: + - name: Datadog + url: https://dashboard.datadog.com + icon: poop + spec: + type_: service + environments: + - name: production + dashboard_url: https://dashboard.datadog.com + health_check_url: https://health.com + service_urls: + - https://service.com +""" + repository = Repository.objects.get(pk=22) + content = generate(repository) + assert validate(content) + assert expected.strip() == content.strip() diff --git a/test/entities/test_yaml_to_objects.py b/test/entities/test_yaml_to_objects.py index 2ba654dd..efec59eb 100644 --- a/test/entities/test_yaml_to_objects.py +++ b/test/entities/test_yaml_to_objects.py @@ -8,54 +8,6 @@ pytestmark = pytest.mark.django_db -@pytest.fixture -def create_component_and_service(component_base_factory, repository_factory, link_factory, group_factory, service_factory): - repository = repository_factory( - id=1, - remote_id=11, - name="test_proj1", - owner="john_doe1", - url="https://github.co m/john_doe1/test_proj1", - provider="github", - ) - service = service_factory( - id=1, - name="service", - owner="platform", - impact="profit", - pagerduty_service="sales/P019873X9", - slack_channel="https://slackchannel", - lifecycle="fixed", - description="original description", - ) - - component = component_base_factory( - id=1, - name="original", - type="database", - description="original description", - kind="component", - owner="platform", - tags=["python", "service"], - service=service, - library=None, - source=repository - ) - group = group_factory(id=1, product_owner="john", project_owner="doe", entity=component) - link_factory( - id=1, - name="Datadog", - url="https://dashboard.datadog.com", - icon="poop", - entity=component, - ) - link_factory( - id=2, - name="Sentry", - url="https://sentry.skypicker.com", - entity=component, - ) - def test_create_base_component(mocker, repository_factory): repository = repository_factory( id=1, @@ -71,7 +23,8 @@ def test_create_base_component(mocker, repository_factory): - apiVersion: v1alpha1 kind_: component metadata: - name: base + name: base-component + label: Base Component owner: platform group: product_owner: john @@ -94,13 +47,14 @@ def test_create_base_component(mocker, repository_factory): component_entity = Entity.objects.first() assert Entity.objects.all().count() == 1 assert component_entity.kind == "component" - assert component_entity.name == "base" + assert component_entity.name == "base-component" + assert component_entity.label == "Base Component" assert component_entity.owner == "platform" assert component_entity.service is None assert component_entity.library is None assert component_entity.type == "database" assert component_entity.links.all().count() == 2 - assert component_entity.groups.all().count() == 1 + assert component_entity.group is not None def test_create_base_component_and_service(mocker, repository_factory): @@ -118,7 +72,8 @@ def test_create_base_component_and_service(mocker, repository_factory): - apiVersion: v1alpha1 kind_: component metadata: - name: base + name: base-service + label: Base Service owner: platform group: product_owner: john @@ -160,12 +115,13 @@ def test_create_base_component_and_service(mocker, repository_factory): assert Entity.objects.all().count() == 1 assert Service.objects.all().count() == 1 assert component_entity.kind == "component" - assert component_entity.name == "base" + assert component_entity.name == "base-service" + assert component_entity.label == "Base Service" assert component_entity.owner == "platform" assert component_entity.library is None assert component_entity.type == "service" assert component_entity.links.all().count() == 2 - assert component_entity.groups.all().count() == 1 + assert component_entity.group is not None assert component_entity.service is not None assert component_entity.service.environments.all().count() == 2 @@ -186,6 +142,7 @@ def test_create_base_component_and_library(mocker, repository_factory): kind_: component metadata: name: base_lib + label: Base Lib owner: platform group: product_owner: john @@ -216,10 +173,11 @@ def test_create_base_component_and_library(mocker, repository_factory): assert Library.objects.all().count() == 1 assert component_entity.kind == "component" assert component_entity.name == "base_lib" + assert component_entity.label == "Base Lib" assert component_entity.owner == "platform" assert component_entity.type == "library" assert component_entity.links.all().count() == 2 - assert component_entity.groups.all().count() == 1 + assert component_entity.group is not None assert component_entity.service is None assert component_entity.library is not None @@ -230,6 +188,7 @@ def test_create_multiple_components_one_service(mocker, repository_factory): kind_: component metadata: name: base + label: Base owner: platform group: product_owner: john @@ -250,6 +209,7 @@ def test_create_multiple_components_one_service(mocker, repository_factory): kind_: component metadata: name: service + label: Service owner: platform group: product_owner: john @@ -299,12 +259,14 @@ def test_create_multiple_components_one_service(mocker, repository_factory): assert Entity.objects.all().count() == 2 assert Service.objects.all().count() == 1 + def test_create_multiple_components_multiple_services(mocker, repository_factory): multiple_components = """ - apiVersion: v1alpha1 kind_: component metadata: name: base + label: Base owner: platform group: product_owner: john @@ -325,6 +287,7 @@ def test_create_multiple_components_multiple_services(mocker, repository_factory kind_: component metadata: name: service + label: Base owner: platform group: product_owner: john @@ -360,7 +323,8 @@ def test_create_multiple_components_multiple_services(mocker, repository_factory - apiVersion: v1alpha1 kind_: component metadata: - name: service the second + name: service-the-second + label: Service The Second owner: platform software group: product_owner: john the first @@ -411,19 +375,138 @@ def test_create_multiple_components_multiple_services(mocker, repository_factory assert Service.objects.all().count() == 2 -def test_update_service(mocker, create_component_and_service): +def test_update_base_component( + mocker, repository_factory, component_base_factory, link_factory, group_factory +): + group = group_factory( + id=1, product_owner="Old John", project_owner="Old Doe" + ) + group.save() + component = component_base_factory( + id=1, + name="old-component", + label="Old Component", + type="database", + description="Old Component Description", + kind="component", + owner="platform", + group=group, + service=None, + library=None, + source__id=1, + source__remote_id=1, + source__owner="jasckson", + source__name="thiwer", + source__url="https://gitlab.com/thiwer/thiwer", + source__provider="gitlab", + ) + component.save() - service_component = """ + link_factory( + id=1, + name="Datadog", + url="https://dashboard.datadog.com", + icon="poop", + entity=component, + ) + + base_component = """ + - apiVersion: v1alpha1 + kind_: component + metadata: + name: new-component + label: New Component + owner: platform + group: + product_owner: New John + project_owner: New Doe + maintainers: [] + description: New Component Description + tags: [] + links: + - name: Datadog + url: https://dashboard.datadog.com + icon: poop + - name: Sentry + url: https://sentry.skypicker.com + spec: + type_: database + """ + + repository_dict = { + "id": component.source.remote_id, + "provider": component.source.provider, + } + mocker.patch("zoo.repos.tasks.get_entity_file_content", return_value=base_component) + + component_entity = Entity.objects.first() + assert Entity.objects.all().count() == 1 + assert component_entity.kind == "component" + assert component_entity.name == "old-component" + assert component_entity.label == "Old Component" + assert component_entity.owner == "platform" + assert component_entity.type == "database" + assert component_entity.links.all().count() == 1 + assert component_entity.group is not None + assert component_entity.group.product_owner == "Old John" + assert component_entity.group.project_owner == "Old Doe" + assert component_entity.service is None + assert component_entity.library is None + + uut.update_project_from_entity_file(proj=repository_dict) + + component_entity = Entity.objects.first() + assert Entity.objects.all().count() == 1 + assert component_entity.kind == "component" + assert component_entity.name == "new-component" + assert component_entity.label == "New Component" + assert component_entity.owner == "platform" + assert component_entity.type == "database" + assert component_entity.links.all().count() == 2 + assert component_entity.group is not None + assert component_entity.group.product_owner == "New John" + assert component_entity.group.project_owner == "New Doe" + assert component_entity.service is None + assert component_entity.library is None + + +def test_update_service_library( + mocker, component_base_factory, service_factory, repository_factory, library_factory, group_factory +): + repository = repository_factory( + id=22, + remote_id=22, + name="test_proj1", + owner="john_doe1", + url="https://github.com/john_doe1/test_proj1", + provider="github", + ) + group_1 = group_factory( + id=1, product_owner="Old John", project_owner="Old Doe" + ) + group_1.save() + + group_2 = group_factory( + id=2, product_owner="Old John", project_owner="Old Doe" + ) + group_2.save() + service = service_factory(repository=repository) + library = library_factory(repository=repository) + component_base_factory(source=repository, service=service, group=group_1) + component_base_factory(source=repository, library=library, group=group_2) + + component_service_and_library = """ - apiVersion: v1alpha1 kind_: component metadata: - name: service + name: new-service + label: New Service owner: platform group: - product_owner: john - project_owner: doe + product_owner: New John + project_owner: New Doe maintainers: [] - description: This is my fancy component + description: New Description tags: [] links: - name: Datadog @@ -449,4 +532,114 @@ def test_update_service(mocker, create_component_and_service): pagerduty_service: pagerduty_service1234 sentry_project: sentry project 15234 lifecycle: production +--- +- apiVersion: v1alpha1 + kind_: component + metadata: + name: new-lib + label: New Lib + owner: platform + group: + product_owner: john + project_owner: doe + maintainers: [] + description: This is my fancy component + tags: [] + links: + - name: Datadog + url: https://dashboard.datadog.com + icon: poop + - name: Sentry + url: https://sentry.skypicker.com + spec: + type_: library + impact: profit + integrations: + sonarqube_project: sonarqube + lifecycle: production """ + + repository_dict = { + "id": repository.remote_id, + "provider": repository.provider, + } + mocker.patch( + "zoo.repos.tasks.get_entity_file_content", + return_value=component_service_and_library, + ) + + assert Entity.objects.all().count() == 2 + assert Service.objects.all().count() == 1 + assert Library.objects.all().count() == 1 + + uut.update_project_from_entity_file(proj=repository_dict) + + assert Entity.objects.all().count() == 2 + assert Service.objects.all().count() == 1 + assert Library.objects.all().count() == 1 + + service_entity = Entity.objects.filter(service__isnull=False).first() + library_entity = Entity.objects.filter(library__isnull=False).first() + + assert service_entity.kind == "component" + assert service_entity.type == "service" + assert service_entity.name == "new-service" + assert service_entity.label == "New Service" + assert service_entity.owner == "platform" + assert service_entity.links.all().count() == 2 + assert service_entity.service.environments.all().count() == 2 + + assert library_entity.kind == "component" + assert library_entity.type == "library" + assert library_entity.name == "new-lib" + assert library_entity.label == "New Lib" + assert library_entity.owner == "platform" + assert library_entity.links.all().count() == 2 + + +def test_delete_entities( + mocker, component_base_factory, service_factory, repository_factory, library_factory,group_factory +): + repository = repository_factory( + id=22, + remote_id=22, + name="test_proj1", + owner="john_doe1", + url="https://github.com/john_doe1/test_proj1", + provider="github", + ) + group_1 = group_factory( + id=1, product_owner="Old John", project_owner="Old Doe" + ) + group_1.save() + + group_2 = group_factory( + id=2, product_owner="Old John", project_owner="Old Doe" + ) + group_2.save() + + service = service_factory(repository=repository) + library = library_factory(repository=repository) + component_base_factory(source=repository, service=service, group=group_1) + component_base_factory(source=repository, library=library, group=group_2) + + component_service_and_library = "" + + repository_dict = { + "id": repository.remote_id, + "provider": repository.provider, + } + mocker.patch( + "zoo.repos.tasks.get_entity_file_content", + return_value=component_service_and_library, + ) + + assert Entity.objects.all().count() == 2 + assert Service.objects.all().count() == 1 + assert Library.objects.all().count() == 1 + + uut.update_project_from_entity_file(proj=repository_dict) + + assert Entity.objects.all().count() == 0 + assert Service.objects.all().count() == 0 + assert Library.objects.all().count() == 0 diff --git a/zoo/api/query.py b/zoo/api/query.py index fd1a1819..12ae4175 100644 --- a/zoo/api/query.py +++ b/zoo/api/query.py @@ -6,7 +6,7 @@ from ..analytics.models import Dependency, DependencyType, DependencyUsage from ..auditing.models import Issue -from ..entities.models import Link, Entity +from ..entities.models import Entity, Link from ..globalsearch.indexer import IndexType from ..globalsearch.meili_client import meili_client from ..libraries.models import Library @@ -204,9 +204,7 @@ def resolve_all_links(self, info, **kwargs): node = types.Link.from_db(link) edges.append(types.LinkConnection.Edge(node=node, cursor=cursor)) - return types.LinkConnection( - page_info=page_info, edges=edges, total_count=total - ) + return types.LinkConnection(page_info=page_info, edges=edges, total_count=total) def resolve_all_libraries(self, info, **kwargs): paginator = Paginator(**kwargs) diff --git a/zoo/api/types.py b/zoo/api/types.py index 3b948d56..b04cc77e 100644 --- a/zoo/api/types.py +++ b/zoo/api/types.py @@ -111,15 +111,15 @@ class Library(graphene.ObjectType): @classmethod def from_db(cls, library): return cls( - owner= library.owner, - name = library.name, - lifecycle = library.lifecycle, - impact = library.impact, - slack_channel = library.slack_channel, - sonarqube_project = library.sonarqube_project, - repository = library.repository_id, - docs_url = library.docs_url, - library_url = library.library_url, + owner=library.owner, + name=library.name, + lifecycle=library.lifecycle, + impact=library.impact, + slack_channel=library.slack_channel, + sonarqube_project=library.sonarqube_project, + repository=library.repository_id, + docs_url=library.docs_url, + library_url=library.library_url, ) @classmethod @@ -189,6 +189,7 @@ class LinkConnection(relay.Connection): class Meta: node = Link + class Entity(graphene.ObjectType): name = graphene.String() type = graphene.String() @@ -227,31 +228,27 @@ def resolve_service(self, info): def resolve_library(self, info): try: - return Library.from_db(libraries_models.Library.objects.get(id=self.library)) + return Library.from_db( + libraries_models.Library.objects.get(id=self.library) + ) except ObjectDoesNotExist: return None def resolve_all_links(self, info, **kwargs): paginator = Paginator(**kwargs) edges = [] - filtered_links = entities_models.Link.objects.filter( - entity_id=self.id - ) + filtered_links = entities_models.Link.objects.filter(entity_id=self.id) total = filtered_links.count() page_info = paginator.get_page_info(total) for i, issue in enumerate( - filtered_links[ - paginator.slice_from: paginator.slice_to # Ignore PEP8Bear - ] + filtered_links[paginator.slice_from : paginator.slice_to] # Ignore PEP8Bear ): cursor = paginator.get_edge_cursor(i + 1) node = Link.from_db(issue) edges.append(LinkConnection.Edge(node=node, cursor=cursor)) - return LinkConnection( - page_info=page_info, edges=edges, total_count=total - ) + return LinkConnection(page_info=page_info, edges=edges, total_count=total) class Meta: interfaces = (relay.Node,) diff --git a/zoo/base/settings.py b/zoo/base/settings.py index 40a39b84..9b04dbad 100644 --- a/zoo/base/settings.py +++ b/zoo/base/settings.py @@ -272,4 +272,5 @@ MEILI_HOST = env("MEILI_HOST") logs.configure_structlog(DEBUG) +ZOO_ALLOWD_ENTITY_KINDS = ["component"] ZOO_ALLOWED_COMPONENT_TYPES = ["service", "library"] diff --git a/zoo/entities/builder.py b/zoo/entities/builder.py index 6f663208..cc9dc60c 100644 --- a/zoo/entities/builder.py +++ b/zoo/entities/builder.py @@ -3,8 +3,9 @@ from zoo.services.models import Environment, Service +# TODO: refactor to registry pattern(or any better one) class EntityBuilder: - def create_entity(self, data, repository): + def sync_entities(self, data, repository): if data["kind_"] != "component": return NotImplemented if data["spec"]["type_"] == "service": @@ -16,32 +17,30 @@ def create_entity(self, data, repository): @staticmethod def _build_base_component(data, repository): - obj, _ = Entity.objects.update_or_create( + group = Group.objects.create( + product_owner=data["metadata"]["group"]["product_owner"], + project_owner=data["metadata"]["group"]["project_owner"], + maintainers=data["metadata"]["group"]["maintainers"], + ) + + obj = Entity.objects.create( name=data["metadata"]["name"], + label=data["metadata"]["label"], kind=data["kind_"], type=data["spec"]["type_"], source=repository, - defaults={ - "owner": data["metadata"]["owner"], - "description": data["metadata"]["description"], - "tags": data["metadata"]["tags"], - }, - ) - - group, _ = Group.objects.update_or_create( - entity=obj, - defaults={ - "product_owner": data["metadata"]["group"]["product_owner"], - "project_owner": data["metadata"]["group"]["project_owner"], - "maintainers": data["metadata"]["group"]["maintainers"] - } + owner=data["metadata"]["owner"], + description=data["metadata"]["description"], + tags=data["metadata"]["tags"], + group=group ) for link in data["metadata"]["links"]: - l, _ = Link.objects.update_or_create( + Link.objects.create( url=link["url"], entity=obj, - defaults={"name": link["name"], "icon": getattr(link, "icon", None)}, + name=link["name"], + icon=getattr(link, "icon", None), ) return obj @@ -49,38 +48,30 @@ def _build_base_component(data, repository): def _build_service(self, data, repository): base_component = self._build_base_component(data, repository) - service, _ = Service.objects.update_or_create( - name=base_component.name, + service = Service.objects.create( + name=base_component.label, owner=base_component.owner, repository=repository, - defaults={ - "lifecycle": data["spec"]["lifecycle"], - "impact": data["spec"]["impact"], - "sentry_project": data["spec"] - .get("integrations", {}) - .get("sentry_project"), - "sonarqube_project": data["spec"] - .get("integrations", {}) - .get("sonarqube_project"), - "slack_channel": data["spec"] - .get("integrations", {}) - .get("slack_channel"), - "pagerduty_service": data["spec"] - .get("integrations", {}) - .get("pagerduty_service"), - "description": data["metadata"]["description"], - }, + lifecycle=data["spec"]["lifecycle"], + impact=data["spec"]["impact"], + sentry_project=data["spec"].get("integrations", {}).get("sentry_project"), + sonarqube_project=data["spec"] + .get("integrations", {}) + .get("sonarqube_project"), + slack_channel=data["spec"].get("integrations", {}).get("slack_channel"), + pagerduty_service=data["spec"] + .get("integrations", {}) + .get("pagerduty_service"), + description=data["metadata"]["description"], ) for env in data["spec"]["environments"]: - e, _ = Environment.objects.update_or_create( + Environment.objects.create( service=service, name=env["name"], - defaults={ - "dashboard_url": env["dashboard_url"], - "health_check_url": env["health_check_url"], - "service_urls": env["service_urls"], - }, + dashboard_url=env["dashboard_url"], + health_check_url=env["health_check_url"], + service_urls=env["service_urls"], ) base_component.service = service @@ -89,20 +80,16 @@ def _build_service(self, data, repository): def _build_library(self, data, repository): base_component = self._build_base_component(data, repository) - library, _ = Library.objects.update_or_create( - name=base_component.name, + library = Library.objects.create( + name=base_component.label, owner=base_component.owner, repository=repository, - defaults={ - "lifecycle": data["spec"]["lifecycle"], - "impact": data["spec"]["impact"], - "sonarqube_project": data["spec"] - .get("integrations", {}) - .get("sonarqube_project"), - "slack_channel": data["spec"] - .get("integrations", {}) - .get("slack_channel"), - }, + lifecycle=data["spec"]["lifecycle"], + impact=data["spec"]["impact"], + sonarqube_project=data["spec"] + .get("integrations", {}) + .get("sonarqube_project"), + slack_channel=data["spec"].get("integrations", {}).get("slack_channel"), ) base_component.library = library diff --git a/zoo/entities/migrations/0004_auto_20210713_1003.py b/zoo/entities/migrations/0004_auto_20210713_1003.py index be6aa4a6..7a228f43 100644 --- a/zoo/entities/migrations/0004_auto_20210713_1003.py +++ b/zoo/entities/migrations/0004_auto_20210713_1003.py @@ -1,29 +1,40 @@ # Generated by Django 2.2.19 on 2021-07-13 10:03 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('entities', '0003_auto_20210708_1154'), + ("entities", "0003_auto_20210708_1154"), ] operations = [ migrations.RemoveField( - model_name='entity', - name='group', + model_name="entity", + name="group", ), migrations.AddField( - model_name='group', - name='entity', - field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', related_query_name='group', to='entities.Entity'), + model_name="group", + name="entity", + field=models.ForeignKey( + default=None, + on_delete=django.db.models.deletion.CASCADE, + related_name="groups", + related_query_name="group", + to="entities.Entity", + ), preserve_default=False, ), migrations.AlterField( - model_name='link', - name='entity', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', related_query_name='link', to='entities.Entity'), + model_name="link", + name="entity", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="links", + related_query_name="link", + to="entities.Entity", + ), ), ] diff --git a/zoo/entities/migrations/0005_auto_20210715_0658.py b/zoo/entities/migrations/0005_auto_20210715_0658.py new file mode 100644 index 00000000..3d722ee3 --- /dev/null +++ b/zoo/entities/migrations/0005_auto_20210715_0658.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.19 on 2021-07-15 06:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("repos", "0008_repositoryenvironment"), + ("entities", "0004_auto_20210713_1003"), + ] + + operations = [ + migrations.AddField( + model_name="entity", + name="label", + field=models.CharField(default=None, max_length=100), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name="entity", + unique_together={("name", "source")}, + ), + ] diff --git a/zoo/entities/migrations/0006_auto_20210715_0713.py b/zoo/entities/migrations/0006_auto_20210715_0713.py new file mode 100644 index 00000000..8cb2ca7f --- /dev/null +++ b/zoo/entities/migrations/0006_auto_20210715_0713.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.19 on 2021-07-15 07:13 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("entities", "0005_auto_20210715_0658"), + ] + + operations = [ + migrations.AlterField( + model_name="entity", + name="library", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="libraries.Library", + ), + ), + migrations.AlterField( + model_name="entity", + name="service", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="services.Service", + ), + ), + ] diff --git a/zoo/entities/migrations/0007_auto_20210715_0934.py b/zoo/entities/migrations/0007_auto_20210715_0934.py new file mode 100644 index 00000000..23ad83d8 --- /dev/null +++ b/zoo/entities/migrations/0007_auto_20210715_0934.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.19 on 2021-07-15 09:34 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('entities', '0006_auto_20210715_0713'), + ] + + operations = [ + migrations.AlterField( + model_name='entity', + name='tags', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=50), blank=True, default=list, size=None), + ), + ] diff --git a/zoo/entities/migrations/0008_auto_20210715_1010.py b/zoo/entities/migrations/0008_auto_20210715_1010.py new file mode 100644 index 00000000..8f1c1e17 --- /dev/null +++ b/zoo/entities/migrations/0008_auto_20210715_1010.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.19 on 2021-07-15 10:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('entities', '0007_auto_20210715_0934'), + ] + + operations = [ + migrations.RemoveField( + model_name='group', + name='entity', + ), + migrations.AddField( + model_name='entity', + name='group', + field=models.OneToOneField(default=None, on_delete=django.db.models.deletion.CASCADE, to='entities.Group'), + ), + ] diff --git a/zoo/entities/migrations/0009_auto_20210719_0927.py b/zoo/entities/migrations/0009_auto_20210719_0927.py new file mode 100644 index 00000000..d653ff0b --- /dev/null +++ b/zoo/entities/migrations/0009_auto_20210719_0927.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.19 on 2021-07-19 09:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('entities', '0008_auto_20210715_1010'), + ] + + operations = [ + migrations.AlterField( + model_name='entity', + name='description', + field=models.CharField(blank=True, default='', max_length=255), + ), + ] diff --git a/zoo/entities/models.py b/zoo/entities/models.py index 5ec7ed80..be508f1e 100644 --- a/zoo/entities/models.py +++ b/zoo/entities/models.py @@ -1,4 +1,3 @@ -from django.contrib.contenttypes.fields import GenericRelation from django.contrib.postgres import fields as pg_fields from django.db import models @@ -11,12 +10,6 @@ class Group(models.Model): default=list, help_text="List of maintainers", ) - entity = models.ForeignKey( - "entities.Entity", - related_name="groups", - related_query_name="group", - on_delete=models.CASCADE, - ) class Link(models.Model): @@ -37,16 +30,24 @@ class Link(models.Model): class Entity(models.Model): + class Meta: + unique_together = ("name", "source") + name = models.CharField(max_length=100) + label = models.CharField(max_length=100) type = models.CharField(max_length=32) - description = models.CharField(max_length=255, null=True, blank=True) + description = models.CharField(max_length=255, default="", blank=True) kind = models.CharField(max_length=16) owner = models.CharField(max_length=50) tags = pg_fields.ArrayField( base_field=models.CharField(max_length=50), blank=True, default=list, - help_text="Used for filtering and defining presets of checks", + ) + group = models.OneToOneField( + "entities.Group", + on_delete=models.CASCADE, + default=None, ) source = models.ForeignKey( "repos.Repository", @@ -55,8 +56,11 @@ class Entity(models.Model): on_delete=models.PROTECT, ) service = models.OneToOneField( - "services.Service", on_delete=models.PROTECT, null=True, blank=True + "services.Service", on_delete=models.CASCADE, null=True, blank=True ) library = models.OneToOneField( - "libraries.Library", on_delete=models.PROTECT, null=True, blank=True + "libraries.Library", on_delete=models.CASCADE, null=True, blank=True ) + + def __str__(self): + return f"{self.name}/{self.source.name}" diff --git a/zoo/entities/yaml_definitions/component_base.yaml b/zoo/entities/yaml_definitions/component_base.yaml index 92f14d69..f7ffbf0d 100644 --- a/zoo/entities/yaml_definitions/component_base.yaml +++ b/zoo/entities/yaml_definitions/component_base.yaml @@ -19,10 +19,8 @@ properties: type: string type: array product_owner: - nullable: true type: string project_owner: - nullable: true type: string type: object links: @@ -39,23 +37,25 @@ properties: type: array name: type: string + label: + type: string owner: type: string tags: items: type: string + nullable: true type: array type: object required: - name + - label - owner spec: properties: type_: enum: - database - - service - - library type: string type: object required: diff --git a/zoo/entities/yaml_definitions/component_library.yaml b/zoo/entities/yaml_definitions/component_library.yaml index 1aa2222c..c099f141 100644 --- a/zoo/entities/yaml_definitions/component_library.yaml +++ b/zoo/entities/yaml_definitions/component_library.yaml @@ -40,6 +40,8 @@ properties: - array name: type: string + label: + type: string owner: type: string tags: @@ -50,6 +52,7 @@ properties: type: object required: - name + - label - owner spec: properties: diff --git a/zoo/entities/yaml_definitions/component_service.yaml b/zoo/entities/yaml_definitions/component_service.yaml index b3a0a545..94426f56 100644 --- a/zoo/entities/yaml_definitions/component_service.yaml +++ b/zoo/entities/yaml_definitions/component_service.yaml @@ -40,6 +40,8 @@ properties: - array name: type: string + label: + type: string owner: type: string tags: @@ -50,6 +52,7 @@ properties: type: object required: - name + - label - owner spec: properties: @@ -79,6 +82,7 @@ properties: service_urls: items: type: string + nullable: true type: array type: object type: array diff --git a/zoo/factories.py b/zoo/factories.py index 12688c43..6d1f9174 100644 --- a/zoo/factories.py +++ b/zoo/factories.py @@ -91,6 +91,9 @@ class Meta: name = Faker("domain_word") service = SubFactory(ServiceFactory) + health_check_url = Faker("uri") + dashboard_url = Faker("uri") + service_urls = [Faker("uri"), Faker("uri")] class UserFactory(DjangoModelFactory): @@ -159,7 +162,7 @@ class Meta: value = Faker("slug") -class GroupFactory(Factory): +class GroupFactory(DjangoModelFactory): class Meta: model = Group @@ -187,7 +190,8 @@ class Meta: model = Entity name = Faker("domain_word") - type = Faker("domain_word") + label = Faker("domain_word") + type = "database" description = Faker("paragraph") kind = Faker("domain_word") owner = Faker("user_name") @@ -202,7 +206,8 @@ class Meta: model = Entity name = Faker("domain_word") - type = Faker("domain_word") + label = Faker("domain_word") + type = "service" description = Faker("paragraph") kind = Faker("domain_word") owner = Faker("user_name") @@ -217,6 +222,7 @@ class Meta: model = Entity name = Faker("domain_word") + label = Faker("domain_word") type = Faker("domain_word") description = Faker("paragraph") kind = Faker("domain_word") diff --git a/zoo/repos/entities_yaml.py b/zoo/repos/entities_yaml.py index fd337fd1..b8539178 100644 --- a/zoo/repos/entities_yaml.py +++ b/zoo/repos/entities_yaml.py @@ -62,21 +62,24 @@ def generate_component_base(component): "kind_": "component", "metadata": { "name": component.name, + "label": component.label, "owner": component.owner, "group": { "product_owner": component.group.product_owner, "project_owner": component.group.project_owner, "maintainers": component.group.maintainers, }, - "description": component.description, - "tags": component.tags, + "description": getattr(component, "description", "null"), + "tags": getattr(component, "tags", []), "links": [], }, "spec": {"type_": component.type}, } for link in component.links.all(): - link_document = {"name": link.name, "url": link.url, "icon": link.icon} + link_document = {"name": link.name, "url": link.url} + if link.icon: + link_document["icon"] = link.icon component_base_document["metadata"]["links"].append(link_document) return component_base_document @@ -100,28 +103,24 @@ def generate_component_library(component_library, component_base_doc): def generate_component_service(component_service, component_base_doc): component_base_doc["spec"] = { "type_": "service", - "lifecycle": component_service.service.lifecycle.value, + "lifecycle": component_service.service.lifecycle, "impact": component_service.service.impact, "analysis": [], "environments": [], "integrations": { - "sonarqube_project": component_service.service.sonarqube_project, - "slack_channel": component_service.service.slack_channel, - "sentry_project": component_service.service.sentry_project, - "pagerduty_service": component_service.service.pagerduty_service, + "sonarqube_project": component_service.service.sonarqube_project or "null", + "slack_channel": component_service.service.slack_channel or "null", + "sentry_project": component_service.service.sentry_project or "null", + "pagerduty_service": component_service.service.pagerduty_service or "null", }, } for environment in component_service.service.environments.all(): environment_document = { - "name", - environment.name, - "dashboard_url", - environment.dashboard_url, - "service_urls", - environment.service_urls, - "health_check_url", - environment.health_check_url, + "name": environment.name, + "dashboard_url": environment.dashboard_url or "null", + "health_check_url": environment.dashboard_url or "null", + "service_urls": environment.service_urls, } component_base_doc["spec"]["environments"].append(environment_document) diff --git a/zoo/repos/tasks.py b/zoo/repos/tasks.py index c3800ca9..3f75b5f8 100644 --- a/zoo/repos/tasks.py +++ b/zoo/repos/tasks.py @@ -12,6 +12,8 @@ from ..auditing import runner from ..auditing.check_discovery import CHECKS as AUDITING_CHECKS from ..entities.builder import EntityBuilder +from ..entities.models import Entity, Group, Link +from ..libraries.models import Library from ..repos.models import Endpoint from ..services.constants import EnviromentType from ..services.models import Environment, Service @@ -171,7 +173,7 @@ def update_project_from_entity_file(proj: Dict) -> None: try: content = get_entity_file_content(proj) except FileNotFoundError as err: - log.info("repos.sync_zoo_yml.file_not_found", error=err) + log.info("repos.sync_entity_yml.file_not_found", error=err) else: if not validate(content): return @@ -187,9 +189,17 @@ def update_or_create_components(data: List, proj: Dict) -> None: except Repository.DoesNotExist: return + def _do_cleanup(): + affected_entities = Entity.objects.filter(source=repository) + Link.objects.filter(entity__in=affected_entities).delete() + Service.objects.filter(repository=repository).delete() + Library.objects.filter(repository=repository).delete() + Entity.objects.filter(source=repository).delete() + + _do_cleanup() entity_builder = EntityBuilder() for component in data: - entity_builder.create_entity(component[0], repository) + entity_builder.sync_entities(component[0], repository) def get_entity_file_content(proj: Dict) -> str: