Feature/manage databases (#250)

* Add LakeFormation service

Methods to grant access to databases and tables.
Also update glue service to add method to return target table data.

* Add models and views to create database access

* Rename GrantTableAccessView

* Raise 404 if table/database not found

* Limit grantable access based on the users access

A user should only be able to grant access to another user that they
themselves have. If the user is a superuser, they can grant any access.

* Bad idea about using an UpdateView

* Revert "Bad idea about using an UpdateView"

This reverts commit 333b0a6.

* Add view to manage existing table access

* Add django-debug-toolbar for local running

* Add view to revoke access to a table

* Grant/revoke LF access when objects are saved/deleted

* Fix HTML bugs

* Add debug toolbar to requirements so can be used in dev environment

* Render form using includes

* Create custom template for select field

* Remove form includes

* Add custom template for checkbox field

* Update select template, add checkbox template

* Try to fix HMTL lint error

* Reformat file

* Remove validation error added for debugging

* Update granting access call

Dont save the model if the API call fails. Also
use the values from the selected accesslevel model
when making API call.

* Create hybrid opt-in when granting access

Move logic away from model save methods, and on
to methods on the model that are called by the
access forms.

* Handle errors when granting permissions

Use DB transactions to keep the database state in
sync with lake formation permissions.
Add validation to check that selected grantable
are valid based on standard permissions.

* Add display_name to AccessLevel model

* Store permissions and grantable perms separately

Refactors code to remove need for duplicate access levels with
grantable flag.

* Rename AccessLevel to Permission

Remove the grantable field as no longer necessary

* Give users DESCRIBE grantable access to the DB

When a user is granted access to a table, grant the user DESCRIBE access
to the database with grantable permission. This is because having access
to describe the database only allows you to see the name of the DB.

* Squash migrations

* Add default PERMISSION objects

* Hide manage access buttons when neccessary

Only show the actions column to manage/revoke table access if the user
is either a superuser, or has grantable permissions on the table.

* Add admin management for users

* Disable some extra linters

* Update helptext for grantable permissions

.github/workflows/super-linter.yml

@@ -44,3 +44,5 @@ jobs:
           PYTHON_MYPY_CONFIG_FILE: mypy.ini
           VALIDATE_KUBERNETES_KUBECONFORM: false # Super-Linter doesn't support https://github.com/jtyr/kubeconform-helm
           VALIDATE_CHECKOV: false # TODO failures to remediate at later date
+          VALIDATE_PYTHON_PYINK: false # we are using Black instead
+          VALIDATE_HTML_PRETTIER: false # incompatible with django templating language

ap/aws/__init__.py

@@ -1,5 +1,6 @@
 from .base import AWSService
 from .glue import GlueService
+from .lakeformation import LakeFormationService
 from .quicksight import QuicksightService
 
-__all__ = ["AWSService", "QuicksightService", "GlueService"]
+__all__ = ["AWSService", "QuicksightService", "GlueService", "LakeFormationService"]

ap/aws/glue.py

@@ -4,8 +4,8 @@
 class GlueService(base.AWSService):
     aws_service_name = "glue"
 
-    def __init__(self, catalog_id=None):
-        super().__init__()
+    def __init__(self, catalog_id=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self.catalog_id = catalog_id or self.account_id
 
     def get_database_list(self, catalog_id=None):
@@ -32,3 +32,17 @@ def get_table_detail(self, database_name, table_name, catalog_id=None):
         if not table:
             return {}
         return table["Table"]
+
+    def get_database_detail(self, database_name, catalog_id=None):
+        database = self._request(
+            "get_database", CatalogId=catalog_id or self.catalog_id, Name=database_name
+        )
+        if not database:
+            return {}
+        return database["Database"]
+
+    def get_database_for_grant(self, database_name, catalog_id=None):
+        database = self.get_database_detail(database_name, catalog_id=catalog_id)
+        if "TargetDatabase" in database:
+            return database["TargetDatabase"]
+        return database

ap/aws/lakeformation.py

@@ -0,0 +1,207 @@
+import sentry_sdk
+import structlog
+
+from ap.aws.base import AWSService
+
+logger = structlog.get_logger(__name__)
+
+
+class LakeFormationService(AWSService):
+    aws_service_name = "lakeformation"
+
+    def __init__(self, catalog_id=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.catalog_id = catalog_id or self.account_id
+        self.clients = {}
+
+    def grant_table_permissions(
+        self,
+        database: str,
+        table: str,
+        principal: str,
+        catalog_id: str = "",
+        resource_catalog_id: str = "",
+        region_name: str = "",
+        permissions: list | None = None,
+        permissions_with_grant_option: list | None = None,
+    ):
+        client = self.get_client(region_name)
+        return client.grant_permissions(
+            Principal={"DataLakePrincipalIdentifier": principal},
+            Resource={
+                "Table": {
+                    "DatabaseName": database,
+                    "Name": table,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            },
+            CatalogId=catalog_id or self.catalog_id,
+            Permissions=permissions or [],
+            PermissionsWithGrantOption=permissions_with_grant_option or [],
+        )
+
+    def get_client(self, region_name: str = ""):
+        region_name = region_name or self.region_name
+        if region_name not in self.clients:
+            self.clients[region_name] = self.boto3_session.client(
+                "lakeformation", region_name=region_name
+            )
+        return self.clients[region_name]
+
+    def grant_database_permissions(
+        self,
+        database: str,
+        principal: str,
+        region_name: str = "",
+        catalog_id: str = "",
+        resource_catalog_id: str = "",
+        permissions: list | None = None,
+        grantable_permissions: list | None = None,
+    ):
+        """
+        Grant the principal permissions to the database.
+        """
+        client = self.get_client(region_name)
+        return client.grant_permissions(
+            Principal={"DataLakePrincipalIdentifier": principal},
+            Resource={
+                "Database": {
+                    "Name": database,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            },
+            Permissions=permissions or ["DESCRIBE"],
+            PermissionsWithGrantOption=grantable_permissions or [],
+            CatalogId=catalog_id or self.catalog_id,
+        )
+
+    def revoke_table_permissions(
+        self,
+        database: str,
+        table: str,
+        principal: str,
+        resource_catalog_id: str = "",
+        region_name: str = "",
+        permissions: list | None = None,
+        grantable_permissions: list | None = None,
+    ):
+        client = self.get_client(region_name)
+        resource = {
+            "Table": {
+                "DatabaseName": database,
+                "Name": table,
+                "CatalogId": resource_catalog_id or self.catalog_id,
+            },
+        }
+        try:
+            return client.revoke_permissions(
+                Principal={"DataLakePrincipalIdentifier": principal},
+                Resource=resource,
+                Permissions=permissions or [],
+                PermissionsWithGrantOption=grantable_permissions or [],
+            )
+        except client.exceptions.InvalidInputException as error:
+            sentry_sdk.capture_exception(error)
+            logger.info(f"Error revoking permissions for {principal}", error=error)
+            raise error
+
+    def revoke_database_permissions(
+        self,
+        database: str,
+        principal: str,
+        region_name: str = "",
+        catalog_id: str = "",
+        resource_catalog_id: str = "",
+        permissions: list | None = None,
+    ):
+        """
+        Grant the principal permissions to the database.
+        """
+        client = self.get_client(region_name)
+        return client.revoke_permissions(
+            Principal={"DataLakePrincipalIdentifier": principal},
+            Resource={
+                "Database": {
+                    "Name": database,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            },
+            Permissions=permissions or ["DESCRIBE"],
+            CatalogId=catalog_id or self.catalog_id,
+        )
+
+    def create_lake_formation_opt_in(
+        self,
+        database: str,
+        principal: str,
+        table: str = "",
+        resource_catalog_id: str = "",
+        region_name: str = "",
+    ):
+        client = self.get_client(region_name or "eu-west-1")
+        if table:
+            resource = {
+                "Table": {
+                    "DatabaseName": database,
+                    "Name": table,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            }
+        else:
+            resource = {
+                "Database": {
+                    "Name": database,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            }
+
+        client.create_lake_formation_opt_in(
+            Principal={"DataLakePrincipalIdentifier": principal}, Resource=resource
+        )
+
+    def delete_lake_formation_opt_in(
+        self,
+        database: str,
+        principal: str,
+        table: str = "",
+        resource_catalog_id: str = "",
+        region_name: str = "",
+    ):
+        client = self.get_client(region_name or "eu-west-1")
+        if table:
+            resource = {
+                "Table": {
+                    "DatabaseName": database,
+                    "Name": table,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            }
+        else:
+            resource = {
+                "Database": {
+                    "Name": database,
+                    "CatalogId": resource_catalog_id or self.catalog_id,
+                },
+            }
+
+        client.delete_lake_formation_opt_in(
+            Principal={"DataLakePrincipalIdentifier": principal}, Resource=resource
+        )
+
+    def list_permissions(self, principal, resource):
+        logger.info(f"Getting permissions for {principal} on {resource}")
+        client = self.get_client(region_name="eu-west-1")
+        response = client.list_permissions(
+            Principal={"DataLakePrincipalIdentifier": principal}, Resource=resource
+        )
+        permissions = response["PrincipalResourcePermissions"]
+
+        if not permissions:
+            return None
+
+        return {
+            "Permissions": response["PrincipalResourcePermissions"][0]["Permissions"],
+            "PermissionsWithGrantOption": response["PrincipalResourcePermissions"][0][
+                "PermissionsWithGrantOption"
+            ],
+        }

ap/database_access/admin.py

@@ -0,0 +1,9 @@
+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.Permission)
+class PermissionAdmin(admin.ModelAdmin):
+    list_display = ("name", "entity", "display_name")
+    ordering = ("name", "entity")

ap/database_access/forms.py

@@ -0,0 +1,128 @@
+from django import forms
+from django.db import transaction
+
+import botocore
+import structlog
+
+from ap.users.models import User
+
+from . import models
+
+logger = structlog.get_logger(__name__)
+
+
+class AccessForm(forms.ModelForm):
+    user = forms.ModelChoiceField(
+        queryset=User.objects.all(),
+        label="Select a user to grant access to",
+        template_name="forms/fields/select.html",
+    )
+    permissions = forms.ModelMultipleChoiceField(
+        queryset=None,
+        widget=forms.CheckboxSelectMultiple,
+        template_name="forms/fields/checkbox.html",
+        help_text="Select all that apply",
+        required=True,
+    )
+    grantable_permissions = forms.ModelMultipleChoiceField(
+        queryset=None,
+        widget=forms.CheckboxSelectMultiple,
+        template_name="forms/fields/checkbox.html",
+        help_text="Select all that apply",
+        required=False,
+    )
+
+    def __init__(self, *args, **kwargs):
+        self.table_name = kwargs.pop("table_name")
+        self.database_name = kwargs.pop("database_name")
+        self.grantable_access = kwargs.pop("grantable_access")
+        super().__init__(*args, **kwargs)
+        self.fields["permissions"].queryset = self.grantable_access
+        self.fields["grantable_permissions"].queryset = self.grantable_access
+
+    class Meta:
+        model = models.TableAccess
+        fields = ["permissions", "grantable_permissions"]
+
+    def clean_user(self):
+        user = self.cleaned_data.get("user")
+        try:
+            self._meta.model.objects.get(name=self.table_name, database_access__user=user)
+            raise forms.ValidationError("Selected user already has access to this table.")
+        except self._meta.model.DoesNotExist:
+            pass
+
+        return user
+
+    @transaction.atomic
+    def save(self, commit=True):
+        instance = super().save(commit=False)
+        instance.name = self.table_name
+        database_access, created = models.DatabaseAccess.objects.get_or_create(
+            user=self.cleaned_data["user"], name=self.database_name
+        )
+        instance.database_access = database_access
+        instance.save()
+        self.save_m2m()
+        instance.grant_lakeformation_permissions(create_hybrid_opt_in=True)
+        if created:
+            database_access.grant_lakeformation_permissions(create_hybrid_opt_in=True)
+        return instance
+
+
+class ManageAccessForm(forms.ModelForm):
+    permissions = forms.ModelMultipleChoiceField(
+        queryset=None,
+        widget=forms.CheckboxSelectMultiple,
+        template_name="forms/fields/checkbox.html",
+        help_text="Select all that apply",
+        required=True,
+    )
+    grantable_permissions = forms.ModelMultipleChoiceField(
+        queryset=None,
+        widget=forms.CheckboxSelectMultiple,
+        template_name="forms/fields/checkbox.html",
+        help_text="Grantable permissions allow the user to grant the selected permissions to other users.",  # noqa
+        required=False,
+    )
+
+    class Meta:
+        model = models.TableAccess
+        fields = ["permissions", "grantable_permissions"]
+
+    def __init__(self, *args, **kwargs):
+        self.grantable_access = kwargs.pop("grantable_access")
+        super().__init__(*args, **kwargs)
+        self.fields["permissions"].queryset = self.grantable_access
+        self.fields["grantable_permissions"].queryset = self.grantable_access
+
+    def clean(self):
+        cleaned_data = super().clean()
+
+        grantable_permissions = cleaned_data.get("grantable_permissions", [])
+        if not grantable_permissions:
+            return cleaned_data
+
+        permissions = cleaned_data.get("permissions", [])
+        for grantable_permission in grantable_permissions:
+            if grantable_permission not in permissions:
+                self.add_error(
+                    "grantable_permissions",
+                    f"{grantable_permission} is not a part of the selected grantable permissions.",
+                )
+
+        return cleaned_data
+
+    def save(self, commit=True):
+        try:
+            with transaction.atomic():
+                instance = super().save(commit=False)
+                instance.revoke_lakeformation_permissions()
+                instance.save()
+                self.save_m2m()
+                instance.grant_lakeformation_permissions()
+                return instance
+        except botocore.exceptions.ClientError as error:
+            logger.info("Updating permissions failed, restoring original permissions", error=error)
+            instance.grant_lakeformation_permissions()
+            raise error