Merge pull request #2 from Panopto/stevielb/m/coveo-improvements

Add the ability to whitelist based on permissions and support push-anonymous permissions

README.md

@@ -46,7 +46,7 @@ python build_standalone.py
 
 This will produce `dist\panopto-connector.exe` which may be copied to and run as a standalone application. Note that `build_standalone.py` will create a windows executable if run from a windows computer, and a linux executable if run from a linux computer.
 
-### Method 3: Install via easy_setup (recommended for hacking/developing)
+### Method 3: Install via easy_setup (recommended for development)
 
 To install the Panopto Index Connector, follow these steps
 
@@ -124,7 +124,7 @@ Next we recommend running the [debug implementation](the-debug-implementation) b
 
 Next, you will choose a target index connector implementation. That will involve either using one of the predefined connector implementations, or developing your own. If choosing an existing implementation, you may need to make some custom code changes to it depending on your exact scenario. Finally, you will define your config file. See below for more details.
 
-When  you are ready to run the connector, you'll do so by running the following commandline: `panopto-index-connector -c <path-to-config-file>`. Once you have tested your connector, we recommend installing it as a service or daemon on the machine it will run on. 
+When you are ready to run the connector, you'll do so by running the following commandline: `panopto-index-connector -c <path-to-config-file>`. Once you have tested your connector, we recommend installing it as a service or daemon on the machine it will run on. 
 
 See below on how to configure your implementation.
 
@@ -153,6 +153,26 @@ panopto_oauth_credentials:
 
 Running the debug connector once you have configured your user and your oauth credentials is recommended to confirm that you are set up to correctly talk to the Panopto APIs, before continuing to your final implementation.
 
+Other configuration options which are helpful to know about:
+```yml
+# Allows you to only send videos to the target matching one of a given permission set
+# First bullet would only sync videos with view permission of a given Panopto groups
+# The second for a given group from an external Identity provider
+# The third would be only videos with panopto public permissions
+# The fourth would be all authenticated users at your organization
+principal_allowlist:
+    - Group:Panopto:mygroup
+    - Group:MyAdProvider:anothergroup
+    - Group:Panopto:Public
+    - Group:Panopto:All Users
+
+# Default if empty or blank is false
+# If true, tells the implementation to not push permissions
+# Note this is only supported out of the box for coveo implementation
+skip_permissions: true
+```
+
+
 ### The Coveo implementation
 
 The Coveo implementation works out of the box with your Coveo push source. You need to define your push source with the fields you select in the configuration of your configuration file. The template configuration file is `coveo.yaml`.

src/panoptoindexconnector/connector.py

@@ -11,6 +11,7 @@
 import json
 import logging
 import os
+import sys
 import time
 
 # Third party
@@ -158,6 +159,39 @@ def save_last_update_time(last_update_time, profile_name):
         file_handle.write(last_update_time.isoformat() + '\n')
 
 
+def should_push(video_content_response, config):
+    """
+    Returns true/false for whether we should push this video.
+
+    Will return "true" if either the permission allowlist is not set, or
+    if the video content contains the allowlisted permission
+    """
+    # if there's no allowlist, proceed
+    if not config.principal_allowlist:
+        return True
+    # else we have a allowlist; let's match against it
+    principals = video_content_response['VideoContent']['Principals']
+    # we'll just walk the permission allowlist and check match against each principal
+    for allowed_principal in config.principal_allowlist:
+        # Format it as <User|Group>:<IdProvider>:<Name>
+        LOG.debug('Considering allowed principal %s', allowed_principal)
+        try:
+            principal_type, id_provider, name = allowed_principal.split(':')
+            assert principal_type in ('User', 'Group')
+        except Exception:  # pylint: disable=broad-except
+            LOG.error('Invalid principal in principal allowlist. Expected format '
+                      '<User|Group>:<IdProvider>:<Name>, received %s', allowed_principal)
+            sys.exit(2)
+        name_key = principal_type + 'name'  # Username or Groupname
+        for principal in principals:
+            LOG.debug('Considering principal %s', principal)
+            principal_name = principal.get(name_key)
+            principal_provider = principal.get('IdentityProvider') or 'Panopto'
+            if principal_name == name and principal_provider == id_provider:
+                return True
+    return False
+
+
 def sync_video_by_id(handler, oauth_token, config, video_id):
     """
     Sync video metadata from Panopto to target by ID
@@ -167,8 +201,11 @@ def sync_video_by_id(handler, oauth_token, config, video_id):
     if video_content_response['Deleted']:
         handler.delete_from_target(video_content_response['Id'])
     else:
-        target_content = handler.convert_to_target(video_content_response)
-        handler.push_to_target(target_content, config)
+        if should_push(video_content_response, config):
+            target_content = handler.convert_to_target(video_content_response)
+            handler.push_to_target(target_content, config)
+        else:
+            LOG.info('Skipping update for video %s as it did not match principal allowlist', video_id)
 
 
 def trigger_rebuild(profile_name):

src/panoptoindexconnector/connector_config.py

@@ -62,7 +62,7 @@ def _get_securely_displayble_config(yaml_config):
                 continue
             node = yaml_config[key]
             for node_key in node:
-                # whitelist -- only username and client id should be shown
+                # allowlist -- only username and client id should be shown
                 if node_key not in ('username', 'client_id', 'grant_type'):
                     node[node_key] = '********'
 
@@ -96,10 +96,20 @@ def polling_frequency(self):
     def polling_retry_minimum(self):
         return timedelta(seconds=self._yaml_config.get('polling_retry_minimum', 300))
 
+    @property
+    def principal_allowlist(self):
+        return self._yaml_config.get('principal_allowlist', None)
+
     @property
     def sleep_seconds(self):
         return self._yaml_config.get('sleep_seconds', 1)
 
+    @property
+    def skip_permissions(self):
+        # ensures that this is parsed correctly where interpreted as bool or string
+        # since yaml flexible; maybe too flexible in this case :)
+        return str(self._yaml_config.get('skip_permissions')).lower() == 'true'
+
     @property
     def target_address(self):
         return self._yaml_config['target_address']

src/panoptoindexconnector/implementations/attivio_implementation.py

@@ -24,11 +24,13 @@
 #########################################################################
 
 
-def convert_to_target(panopto_content, field_mapping):
+def convert_to_target(panopto_content, config):
     """
     Implement this method to convert to target format
     """
 
+    field_mapping = config.field_mapping
+
     target_content = {field_mapping['Id']: panopto_content['Id']}
 
     target_content['fields'] = {

src/panoptoindexconnector/implementations/coveo_implementation.py

@@ -64,11 +64,13 @@
 """
 
 
-def convert_to_target(panopto_content, field_mapping):
+def convert_to_target(panopto_content, config):
     """
-    Implement this method to convert from panopto content format to target format
+    Implement this method to convert to target format
     """
 
+    field_mapping = config.field_mapping
+
     target_content = {
         field_mapping['Id']: panopto_content['Id'],
         'documenttype': 'Panopto',
@@ -81,23 +83,27 @@ def convert_to_target(panopto_content, field_mapping):
             target_content[target_field] = panopto_content['VideoContent'][key]
 
     # Principals
-    target_content['permissions'] = [
-        {
-            'allowedPermissions': [
-                {
-                    'identityType': 'Group' if principal.get('Groupname') else 'User',
-                    'identity': principal.get('Email') or principal.get('Groupname') or '[email protected]'
-                }
-                for principal in panopto_content['VideoContent']['Principals']
-                if principal.get('Groupname') != 'Public'
-                and (principal.get('Email') or principal.get('Groupname') or principal.get('Username') == 'admin')
-            ]
-        }
-    ]
-    target_content['permissions'][0]['allowAnonymous'] = any(
-        principal.get('Groupname') == 'Public'
-        for principal in panopto_content['VideoContent']['Principals']
-    )
+    if not config.skip_permissions:
+        target_content['permissions'] = [
+            {
+                'allowedPermissions': [
+                    {
+                        'identityType': 'Group' if principal.get('Groupname') else 'User',
+                        'identity': principal.get('Email') or principal.get('Groupname') or '[email protected]'
+                    }
+                    for principal in panopto_content['VideoContent']['Principals']
+                    if principal.get('Groupname') != 'Public'
+                    and (principal.get('Email') or principal.get('Groupname') or principal.get('Username') == 'admin')
+                ]
+            }
+        ]
+        target_content['permissions'][0]['allowAnonymous'] = any(
+            principal.get('Groupname') == 'Public'
+            for principal in panopto_content['VideoContent']['Principals']
+        )
+    else:
+        # https://docs.coveo.com/en/107/index-content/simple-permission-model-definition-examples
+        target_content['permissions'] = [{"allowAnonymous": True}]
 
     LOG.debug('Converted document is %s', json.dumps(target_content, indent=2))
 
@@ -129,7 +135,12 @@ def push_needed_security_mappings(target_content, config):
     Push em
     """
 
-    allow_permissions = target_content['permissions'][0]['allowedPermissions']
+    # We use a single rule with potentially multiple values, so we always grab the first
+    principal = target_content['permissions'][0]
+    # If this video has allow anonymous on it, there is nothing to do
+    if principal.get('allowAnonymous', False):
+        return
+    allow_permissions = principal['allowedPermissions']
     needed_permissions = [p for p in allow_permissions if should_map_security(p['identity'])]
     if needed_permissions:
         ensure_each_security_mapping(target_content, config, needed_permissions)

src/panoptoindexconnector/implementations/debug_implementation.py

@@ -26,11 +26,13 @@
 # since this is debug, we'll disable using all the args
 # pylint: disable=unused-argument
 
-def convert_to_target(panopto_content, field_mapping):
+def convert_to_target(panopto_content, config):
     """
     Implement this method to convert to target format
     """
 
+    field_mapping = config.field_mapping
+
     LOG.info('Received the following panopto content: %s', json.dumps(panopto_content, indent=2))
 
     target_content = {'id': panopto_content['Id']}

src/panoptoindexconnector/implementations/template.yaml

@@ -25,6 +25,18 @@ target_credentials:
 # The name of your implementation
 target_implementation: debug_implementation
 
+# Should we allowlist videos based on source permissions
+# leave this blank if you will not allowlist which videos are pushed
+# based on the source Panopto permissions
+principal_allowlist:
+#   - User:Panopto:myuser
+#   - Group:MyIdentityProvider:friends-and-family
+
+# Set to "true" if we should not push permissions to the target;
+# often used with the principal_allowlist to control permissions by
+# what is synced rather than matching the ID Provider on the target
+skip_permissions: false
+
 # Define the mapping from Panopto fields to the target field names
 field_mapping:
     # Top level data

src/panoptoindexconnector/implementations/template_implementation.py

@@ -24,10 +24,13 @@
 #########################################################################
 
 
-def convert_to_target(panopto_content, field_mapping):
+def convert_to_target(panopto_content, config):
     """
-    Implement this method to convert from panopto content format to target format
+    Implement this method to convert to target format
     """
+    # You'll probably only use the field mapping, get it as such
+    # Other relevant values: config.skip_permissions
+    # field_mapping = config.field_mapping
 
     raise NotImplementedError("This is only a template")
 

src/panoptoindexconnector/target_handler.py

@@ -53,7 +53,7 @@ def convert_to_target(self, panopto_video_content):
         Implement this method to convert to target format
         """
         return self._implementation_module.convert_to_target(
-            panopto_video_content, self._config.field_mapping)
+            panopto_video_content, self._config)
 
     def initialize(self):
         """

test/test_conversions.py

@@ -22,7 +22,7 @@ def test_attivio_conversion():
     from panoptoindexconnector.implementations import attivio_implementation as implementation
     from panoptoindexconnector.connector_config import ConnectorConfig
     attivio_path = os.path.join(DIR, '..', 'src', 'panoptoindexconnector', 'implementations', 'attivio.yaml')
-    field_mapping = ConnectorConfig(attivio_path).field_mapping
+    config = ConnectorConfig(attivio_path)
 
     # Dummy content and useful example
     panopto_content = {
@@ -43,6 +43,6 @@ def test_attivio_conversion():
     }
 
     # Test conversion and assert attivio formatting
-    attivio_content = implementation.convert_to_target(panopto_content, field_mapping)
+    attivio_content = implementation.convert_to_target(panopto_content, config)
 
-    assert attivio_content[field_mapping['Id']] == panopto_content['Id']
+    assert attivio_content[config.field_mapping['Id']] == panopto_content['Id']