From d2d39c8f19a64241b102a9cd137328108efc5590 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Mon, 20 Dec 2021 19:36:53 +0100 Subject: [PATCH 01/19] Graph Connector --- MANIFEST | 1 + requirements.txt | 138 ++--- setup.py | 4 +- src/panoptoindexconnector/.gitignore | 1 + src/panoptoindexconnector/connector_config.py | 5 + .../implementations/__init__.py | 1 + .../implementations/graph.yaml | 54 ++ .../implementations/graph_implementation.py | 569 ++++++++++++++++++ .../implementations/graph_schema.json | 76 +++ tools.py | 6 +- 10 files changed, 756 insertions(+), 99 deletions(-) create mode 100644 MANIFEST create mode 100644 src/panoptoindexconnector/implementations/graph.yaml create mode 100644 src/panoptoindexconnector/implementations/graph_implementation.py create mode 100644 src/panoptoindexconnector/implementations/graph_schema.json diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000..0b2fd0b --- /dev/null +++ b/MANIFEST @@ -0,0 +1 @@ +src\panoptoindexconnector\implementations\graph_schema.json \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8b51a69..d6d76fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,105 +1,53 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile # To update, run: # # pip-compile --output-file=requirements.txt requirements.in # -e . - # via -r requirements.in -astroid==2.4.2 - # via pylint -atomicwrites==1.4.0 - # via pytest -attrs==20.3.0 - # via pytest -certifi==2020.12.5 - # via requests -chardet==4.0.0 - # via requests -click==7.1.2 - # via pip-tools -colorama==0.4.4 - # via - # pylint - # pytest -coverage==5.3 - # via - # -r requirements.in - # pytest-cov -flake8==3.8.4 - # via -r requirements.in -idna==2.10 - # via requests -importlib-metadata==3.3.0 - # via - # flake8 - # pluggy - # pytest -iniconfig==1.1.1 - # via pytest -isort==5.6.4 - # via pylint -lazy-object-proxy==1.4.3 - # via astroid -mccabe==0.6.1 - # via - # flake8 - # pylint -mock==4.0.3 - # via -r requirements.in -packaging==20.8 - # via pytest -pip-tools==5.4.0 - # via -r requirements.in -pluggy==0.13.1 - # via pytest -py==1.10.0 - # via pytest -pycodestyle==2.6.0 - # via - # -r requirements.in - # flake8 -pyflakes==2.2.0 - # via - # -r requirements.in - # flake8 -pylint==2.6.0 - # via -r requirements.in -pyparsing==2.4.7 - # via packaging -pyreadline==2.1 - # via panoptoindexconnector -pytest==6.2.1 - # via - # -r requirements.in - # pytest-cov -pytest-cov==2.10.1 - # via -r requirements.in -pytest-runner==5.2 - # via -r requirements.in -requests==2.25.1 - # via panoptoindexconnector -ruamel.yaml==0.15.50 - # via panoptoindexconnector -six==1.15.0 - # via - # astroid - # pip-tools -toml==0.10.2 - # via - # pylint - # pytest -typed-ast==1.4.1 - # via astroid -typing-extensions==3.7.4.3 - # via importlib-metadata -urllib3==1.26.6 - # via requests -wrapt==1.12.1 - # via astroid -zipp==3.4.0 - # via importlib-metadata +astroid==2.4.2 # via pylint +atomicwrites==1.4.0 # via pytest +attrs==20.3.0 # via pytest +certifi==2020.12.5 # via requests +cffi==1.15.0 # via cryptography +chardet==4.0.0 # via requests +click==7.1.2 # via pip-tools +colorama==0.4.4 # via pylint, pytest +coverage==5.3 # via -r requirements.in, pytest-cov +cryptography==36.0.0 # via msal, pyjwt +flake8==3.8.4 # via -r requirements.in +idna==2.10 # via requests +importlib-metadata==3.3.0 # via flake8, pluggy, pytest +iniconfig==1.1.1 # via pytest +isort==5.6.4 # via pylint +lazy-object-proxy==1.4.3 # via astroid +mccabe==0.6.1 # via flake8, pylint +mock==4.0.3 # via -r requirements.in +msal==1.16.0 # via panoptoindexconnector +packaging==20.8 # via pytest +pip-tools==5.4.0 # via -r requirements.in +pluggy==0.13.1 # via pytest +py==1.10.0 # via pytest +pycodestyle==2.6.0 # via -r requirements.in, flake8 +pycparser==2.21 # via cffi +pyflakes==2.2.0 # via -r requirements.in, flake8 +pyjwt[crypto]==2.3.0 # via msal +pylint==2.6.0 # via -r requirements.in +pyparsing==2.4.7 # via packaging +pyreadline==2.1 # via panoptoindexconnector +pytest-cov==2.10.1 # via -r requirements.in +pytest-runner==5.2 # via -r requirements.in +pytest==6.2.1 # via -r requirements.in, pytest-cov +requests==2.25.1 # via msal, panoptoindexconnector +ruamel.yaml==0.15.50 # via panoptoindexconnector +six==1.15.0 # via astroid, pip-tools +toml==0.10.2 # via pylint, pytest +typed-ast==1.4.1 # via astroid +typing-extensions==3.7.4.3 # via importlib-metadata +urllib3==1.26.6 # via requests +wrapt==1.12.1 # via astroid +zipp==3.4.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/setup.py b/setup.py index 9378245..07e5356 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,9 @@ def readme(): with open('README.md') as readme_file: return readme_file.read() + REQUIRES = [ + 'msal', 'pyreadline', 'requests', 'ruamel.yaml<=15.66.0', @@ -26,7 +28,7 @@ def readme(): author_email='sbianamara@panopto.com', description=('A general application for connecting a panopto search index to an external source'), long_description=readme(), - keywords=['python', 'panopto', 'connector', 'attivio', 'coveo'], + keywords=['python', 'panopto', 'connector', 'attivio', 'coveo', 'graph'], install_requires=REQUIRES, package_data={ diff --git a/src/panoptoindexconnector/.gitignore b/src/panoptoindexconnector/.gitignore index dd7fb90..5893706 100644 --- a/src/panoptoindexconnector/.gitignore +++ b/src/panoptoindexconnector/.gitignore @@ -2,5 +2,6 @@ *.yaml !attivio.yaml !coveo.yaml +!graph.yaml !debug.yaml !template.yaml diff --git a/src/panoptoindexconnector/connector_config.py b/src/panoptoindexconnector/connector_config.py index 764b757..6796ffc 100644 --- a/src/panoptoindexconnector/connector_config.py +++ b/src/panoptoindexconnector/connector_config.py @@ -118,6 +118,10 @@ def target_address(self): def target_credentials(self): return self._yaml_config['target_credentials'] + @property + def target_connection(self): + return self._yaml_config['target_connection'] + @property def target_implementation(self): return self._yaml_config['target_implementation'] @@ -127,5 +131,6 @@ class InvalidConfiguration(Exception): """ The configuration specified for the connector is not valid yaml """ + def __init__(self, message): super().__init__('Parse failure: ' + message) diff --git a/src/panoptoindexconnector/implementations/__init__.py b/src/panoptoindexconnector/implementations/__init__.py index f4555a8..b3d486e 100644 --- a/src/panoptoindexconnector/implementations/__init__.py +++ b/src/panoptoindexconnector/implementations/__init__.py @@ -5,5 +5,6 @@ import panoptoindexconnector.implementations.attivio_implementation import panoptoindexconnector.implementations.coveo_implementation import panoptoindexconnector.implementations.debug_implementation +import panoptoindexconnector.implementations.graph_implementation import panoptoindexconnector.implementations.iterator_implementation import panoptoindexconnector.implementations.template_implementation # noqa diff --git a/src/panoptoindexconnector/implementations/graph.yaml b/src/panoptoindexconnector/implementations/graph.yaml new file mode 100644 index 0000000..562fdb4 --- /dev/null +++ b/src/panoptoindexconnector/implementations/graph.yaml @@ -0,0 +1,54 @@ +# +# Panopto index connector configuration file +# + + +# The address to your panopto site +panopto_site_address: https://your.site.panopto.com + +# The oauth credentials to connect to the panopto API +panopto_oauth_credentials: + username: myconnectoruser + password: mypassword + client_id: 123 + client_secret: 456 + grant_type: password + +# Your index integration target endpoint +target_address: https://graph.microsoft.com/v1.0/external/connections + +# Your graph data for the connector +target_credentials: + tenant_id: 00000000-0000-0000-0000-000000000000 + client_id: 00000000-0000-0000-0000-000000000000 + client_secret: myclientsecret + grant_type: client_credentials + scopes: ["https://graph.microsoft.com/.default"] + authority: https://login.microsoftonline.com + +# Your graph connection (connector) that will be used to push Panopto items to +target_connection: + # 'id' can only have ASCII alphanumeric characters and no empty spaces. + id: sampleConnectionId + name: Sample connection name + description: Sample connection description + +# The name of your implementation +target_implementation: graph_implementation + +# Define the mapping from Panopto fields to the target field names +field_mapping: + + # Id in panopto maps to id in graph + Id: id + + # Top level data + Info: + Title: title + Url: uri + + # Content data + Metadata: + Folder: folder + ThumbnailUrl: thumbnailUrl + Summary: description diff --git a/src/panoptoindexconnector/implementations/graph_implementation.py b/src/panoptoindexconnector/implementations/graph_implementation.py new file mode 100644 index 0000000..0ceb8bc --- /dev/null +++ b/src/panoptoindexconnector/implementations/graph_implementation.py @@ -0,0 +1,569 @@ +""" +Methods for the connector application to convert and sync content to the target endpoint + +Implement these methods for the connector application +""" + +# Standard Library Imports +import json +import logging +import os +import time + +# Third party +import requests +import msal + +# Global constants +DIR = os.path.dirname(os.path.realpath(__file__)) +LOG = logging.getLogger(__name__) +APP_TEMP_DIR = str.lower(DIR).replace("panoptoindexconnector\implementations", "") +TOKEN_CACHE = os.path.join(APP_TEMP_DIR, 'token_cache.bin') + + +######################################################################### +# +# Exported methods to implement +# +######################################################################### + + +def convert_to_target(panopto_content, config): + """ + Convert Panopto content to target format + """ + + field_mapping = config.field_mapping + video_content = panopto_content["VideoContent"] + + # Set main properties (id and value) + target_content = set_main_fields_to_new_object(video_content) + + # Set property fields + set_properties(field_mapping, panopto_content, target_content) + + # Set acl (account controll list) + set_principals(config, panopto_content, target_content) + + LOG.debug('Converted document is %s', json.dumps(target_content, indent=2)) + + return target_content + + +def push_to_target(target_content, config): + """ + Push converted Panopto content to the target + """ + + # If target content is None (case when none of permissions are set skip sync) + if not target_content: + LOG.warn("Content has been skipped for sync to target") + return + + content_id = target_content.get("id") + + LOG.info("Pushing content (%s) to target", content_id) + + # Get token + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json' + } + + target_address = config.target_address + connection_id = config.target_connection["id"] + + url = f"{target_address}/{connection_id}/items/{content_id}" + + response = requests.put(url, headers=headers, json=target_content) + + if response.status_code == 200: + LOG.info(f"Content ({content_id}) has been pushed to target!") + else: + LOG.error(f"Content ({content_id}) has NOT been pushed to target! " + + f"Target Content: {target_content}. Response: {response.text}") + + +def delete_from_target(video_id, config): + """ + Delete Panopto content from the target + """ + + LOG.info(f"Deleting video from target by id: {video_id}") + + # Get token + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}' + } + + target_address = config.target_address + connection_id = config.target_connection["id"] + url = f"{target_address}/{connection_id}/items/{video_id}" + + response = requests.delete(url, headers=headers) + + if response.status_code == 200: + LOG.info(f"Video ({video_id}) has been deleted from target!") + elif response.status_code == 404: + LOG.info(f"Video ({video_id}) not found to be delete from target!") + + +# +# Initialize and teardown steps here +# + +def initialize(config): + """ + Create connection and register schema if not exists + """ + + try: + ensure_connection_availability(config) + except Exception as ex: + LOG.error(f'Error occurred while initializing graph connector!. Error: {ex}') + raise + + +def teardown(config): + """ + Delete token cache file + """ + + if os.path.exists(TOKEN_CACHE): + os.remove(TOKEN_CACHE) + + +######################################################################### +# +# Local helpers +# +######################################################################### + + +def set_main_fields_to_new_object(video_content): + """ + Set id and content properties to new object + """ + + return { + "id": video_content["Id"], + "content": { + "type": "text", + "value": "{0}{1}".format(video_content["Title"], " " + video_content["Summary"] if video_content["Summary"] else "") + } + } + + +def set_properties(field_mapping, panopto_content, target_content): + """ + Set properties + """ + + # Set properties from Info yaml + target_content['properties'] = { + field: panopto_content['VideoContent'][key] + for key, field in field_mapping['Info'].items() + if panopto_content['VideoContent'][key] + } + + # Set properties from Metadata yaml + target_content["properties"].update({ + field: panopto_content['VideoContent'][key] + for key, field in field_mapping['Metadata'].items() + if panopto_content['VideoContent'][key] + }) + + +def set_principals(config, panopto_content, target_content): + """ + Set principals - Everyone (in tenant), User or Group. + If none of permissions are set, target_content will be set to None and skipped for sync + """ + + if not config.skip_permissions: + if is_public_or_all_users_principals(panopto_content): + set_principals_to_all(config, target_content) + elif is_user_principals(panopto_content): + set_principals_to_user(config, panopto_content, target_content) + else: + set_principals_to_all(config, target_content) + + if not target_content["acl"]: + target_content = None + + LOG.warn("Target content will be skipped to push to target since none of permissions have applied!") + + +def get_unique_principals(panopto_content): + """ + Get unique principals to avoid duplicate permissions on synced item + """ + + unique_content_principals = [] + + for principal in panopto_content['VideoContent']['Principals']: + if principal not in unique_content_principals: + unique_content_principals.append(principal) + + return unique_content_principals + + +def get_user_unique_not_panopto_principals(panopto_content): + """ + Get unique user not Panopto principals to avoid duplicate permissions on synced item + """ + + unique_user_not_panopto_content_principals = [] + + for principal in panopto_content['VideoContent']['Principals']: + if (principal not in unique_user_not_panopto_content_principals and + principal.get('Username') and principal.get('Email') and + principal.get('IdentityProvider') and principal.get('IdentityProvider') != 'Panopto'): + + unique_user_not_panopto_content_principals.append(principal) + + return unique_user_not_panopto_content_principals + + +def is_public_or_all_users_principals(panopto_content): + """ + Check if session contains Public or All Users group permission + Returns: True or False + """ + + return any( + principal.get('Groupname') == 'Public' + or principal.get('Groupname') == 'All Users' + for principal in get_unique_principals(panopto_content) + ) + + +def is_user_principals(panopto_content): + """ + Check is session contains user non Panopto permission + Returns: True or False + """ + + return True if get_user_unique_not_panopto_principals(panopto_content) else False + + +def set_principals_to_all(config, target_content): + """ + Set session permission to all users from tenant + """ + + target_content["acl"] = [{ + "type": "everyone", + "value": config.target_credentials["tenant_id"], + "accessType": "grant" + }] + + +def set_principals_to_user(config, panopto_content, target_content): + """ + Set session permission to user + """ + + target_content["acl"] = [] + + for principal in get_user_unique_not_panopto_principals(panopto_content): + aad_user_info = get_aad_user_info(config, principal) + + if aad_user_info: + acl = { + "type": "user", + "value": aad_user_info["id"], + "accessType": "grant" + } + target_content["acl"].append(acl) + + +def get_aad_user_info(config, principal): + """ + Get user info from azure active directory by email + Returns: If found returns json response, else returns None + """ + + # Get token + token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {token}' + } + + url = "https://graph.microsoft.com/v1.0/users/{0}".format( + principal.get("Email")) + + response = requests.get(url, headers=headers) + + if response.status_code == 200: + return response.json() + + LOG.warn("Unable to get user's info by email: {0}. Response: {1}".format(principal.get("Email"), response.json())) + + return None + + +def ensure_connection_availability(config): + """ + Ensure that connection is ready for syncing. + If connection is not created, create connection. + If connection is created but schema is not registered, register schema. + """ + + # Get connection if exists + connection_response = get_connection(config) + + # If connection exists + if connection_response.status_code == 200: + # If connection is ready (contains already schema) return from methon + if connection_response.json()["state"] == "ready": + LOG.info("Connection is already created and Ready!") + return + + # If connection limit exceeded inform user and return from method + if connection_response.json()["state"] == "limitExceeded": + LOG.warn("Connection is already created but LIMIT EXCEEDED! " + + "Connection quota must be extended to be able to push new items. " + + "Updating or deleting items will work.") + return + + if connection_response.json()["state"] == "draft": + LOG.info("Connection is already created but not ready (Schema is not registered).") + + # If connection doesn't exit create it + elif connection_response.status_code == 404: + create_connection(config) + + # Check if schema is registered for connection and register it if not + ensure_schema_for_connection(config) + + +def get_access_token(config): + """ + Get access token from cache or a new one + """ + + target_credentials = config.target_credentials + + client_id = target_credentials["client_id"] + client_secret = target_credentials["client_secret"] + authority_url = target_credentials["authority"] + tenant_id = target_credentials["tenant_id"] + scopes = target_credentials["scopes"] + + # Load access token from cache file + token_cache = load_token_cache() + + auth_app = msal.ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"{authority_url}/{tenant_id}", + token_cache=token_cache) + + # Try to get a token from cache + response = auth_app.acquire_token_silent(scopes, account=None) + + # No cached token found. Create a new one + if not response: + response = auth_app.acquire_token_for_client(scopes) + + # Save token to cache file if modified + save_token_cache(token_cache) + + return response['access_token'] + + +def load_token_cache(): + """ + Load token from cache file + """ + + cache = msal.SerializableTokenCache() + + if os.path.exists(TOKEN_CACHE): + with open(TOKEN_CACHE, "r") as tc: + cache.deserialize(tc.read()) + + return cache + + +def save_token_cache(cache: msal.token_cache.SerializableTokenCache): + """ + Save token to cache file + """ + + if cache.has_state_changed: + with open(TOKEN_CACHE, "w") as tc: + tc.write(cache.serialize()) + + +def get_connection(config): + """ + Get connection + Returns: Response + """ + + # Get token + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json' + } + + target_address = config.target_address + connection_id = config.target_connection["id"] + url = f"{target_address}/{connection_id}" + + return requests.get(url, headers=headers) + + +def create_connection(config): + """ + Create connection + Returns: Json response + """ + + LOG.info("Creating connection...") + + connection_data = config.target_connection + + # Get token + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json' + } + + body_data = { + "id": connection_data["id"], + "name": connection_data["name"], + "description": connection_data["description"] + } + + response = requests.post(config.target_address, headers=headers, json=body_data) + + if response.status_code != 201: + LOG.error("Connection has not been created! %s", response.json()) + + response.raise_for_status() + + LOG.info("Connection has been created!") + + return response.json() + + +def ensure_schema_for_connection(config): + """ + Ensure schema for connection + """ + + schema_response = get_schema_for_connection(config) + + # Register schema if not found + if schema_response.status_code == 404: + register_schema_for_connection(config) + + +def get_schema_for_connection(config): + """ + Get schema for connection + Returns: Response + """ + + target_address = config.target_address + target_connection_id = config.target_connection["id"] + + url = f"{target_address}/{target_connection_id}/schema" + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}' + } + + response = requests.get(url, headers=headers) + + return response + + +def register_schema_for_connection(config): + """ + Register schema for connection + Returns: Response + """ + + LOG.info("Registering schema, this may take a moment...") + + target_address = config.target_address + target_connection_id = config.target_connection["id"] + + url = f"{target_address}/{target_connection_id}/schema" + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json' + } + + schema_path = os.path.join(APP_TEMP_DIR, 'graph_schema.json') + + with open(schema_path, "r") as schema_file: + schema_json = json.load(schema_file) + + response = requests.post(url, headers=headers, json=schema_json) + + # Schema is accepted to be registered + if response.status_code == 202: + LOG.info("Schema has been posted and accepted!") + + # Check connection operation status until complete status, so we can proceed with sync + check_connection_operation_status(config, response.headers["Location"]) + + # If schema already exists but still not registered (409 - conflict) + elif response.status_code == 409: + LOG.info("Schema is already posted but still not registered. Please try later. %s", response.json()) + else: + LOG.info("Error while registering connection schema. %s", response.json()) + + response.raise_for_status() + + +def check_connection_operation_status(config, operation_url): + """ + Registering schema may take time so we need to wait + until the schema is registered and connection is ready for sync. + """ + + LOG.info("Checking connection operation status...It may take time until the schema is registered.") + + access_token = get_access_token(config) + + # Set headers + headers = { + 'Authorization': f'Bearer {access_token}' + } + + while True: + response = requests.get(operation_url, headers=headers) + response.raise_for_status() + + response_json = response.json() + if response.status_code == 200 and response_json["status"] == "completed": + LOG.info("Connection is ready!") + break + + # Wait 3 seconds until next check + time.sleep(3) diff --git a/src/panoptoindexconnector/implementations/graph_schema.json b/src/panoptoindexconnector/implementations/graph_schema.json new file mode 100644 index 0000000..cf0c15c --- /dev/null +++ b/src/panoptoindexconnector/implementations/graph_schema.json @@ -0,0 +1,76 @@ +{ + "BaseType": "microsoft.graph.externalItem", + "Properties": [ + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Labels": [ + 0 + ], + "Name": "title", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Labels": [], + "Name": "description", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": false, + "Labels": [ + 1 + ], + "Name": "uri", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": false, + "Labels": [], + "Name": "folder", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": false, + "Labels": [], + "Name": "thumbnailUrl", + "Type": 0, + "AdditionalData": null, + "ODataType": null + } + ], + "Id": null, + "ODataType": null, + "AdditionalData": { + "@odata.context": { + "ValueKind": 3 + } + } +} \ No newline at end of file diff --git a/tools.py b/tools.py index 137c35b..295b85a 100644 --- a/tools.py +++ b/tools.py @@ -32,9 +32,9 @@ def data_files(): """ Translates the manifest into the datafiles """ - # with open(os.path.join(DIR, 'MANIFEST')) as manifest: - # return [line.strip() for line in manifest if line.strip() and '#' not in line] - return [] + with open(os.path.join(DIR, 'MANIFEST')) as manifest: + return [line.strip() for line in manifest if line.strip() and '#' not in line] + def restore_icon(res_dir=RES_DIR): """ From f09fc78d5ca01cb01eb8b48d9397b348d633376b Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Tue, 21 Dec 2021 14:21:51 +0100 Subject: [PATCH 02/19] Graph connector - Added additional searchable fields to schema and yaml file + nit fixes --- .../implementations/graph.yaml | 6 +- .../implementations/graph_implementation.py | 22 +++---- .../implementations/graph_schema.json | 61 ++++++++++++++++++- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/panoptoindexconnector/implementations/graph.yaml b/src/panoptoindexconnector/implementations/graph.yaml index 562fdb4..35acb5c 100644 --- a/src/panoptoindexconnector/implementations/graph.yaml +++ b/src/panoptoindexconnector/implementations/graph.yaml @@ -46,9 +46,13 @@ field_mapping: Info: Title: title Url: uri + ThumbnailUrl: thumbnailUrl # Content data Metadata: Folder: folder - ThumbnailUrl: thumbnailUrl Summary: description + MachineTranscription: machineTranscription + HumanTranscription: humanTranscription + ScreenCapture: screenCapture + Presentation: presentation diff --git a/src/panoptoindexconnector/implementations/graph_implementation.py b/src/panoptoindexconnector/implementations/graph_implementation.py index 0ceb8bc..75f1e3c 100644 --- a/src/panoptoindexconnector/implementations/graph_implementation.py +++ b/src/panoptoindexconnector/implementations/graph_implementation.py @@ -155,7 +155,7 @@ def set_main_fields_to_new_object(video_content): "id": video_content["Id"], "content": { "type": "text", - "value": "{0}{1}".format(video_content["Title"], " " + video_content["Summary"] if video_content["Summary"] else "") + "value": "{0} {1}".format(video_content["Id"], video_content["Title"]) } } @@ -214,21 +214,21 @@ def get_unique_principals(panopto_content): return unique_content_principals -def get_user_unique_not_panopto_principals(panopto_content): +def get_unique_external_user_principals(panopto_content): """ Get unique user not Panopto principals to avoid duplicate permissions on synced item """ - unique_user_not_panopto_content_principals = [] + unique_external_user_principals = [] - for principal in panopto_content['VideoContent']['Principals']: - if (principal not in unique_user_not_panopto_content_principals and - principal.get('Username') and principal.get('Email') and - principal.get('IdentityProvider') and principal.get('IdentityProvider') != 'Panopto'): + for p in panopto_content['VideoContent']['Principals']: + if (p not in unique_external_user_principals and + p.get('Username') and p.get('Email') and + p.get('IdentityProvider') and p.get('IdentityProvider') != 'Panopto'): - unique_user_not_panopto_content_principals.append(principal) + unique_external_user_principals.append(p) - return unique_user_not_panopto_content_principals + return unique_external_user_principals def is_public_or_all_users_principals(panopto_content): @@ -250,7 +250,7 @@ def is_user_principals(panopto_content): Returns: True or False """ - return True if get_user_unique_not_panopto_principals(panopto_content) else False + return bool(get_unique_external_user_principals(panopto_content)) def set_principals_to_all(config, target_content): @@ -272,7 +272,7 @@ def set_principals_to_user(config, panopto_content, target_content): target_content["acl"] = [] - for principal in get_user_unique_not_panopto_principals(panopto_content): + for principal in get_unique_external_user_principals(panopto_content): aad_user_info = get_aad_user_info(config, principal) if aad_user_info: diff --git a/src/panoptoindexconnector/implementations/graph_schema.json b/src/panoptoindexconnector/implementations/graph_schema.json index cf0c15c..a1afa20 100644 --- a/src/panoptoindexconnector/implementations/graph_schema.json +++ b/src/panoptoindexconnector/implementations/graph_schema.json @@ -1,6 +1,17 @@ { "BaseType": "microsoft.graph.externalItem", "Properties": [ + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Name": "id", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, { "Aliases": [], "IsQueryable": true, @@ -46,7 +57,7 @@ "IsQueryable": true, "IsRefinable": false, "IsRetrievable": true, - "IsSearchable": false, + "IsSearchable": true, "Labels": [], "Name": "folder", "Type": 0, @@ -64,6 +75,54 @@ "Type": 0, "AdditionalData": null, "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Labels": [], + "Name": "machineTranscription", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Labels": [], + "Name": "humanTranscription", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Labels": [], + "Name": "screenCapture", + "Type": 0, + "AdditionalData": null, + "ODataType": null + }, + { + "Aliases": [], + "IsQueryable": true, + "IsRefinable": false, + "IsRetrievable": true, + "IsSearchable": true, + "Labels": [], + "Name": "presentation", + "Type": 0, + "AdditionalData": null, + "ODataType": null } ], "Id": null, From 0fcb684d296083c7ed729392371b5fcb6a93beb1 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Wed, 22 Dec 2021 14:27:50 +0100 Subject: [PATCH 03/19] Graph Connector - Renamed graph to microsoft_graph and fixed logic to retrieve Panopto icon --- MANIFEST | 2 +- setup.py | 2 +- src/panoptoindexconnector/.gitignore | 2 +- .../implementations/__init__.py | 2 +- .../{graph.yaml => microsoft_graph.yaml} | 13 +++++++--- ...n.py => microsoft_graph_implementation.py} | 26 +++++++++++-------- ...chema.json => microsoft_graph_schema.json} | 0 tools.py | 17 +++++++----- 8 files changed, 38 insertions(+), 26 deletions(-) rename src/panoptoindexconnector/implementations/{graph.yaml => microsoft_graph.yaml} (75%) rename src/panoptoindexconnector/implementations/{graph_implementation.py => microsoft_graph_implementation.py} (94%) rename src/panoptoindexconnector/implementations/{graph_schema.json => microsoft_graph_schema.json} (100%) diff --git a/MANIFEST b/MANIFEST index 0b2fd0b..0f86689 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1 +1 @@ -src\panoptoindexconnector\implementations\graph_schema.json \ No newline at end of file +src\panoptoindexconnector\implementations\microsoft_graph_schema.json \ No newline at end of file diff --git a/setup.py b/setup.py index 07e5356..9acc3ff 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def readme(): author_email='sbianamara@panopto.com', description=('A general application for connecting a panopto search index to an external source'), long_description=readme(), - keywords=['python', 'panopto', 'connector', 'attivio', 'coveo', 'graph'], + keywords=['python', 'panopto', 'connector', 'attivio', 'coveo', 'microsoft_graph'], install_requires=REQUIRES, package_data={ diff --git a/src/panoptoindexconnector/.gitignore b/src/panoptoindexconnector/.gitignore index 5893706..30a5b41 100644 --- a/src/panoptoindexconnector/.gitignore +++ b/src/panoptoindexconnector/.gitignore @@ -2,6 +2,6 @@ *.yaml !attivio.yaml !coveo.yaml -!graph.yaml +!microsoft_graph.yaml !debug.yaml !template.yaml diff --git a/src/panoptoindexconnector/implementations/__init__.py b/src/panoptoindexconnector/implementations/__init__.py index b3d486e..8fad8f9 100644 --- a/src/panoptoindexconnector/implementations/__init__.py +++ b/src/panoptoindexconnector/implementations/__init__.py @@ -5,6 +5,6 @@ import panoptoindexconnector.implementations.attivio_implementation import panoptoindexconnector.implementations.coveo_implementation import panoptoindexconnector.implementations.debug_implementation -import panoptoindexconnector.implementations.graph_implementation import panoptoindexconnector.implementations.iterator_implementation +import panoptoindexconnector.implementations.microsoft_graph_implementation import panoptoindexconnector.implementations.template_implementation # noqa diff --git a/src/panoptoindexconnector/implementations/graph.yaml b/src/panoptoindexconnector/implementations/microsoft_graph.yaml similarity index 75% rename from src/panoptoindexconnector/implementations/graph.yaml rename to src/panoptoindexconnector/implementations/microsoft_graph.yaml index 35acb5c..5736982 100644 --- a/src/panoptoindexconnector/implementations/graph.yaml +++ b/src/panoptoindexconnector/implementations/microsoft_graph.yaml @@ -17,7 +17,7 @@ panopto_oauth_credentials: # Your index integration target endpoint target_address: https://graph.microsoft.com/v1.0/external/connections -# Your graph data for the connector +# Your Microsoft graph data for the connector target_credentials: tenant_id: 00000000-0000-0000-0000-000000000000 client_id: 00000000-0000-0000-0000-000000000000 @@ -26,7 +26,7 @@ target_credentials: scopes: ["https://graph.microsoft.com/.default"] authority: https://login.microsoftonline.com -# Your graph connection (connector) that will be used to push Panopto items to +# Your Microsoft graph connection (connector) that will be used to push Panopto items to target_connection: # 'id' can only have ASCII alphanumeric characters and no empty spaces. id: sampleConnectionId @@ -34,12 +34,17 @@ target_connection: description: Sample connection description # The name of your implementation -target_implementation: graph_implementation +target_implementation: microsoft_graph_implementation + +# Set to "true" if we should not push permissions to the target; +# often used with the principal_allowlist to control permissions by +# what is synced rather than matching the ID Provider on the target +skip_permissions: false # Define the mapping from Panopto fields to the target field names field_mapping: - # Id in panopto maps to id in graph + # Id in panopto maps to id in Microsoft graph Id: id # Top level data diff --git a/src/panoptoindexconnector/implementations/graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py similarity index 94% rename from src/panoptoindexconnector/implementations/graph_implementation.py rename to src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 75f1e3c..5aff76f 100644 --- a/src/panoptoindexconnector/implementations/graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -43,7 +43,11 @@ def convert_to_target(panopto_content, config): set_properties(field_mapping, panopto_content, target_content) # Set acl (account controll list) - set_principals(config, panopto_content, target_content) + principals_are_set = set_principals(config, panopto_content, target_content) + + # If none of principals are set to content, set skip_sync property to skip pushing content to target + if not principals_are_set: + target_content["skip_sync"] = True LOG.debug('Converted document is %s', json.dumps(target_content, indent=2)) @@ -55,8 +59,7 @@ def push_to_target(target_content, config): Push converted Panopto content to the target """ - # If target content is None (case when none of permissions are set skip sync) - if not target_content: + if target_content.get("skip_sync"): LOG.warn("Content has been skipped for sync to target") return @@ -126,7 +129,7 @@ def initialize(config): try: ensure_connection_availability(config) except Exception as ex: - LOG.error(f'Error occurred while initializing graph connector!. Error: {ex}') + LOG.error(f'Error occurred while initializing microsoft graph connector!. Error: {ex}') raise @@ -183,7 +186,7 @@ def set_properties(field_mapping, panopto_content, target_content): def set_principals(config, panopto_content, target_content): """ Set principals - Everyone (in tenant), User or Group. - If none of permissions are set, target_content will be set to None and skipped for sync + Returns: True if any of principals are set, otherwise False """ if not config.skip_permissions: @@ -194,15 +197,16 @@ def set_principals(config, panopto_content, target_content): else: set_principals_to_all(config, target_content) - if not target_content["acl"]: - target_content = None + if not target_content.get("acl"): + LOG.warn("Target content will be skipped to push to target since none of principals have applied!") + return False - LOG.warn("Target content will be skipped to push to target since none of permissions have applied!") + return True def get_unique_principals(panopto_content): """ - Get unique principals to avoid duplicate permissions on synced item + Get unique principals to avoid duplicate principals on synced item """ unique_content_principals = [] @@ -216,7 +220,7 @@ def get_unique_principals(panopto_content): def get_unique_external_user_principals(panopto_content): """ - Get unique user not Panopto principals to avoid duplicate permissions on synced item + Get unique external Panopto principals to avoid duplicate principals on synced item """ unique_external_user_principals = [] @@ -518,7 +522,7 @@ def register_schema_for_connection(config): 'Content-Type': 'application/json' } - schema_path = os.path.join(APP_TEMP_DIR, 'graph_schema.json') + schema_path = os.path.join(APP_TEMP_DIR, 'microsoft_graph_schema.json') with open(schema_path, "r") as schema_file: schema_json = json.load(schema_file) diff --git a/src/panoptoindexconnector/implementations/graph_schema.json b/src/panoptoindexconnector/implementations/microsoft_graph_schema.json similarity index 100% rename from src/panoptoindexconnector/implementations/graph_schema.json rename to src/panoptoindexconnector/implementations/microsoft_graph_schema.json diff --git a/tools.py b/tools.py index 295b85a..57e1421 100644 --- a/tools.py +++ b/tools.py @@ -1,10 +1,6 @@ - import os import posixpath -try: - from urllib import urlretrieve -except ImportError: - from urllib.request import urlretrieve +import requests DIR = os.path.abspath(os.path.dirname(__file__)) @@ -48,9 +44,16 @@ def restore_icon(res_dir=RES_DIR): url = 'https://www.panopto.com/wp-content/themes/panopto/library/images/favicons/favicon.ico' logo = os.path.join(res_dir, 'panopto.ico') - urlretrieve(url, logo) + # Get favicon icon and store to res\panopto.ico + r = requests.get(url) + + if r.status_code == 200: + with open(logo, 'wb') as f: + f.write(r.content) + + return to_posix(logo) - return to_posix(logo) + return "" # From 3bf1a771108d8f9611e3025a79d5f6f34e5000fa Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Mon, 3 Jan 2022 15:17:05 +0100 Subject: [PATCH 04/19] Graph Connector - added appropriate list format for scopes --- src/panoptoindexconnector/implementations/microsoft_graph.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph.yaml b/src/panoptoindexconnector/implementations/microsoft_graph.yaml index 5736982..9d8dbb9 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph.yaml +++ b/src/panoptoindexconnector/implementations/microsoft_graph.yaml @@ -23,8 +23,9 @@ target_credentials: client_id: 00000000-0000-0000-0000-000000000000 client_secret: myclientsecret grant_type: client_credentials - scopes: ["https://graph.microsoft.com/.default"] authority: https://login.microsoftonline.com + scopes: + - https://graph.microsoft.com/.default # Your Microsoft graph connection (connector) that will be used to push Panopto items to target_connection: From bf57a7a63c2d11a94321436a0fd067a9aa9aaca1 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Fri, 7 Jan 2022 10:23:16 +0100 Subject: [PATCH 05/19] Graph connector - improved connection response handling --- .../microsoft_graph_implementation.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 5aff76f..20dce0b 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -342,10 +342,22 @@ def ensure_connection_availability(config): if connection_response.json()["state"] == "draft": LOG.info("Connection is already created but not ready (Schema is not registered).") + # If request is Unauthorized + elif connection_response.status_code == 401: + LOG.error("Unable to get connection because of Unauthorized request!. " + + "Please check if Client contains 'ExternalConnection.ReadWrite.OwnedBy' and " + + "'ExternalItem.ReadWrite.OwnedBy' API permissions.") + + connection_response.raise_for_status() + # If connection doesn't exit create it elif connection_response.status_code == 404: + LOG.info("Connection doesn't exist!") create_connection(config) + else: + connection_response.raise_for_status() + # Check if schema is registered for connection and register it if not ensure_schema_for_connection(config) @@ -415,6 +427,8 @@ def get_connection(config): Returns: Response """ + LOG.info("Getting connection: %s", config.target_connection["id"]) + # Get token access_token = get_access_token(config) From 729e36448a1e3c93d34ba73c688a80b01831b86a Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Wed, 23 Feb 2022 10:29:07 +0100 Subject: [PATCH 06/19] #115900 - TeamsV2: Graph Connector doesn't stop syncing when the MSFT tenant quota has been exceeded --- src/panoptoindexconnector/connector.py | 7 +++-- .../custom_exceptions.py | 12 +++++++++ .../microsoft_graph_implementation.py | 27 +++++++++++++++---- 3 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 src/panoptoindexconnector/custom_exceptions.py diff --git a/src/panoptoindexconnector/connector.py b/src/panoptoindexconnector/connector.py index 96e60b4..51096f9 100644 --- a/src/panoptoindexconnector/connector.py +++ b/src/panoptoindexconnector/connector.py @@ -22,6 +22,7 @@ from panoptoindexconnector.connector_config import ConnectorConfig, InvalidConfiguration from panoptoindexconnector.helpers import format_request_secure from panoptoindexconnector.target_handler import TargetHandler +from panoptoindexconnector.custom_exceptions import CustomExceptions # 2 minute grace period on oauth expiration @@ -297,9 +298,9 @@ def sync(config, last_update_time): new_last_update_time = last_update_time - handler.initialize() - try: + handler.initialize() + for _ in range(1000): # Renew the oauth token if needed oauth_token, expiration = renew_oauth_token_if_needed( @@ -334,6 +335,8 @@ def sync(config, last_update_time): except requests.exceptions.HTTPError as ex: LOG.exception('Received error response %s | %s', ex.response.status_code, ex.response.text) exception = ex + except CustomExceptions.QuotaLimitExceededError as ex: + exception = ex except Exception as ex: # pylint: disable=broad-except LOG.exception('Received general exception') exception = ex diff --git a/src/panoptoindexconnector/custom_exceptions.py b/src/panoptoindexconnector/custom_exceptions.py new file mode 100644 index 0000000..6534253 --- /dev/null +++ b/src/panoptoindexconnector/custom_exceptions.py @@ -0,0 +1,12 @@ +class CustomExceptions: + """ + User-defined exceptions + """ + + class Error(Exception): + """Base class for other exceptions""" + pass + + class QuotaLimitExceededError(Error): + """Raised when the Tenant quota has exceeded""" + pass \ No newline at end of file diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 20dce0b..b88640d 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -14,6 +14,9 @@ import requests import msal +# Local +from panoptoindexconnector.custom_exceptions import CustomExceptions + # Global constants DIR = os.path.dirname(os.path.realpath(__file__)) LOG = logging.getLogger(__name__) @@ -85,6 +88,16 @@ def push_to_target(target_content, config): if response.status_code == 200: LOG.info(f"Content ({content_id}) has been pushed to target!") + # If request is forbidden + elif response.status_code == 403: + error = response.json()["error"] + + if error and error.get("innerError"): + innerError = error.get("innerError") + if innerError.get('code') == "TenantQuotaExceeded": + raise CustomExceptions.QuotaLimitExceededError(innerError.get("message")) + + response.raise_for_status() else: LOG.error(f"Content ({content_id}) has NOT been pushed to target! " + f"Target Content: {target_content}. Response: {response.text}") @@ -128,6 +141,8 @@ def initialize(config): try: ensure_connection_availability(config) + except CustomExceptions.QuotaLimitExceededError: + raise except Exception as ex: LOG.error(f'Error occurred while initializing microsoft graph connector!. Error: {ex}') raise @@ -332,12 +347,14 @@ def ensure_connection_availability(config): LOG.info("Connection is already created and Ready!") return - # If connection limit exceeded inform user and return from method + # If connection limit exceeded inform user and stop further processing by raising error if connection_response.json()["state"] == "limitExceeded": - LOG.warn("Connection is already created but LIMIT EXCEEDED! " + - "Connection quota must be extended to be able to push new items. " + - "Updating or deleting items will work.") - return + LOG.error("Connection is already created but LIMIT EXCEEDED!") + + raise CustomExceptions.QuotaLimitExceededError( + "Tenant quota has been reached! " + + "To continue adding items to the connection the tenant admin must contact Microsoft or delete some content." + ) if connection_response.json()["state"] == "draft": LOG.info("Connection is already created but not ready (Schema is not registered).") From f8686acc304a7c027ffb8068e71c0523e131e178 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Wed, 23 Feb 2022 10:42:39 +0100 Subject: [PATCH 07/19] #115825-TeamsV2: Graph Connector continues to check non-AAD users it cannot access for each item --- .../microsoft_graph_implementation.py | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 20dce0b..25f92ca 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -20,6 +20,8 @@ APP_TEMP_DIR = str.lower(DIR).replace("panoptoindexconnector\implementations", "") TOKEN_CACHE = os.path.join(APP_TEMP_DIR, 'token_cache.bin') +# Stored users to prevent unnecessary API calls to get id +users = [] ######################################################################### # @@ -277,17 +279,47 @@ def set_principals_to_user(config, panopto_content, target_content): target_content["acl"] = [] for principal in get_unique_external_user_principals(panopto_content): - aad_user_info = get_aad_user_info(config, principal) - if aad_user_info: + principal_user_email = principal.get("Email") + + # Try to get user id from users list + user_id = get_user_id_from_users_list(principal_user_email) + + # If user doean't exist in list, try to get from AAD calling API + if not user_id: + # Get user from AAD + aad_user_info = get_aad_user_info(config, principal) + + if aad_user_info: + user_id = aad_user_info["id"] + + # Add user to list to prevent further API calls for the same user + users.append({principal_user_email: user_id}) + + if user_id: acl = { "type": "user", - "value": aad_user_info["id"], + "value": user_id, "accessType": "grant" } target_content["acl"].append(acl) +def get_user_id_from_users_list(user_email): + """ + Get user id from users list by user e-mail to prevent unnecessary API call to AAD + """ + + user_id = None + + for user in users: + if user.get(user_email): + user_id = user.get(user_email) + break + + return user_id + + def get_aad_user_info(config, principal): """ Get user info from azure active directory by email From 520934edeb5c4b358df8f88f50e20f7311845a90 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Mon, 28 Feb 2022 12:11:19 +0100 Subject: [PATCH 08/19] #115825-TeamsV2: Graph Connector continues to check non-AAD users it cannot access for each item (Clear users list before each sync attempt) --- .../implementations/microsoft_graph_implementation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 25f92ca..91272c2 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -129,6 +129,10 @@ def initialize(config): """ try: + # Clear users list before each sync attempt to keep up to date AAD users info + users.clear() + + # Ensure connection for sync ensure_connection_availability(config) except Exception as ex: LOG.error(f'Error occurred while initializing microsoft graph connector!. Error: {ex}') From 36adfbd1c9d219aecfed673541fcfff8a85594bf Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Wed, 2 Mar 2022 18:04:17 +0100 Subject: [PATCH 09/19] MS Graph Connector - Replaced users list with dictionary --- .../microsoft_graph_implementation.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 91272c2..e62b5fe 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -21,7 +21,7 @@ TOKEN_CACHE = os.path.join(APP_TEMP_DIR, 'token_cache.bin') # Stored users to prevent unnecessary API calls to get id -users = [] +users = {} ######################################################################### # @@ -286,8 +286,8 @@ def set_principals_to_user(config, panopto_content, target_content): principal_user_email = principal.get("Email") - # Try to get user id from users list - user_id = get_user_id_from_users_list(principal_user_email) + # Try to get user id from users dictionary + user_id = users.get(principal_user_email) # If user doean't exist in list, try to get from AAD calling API if not user_id: @@ -298,7 +298,7 @@ def set_principals_to_user(config, panopto_content, target_content): user_id = aad_user_info["id"] # Add user to list to prevent further API calls for the same user - users.append({principal_user_email: user_id}) + users[principal_user_email] = user_id if user_id: acl = { @@ -309,21 +309,6 @@ def set_principals_to_user(config, panopto_content, target_content): target_content["acl"].append(acl) -def get_user_id_from_users_list(user_email): - """ - Get user id from users list by user e-mail to prevent unnecessary API call to AAD - """ - - user_id = None - - for user in users: - if user.get(user_email): - user_id = user.get(user_email) - break - - return user_id - - def get_aad_user_info(config, principal): """ Get user info from azure active directory by email From d4a5963725241850d6f0bd17a0f317ecab309e47 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Wed, 2 Mar 2022 18:21:59 +0100 Subject: [PATCH 10/19] #116340-Graph connector: update the version number to 12.0 --- version_info | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version_info b/version_info index dad5c7c..e2843d6 100644 --- a/version_info +++ b/version_info @@ -6,8 +6,8 @@ VSVersionInfo( ffi=FixedFileInfo( # filevers and prodvers should be always a tuple with four items: (1, 2, 3, 4) # Set not needed items to zero 0. - filevers=(11, 3, 0, 0), - prodvers=(11, 3, 0, 0), + filevers=(12, 0, 0, 0), + prodvers=(12, 0, 0, 0), # Contains a bitmask that specifies the valid bits 'flags'r mask=0x3f, # Contains a bitmask that specifies the Boolean attributes of the file. @@ -31,12 +31,12 @@ VSVersionInfo( u'040904b0', [ StringStruct(u'FileDescription', u'Connects a Panopto search index to an external engine'), - StringStruct(u'FileVersion', u'11.3.0'), + StringStruct(u'FileVersion', u'12.0.0'), StringStruct(u'InternalName', u'Panopto Index Connector'), StringStruct(u'LegalCopyright', u'© Panopto 2019'), StringStruct(u'OriginalFilename', u'panopto-connector.exe'), StringStruct(u'ProductName', u'Panopto Index Connector'), - StringStruct(u'ProductVersion', u'11.3.0'), + StringStruct(u'ProductVersion', u'12.0.0'), ] ) ] From a80751abf0c74971ca3989838d3acca6c4f3a3ab Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Thu, 3 Mar 2022 15:35:53 +0100 Subject: [PATCH 11/19] #116419-Add throttling delay to 0.25 sec for Microsoft Graph connector --- .../implementations/microsoft_graph.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph.yaml b/src/panoptoindexconnector/implementations/microsoft_graph.yaml index 9d8dbb9..cad48e3 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph.yaml +++ b/src/panoptoindexconnector/implementations/microsoft_graph.yaml @@ -42,6 +42,10 @@ target_implementation: microsoft_graph_implementation # what is synced rather than matching the ID Provider on the target skip_permissions: false +# Sleep time to avoid limitation between synced items per second. +# Microsoft Graph Connector has limitation of 4 entries per second. +sleep_seconds: 0.25 + # Define the mapping from Panopto fields to the target field names field_mapping: From 87124997a475a5dc74137508588488611a705554 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Thu, 3 Mar 2022 19:04:04 +0100 Subject: [PATCH 12/19] #116340-Graph connector: update the version number to 11.12 --- version_info | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version_info b/version_info index e2843d6..9ee0ad0 100644 --- a/version_info +++ b/version_info @@ -6,8 +6,8 @@ VSVersionInfo( ffi=FixedFileInfo( # filevers and prodvers should be always a tuple with four items: (1, 2, 3, 4) # Set not needed items to zero 0. - filevers=(12, 0, 0, 0), - prodvers=(12, 0, 0, 0), + filevers=(11, 12, 0, 0), + prodvers=(11, 12, 0, 0), # Contains a bitmask that specifies the valid bits 'flags'r mask=0x3f, # Contains a bitmask that specifies the Boolean attributes of the file. @@ -31,12 +31,12 @@ VSVersionInfo( u'040904b0', [ StringStruct(u'FileDescription', u'Connects a Panopto search index to an external engine'), - StringStruct(u'FileVersion', u'12.0.0'), + StringStruct(u'FileVersion', u'11.12.0'), StringStruct(u'InternalName', u'Panopto Index Connector'), StringStruct(u'LegalCopyright', u'© Panopto 2019'), StringStruct(u'OriginalFilename', u'panopto-connector.exe'), StringStruct(u'ProductName', u'Panopto Index Connector'), - StringStruct(u'ProductVersion', u'12.0.0'), + StringStruct(u'ProductVersion', u'11.12.0'), ] ) ] From 79ca45533ecd091742f290ccafd57453e4894fb7 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Thu, 3 Mar 2022 19:43:10 +0100 Subject: [PATCH 13/19] #115900-TeamsV2: Graph Connector doesn't stop syncing when the MSFT tenant quota has been exceeded --- src/panoptoindexconnector/connector.py | 1 + .../microsoft_graph_implementation.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/panoptoindexconnector/connector.py b/src/panoptoindexconnector/connector.py index 51096f9..201ca7c 100644 --- a/src/panoptoindexconnector/connector.py +++ b/src/panoptoindexconnector/connector.py @@ -336,6 +336,7 @@ def sync(config, last_update_time): LOG.exception('Received error response %s | %s', ex.response.status_code, ex.response.text) exception = ex except CustomExceptions.QuotaLimitExceededError as ex: + # No need to log here since it will be logged in caller method ("run" method) exception = ex except Exception as ex: # pylint: disable=broad-except LOG.exception('Received general exception') diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index a47bf4c..2f15fec 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -99,10 +99,9 @@ def push_to_target(target_content, config): if innerError.get('code') == "TenantQuotaExceeded": raise CustomExceptions.QuotaLimitExceededError(innerError.get("message")) - response.raise_for_status() + log_error_for_not_pushed_content(content_id, target_content, response.text) else: - LOG.error(f"Content ({content_id}) has NOT been pushed to target! " + - f"Target Content: {target_content}. Response: {response.text}") + log_error_for_not_pushed_content(content_id, target_content, response.text) def delete_from_target(video_id, config): @@ -623,3 +622,12 @@ def check_connection_operation_status(config, operation_url): # Wait 3 seconds until next check time.sleep(3) + + +def log_error_for_not_pushed_content(content_id, target_content, response_text): + """ + Logs error for not pushed content to target + """ + + LOG.error(f"Content ({content_id}) has NOT been pushed to target! " + + f"Target Content: {target_content}. Response: {response_text}") \ No newline at end of file From 4e2665650daf34f0c0bc0e80cf3e95d1c17027d1 Mon Sep 17 00:00:00 2001 From: Hiroshi Ohno Date: Thu, 3 Mar 2022 13:41:29 -0800 Subject: [PATCH 14/19] Update README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a852d9..4cce95f 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,10 @@ Unless required by applicable law or agreed to in writing, software distributed This product will connect Panopto to your external search index. This can be done by either leveraging a current connector implementation, or creating your own (more on implementations below). The following sections cover how to install and run the Panopto Index Connector, how to configure an implementation for your schema, and how to develop a custom implementation if need be. -## Installation +## Installation for Microsoft 365 +Please follow [this document](https://docs.google.com/document/d/1CbnS4VnoKponmPx0CaxncpjPRpT7NEJ50XamLHTm4J4/edit?usp=sharing). + +## Installation for platforms other than Microsoft 365 ### Method 1: From official exe (recommended for out of the box) If you do not need to create or customize a connector's implementation, and you are running on Windows, you may use the officially published `panopto-connector.exe` in the [releases tab](https://github.com/Panopto/panopto-index-connector/releases/). For other use cases, use methods 2 or 3. @@ -259,6 +262,10 @@ field_mapping: It assumes that Panopto user names map to Attivio user names, as will be the case in the presence of a shared identity provider. The Panopto API returns the username, identity provider, and email of each user with permission to search for a given video, and you may need to customize the permissions handling in `attivio_implementation.py` to match your Attivio configuration (see [below](building-or-customizing-an-implementation)). +### The Microsoft Graph Connector implementation + +The Microsoft Graph Connector implementation, under `microsoft_graph_implementation.py`, works out of the box with Microsoft 365. To configure this implementation, you'll need to configure the values in `microsoft_graph.yaml`. + ### Developing or customizing an implementation When creating or customizing an implementation, you can start by copying either an existing implementation and its config file, or to start from scratch, copy the template (`template_implementation.py` and `template.yaml`). For this tutorial, we'll call our implementation `my_implementation`. From 524d2bea0a672ec1c1faab6cf55bc2f258c97a77 Mon Sep 17 00:00:00 2001 From: Hiroshi Ohno Date: Fri, 4 Mar 2022 09:10:45 -0800 Subject: [PATCH 15/19] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4cce95f..1e3d889 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,8 @@ It assumes that Panopto user names map to Attivio user names, as will be the cas ### The Microsoft Graph Connector implementation -The Microsoft Graph Connector implementation, under `microsoft_graph_implementation.py`, works out of the box with Microsoft 365. To configure this implementation, you'll need to configure the values in `microsoft_graph.yaml`. +The Microsoft Graph Connector implementation, under `microsoft_graph_implementation.py`, works out of the box with Microsoft 365. +To configure this implementation, please refer [this document](https://docs.google.com/document/d/1CbnS4VnoKponmPx0CaxncpjPRpT7NEJ50XamLHTm4J4/edit?usp=sharing). ### Developing or customizing an implementation From 7b3f22834598b79831d15a48613a35e2fd489ca1 Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Fri, 4 Mar 2022 18:56:51 +0100 Subject: [PATCH 16/19] #116458-GraphConnector hits API Quota exceeded timeout within 30 seconds with sleep_seconds: 0.25 against zoom-test.staging --- src/panoptoindexconnector/implementations/microsoft_graph.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph.yaml b/src/panoptoindexconnector/implementations/microsoft_graph.yaml index cad48e3..0087fa4 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph.yaml +++ b/src/panoptoindexconnector/implementations/microsoft_graph.yaml @@ -44,7 +44,7 @@ skip_permissions: false # Sleep time to avoid limitation between synced items per second. # Microsoft Graph Connector has limitation of 4 entries per second. -sleep_seconds: 0.25 +sleep_seconds: 0.6 # Define the mapping from Panopto fields to the target field names field_mapping: From 1d217f975b48011be36af27958f12c517059841d Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Fri, 4 Mar 2022 19:01:09 +0100 Subject: [PATCH 17/19] #116458-GraphConnector_hits_API_Quota_exceeded_timeout_within_30_seconds_with_sleep_seconds__0.25_against_zoom-test.staging_(added comment for throttling) --- src/panoptoindexconnector/implementations/microsoft_graph.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph.yaml b/src/panoptoindexconnector/implementations/microsoft_graph.yaml index 0087fa4..16145a7 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph.yaml +++ b/src/panoptoindexconnector/implementations/microsoft_graph.yaml @@ -44,6 +44,7 @@ skip_permissions: false # Sleep time to avoid limitation between synced items per second. # Microsoft Graph Connector has limitation of 4 entries per second. +# Panopto API allows 100 request per minute. sleep_seconds: 0.6 # Define the mapping from Panopto fields to the target field names From b4e7d8f088c6778c37a2ad14d36dd4ef671f41be Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Fri, 4 Mar 2022 20:57:17 +0100 Subject: [PATCH 18/19] #115825 - TeamsV2: Graph Connector continues to check non-AAD users it cannot access for each item --- .../implementations/microsoft_graph_implementation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py index 2f15fec..8e45b5e 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py +++ b/src/panoptoindexconnector/implementations/microsoft_graph_implementation.py @@ -299,12 +299,13 @@ def set_principals_to_user(config, panopto_content, target_content): for principal in get_unique_external_user_principals(panopto_content): principal_user_email = principal.get("Email") + user_id = None # Try to get user id from users dictionary - user_id = users.get(principal_user_email) - - # If user doean't exist in list, try to get from AAD calling API - if not user_id: + if principal_user_email in users: + user_id = users.get(principal_user_email) + # If user doesn't exist in dictionary, try to get from AAD calling API + else: # Get user from AAD aad_user_info = get_aad_user_info(config, principal) From eb893d4c07891aeb6edc45eda83718664cd41cac Mon Sep 17 00:00:00 2001 From: Nedim Deliahmetovic Date: Fri, 4 Mar 2022 21:07:08 +0100 Subject: [PATCH 19/19] Removed unnecessary comment --- src/panoptoindexconnector/implementations/microsoft_graph.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/panoptoindexconnector/implementations/microsoft_graph.yaml b/src/panoptoindexconnector/implementations/microsoft_graph.yaml index 16145a7..21ace51 100644 --- a/src/panoptoindexconnector/implementations/microsoft_graph.yaml +++ b/src/panoptoindexconnector/implementations/microsoft_graph.yaml @@ -43,7 +43,6 @@ target_implementation: microsoft_graph_implementation skip_permissions: false # Sleep time to avoid limitation between synced items per second. -# Microsoft Graph Connector has limitation of 4 entries per second. # Panopto API allows 100 request per minute. sleep_seconds: 0.6