From 197b889d287d8be7f83d240240230244dd9b3dd7 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:22:45 -0500 Subject: [PATCH 01/49] Pin pydantic to 2.7.1 - Temporarily install bump-pydantic to help with upgrade - Temporarily install eval_type_backport to get around issues with 'bool | None' that don't appear to be in this codebase. - Use StringConstraints instead of ConstrainedStr on FidesKey and FidesCollectionKey --- requirements.txt | 4 +++- src/fideslang/models.py | 10 +++++++--- src/fideslang/validation.py | 4 ++-- tests/conftest.py | 1 - 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index cc280885..64279819 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -pydantic>=1.8.1,<1.11.0 +pydantic==2.7.1 pyyaml>=5,<7 packaging>=20.0 +bump-pydantic==0.8.0 +eval_type_backport==0.2.0 \ No newline at end of file diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 6d380286..399fefa2 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -12,7 +12,7 @@ from pydantic import ( AnyUrl, BaseModel, - ConstrainedStr, + StringConstraints, Field, HttpUrl, PositiveInt, @@ -246,7 +246,7 @@ class SpecialCategoryLegalBasisEnum(str, Enum): class DataCategory(FidesModel, DefaultModel): """The DataCategory resource model.""" - parent_key: Optional[FidesKey] + parent_key: Optional[FidesKey] = None _matching_parent_key: classmethod = matching_parent_key_validator _no_self_reference: classmethod = no_self_reference_validator @@ -301,6 +301,7 @@ class DataSubject(FidesModel, DefaultModel): rights: Optional[DataSubjectRights] = Field(description=DataSubjectRights.__doc__) automated_decisions_or_profiling: Optional[bool] = Field( + default=False, description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.", ) @@ -369,6 +370,7 @@ class FidesMeta(BaseModel): description="The type of the identity data that should be used to query this collection for a DSR." ) primary_key: Optional[bool] = Field( + default=False, description="Whether the current field can be considered a primary key of the current collection" ) data_type: Optional[str] = Field( @@ -378,9 +380,11 @@ class FidesMeta(BaseModel): description="Optionally specify the allowable field length. Fides will not generate values that exceed this size." ) return_all_elements: Optional[bool] = Field( + default=False, description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False." ) read_only: Optional[bool] = Field( + default=False, description="Optionally specify if a field is read-only, meaning it can't be updated or deleted." ) @@ -471,7 +475,7 @@ def validate_object_fields( # type: ignore DatasetField.update_forward_refs() -class FidesCollectionKey(ConstrainedStr): +class FidesCollectionKey(StringConstraints): """ Dataset.Collection name where both dataset and collection names are valid FidesKeys """ diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 6a0236c3..97edfae7 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -6,7 +6,7 @@ from typing import Dict, Generator, List, Optional, Pattern, Set, Tuple from packaging.version import Version -from pydantic import ConstrainedStr +from pydantic import StringConstraints class FidesValidationError(ValueError): @@ -26,7 +26,7 @@ def validate(cls, value: str) -> Version: return Version(value) -class FidesKey(ConstrainedStr): +class FidesKey(StringConstraints): """ A FidesKey type that creates a custom constrained string. """ diff --git a/tests/conftest.py b/tests/conftest.py index 6dbebc9c..1d2b15ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ import pytest import yaml - from fideslang import models From a644507af987ccba35fdc7f7322695fd199cacdc Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:27:12 -0500 Subject: [PATCH 02/49] Replace "class Config" with "model_config" which is a ConfigDict. "orm_mode" has been renamed to "from_attributes" Affects: FidesModel, Cookies, Evaluation, PrivacyDeclaration, and System. --- src/fideslang/models.py | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 399fefa2..fc761c45 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -74,11 +74,7 @@ class FidesModel(BaseModel): tags: Optional[List[str]] = None name: Optional[str] = name_field description: Optional[str] = description_field - - class Config: - "Config for the FidesModel" - extra = "ignore" - orm_mode = True + model_config = ConfigDict(extra="ignore", from_attributes=True) class DefaultModel(BaseModel): @@ -259,10 +255,7 @@ class Cookies(BaseModel): path: Optional[str] domain: Optional[str] - class Config: - """Config for the cookies""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) class DataSubjectRights(BaseModel): @@ -643,11 +636,7 @@ class Evaluation(BaseModel): default="", description="A human-readable string response for the evaluation.", ) - - class Config: - "Config for the Evaluation" - extra = "ignore" - orm_mode = True + model_config = ConfigDict(extra="ignore", from_attributes=True) # Organization @@ -836,11 +825,7 @@ class PrivacyDeclaration(BaseModel): cookies: Optional[List[Cookies]] = Field( description="Cookies associated with this data use to deliver services and functionality", ) - - class Config: - """Config for the Privacy Declaration""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) class SystemMetadata(BaseModel): @@ -1069,11 +1054,7 @@ def privacy_declarations_reference_data_flows( ], f"PrivacyDeclaration '{value.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource." return value - - class Config: - """Class for the System config""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) # Taxonomy From 5cf361fa25cdfd97f0f5c530be3b1ae9a46e1e55 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:31:55 -0500 Subject: [PATCH 03/49] root_validator has been deprecated in favor of model_validator --- src/fideslang/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index fc761c45..cce82973 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -16,7 +16,6 @@ Field, HttpUrl, PositiveInt, - root_validator, validator, ) @@ -274,7 +273,7 @@ class DataSubjectRights(BaseModel): description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) - @root_validator() + @model_validator() @classmethod def include_exclude_has_values(cls, values: Dict) -> Dict: """ @@ -875,7 +874,7 @@ class DataFlow(BaseModel): description="An array of data categories describing the data in transit.", ) - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) @classmethod def user_special_case(cls, values: Dict) -> Dict: """ From 6c1be9677d68fc6532b371daf3173567d385b79f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:36:29 -0500 Subject: [PATCH 04/49] @validator has been deprecated in favor of field_validator FidesMeta.valid_data_type. Dataset.valid_meta, Dataflow.verify_type_is_flowable replaced here. --- src/fideslang/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index cce82973..1d71eb9c 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -380,7 +380,7 @@ class FidesMeta(BaseModel): description="Optionally specify if a field is read-only, meaning it can't be updated or deleted." ) - @validator("data_type") + @field_validator("data_type") @classmethod def valid_data_type(cls, value: Optional[str]) -> Optional[str]: """Validate that all annotated data types exist in the taxonomy""" @@ -415,7 +415,7 @@ class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat): description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", ) - @validator("fides_meta") + @field_validator("fides_meta") @classmethod def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: """Validate upfront that the return_all_elements flag can only be specified on array fields""" @@ -889,7 +889,7 @@ def user_special_case(cls, values: Dict) -> Dict: return values - @validator("type") + @field_validator("type") @classmethod def verify_type_is_flowable(cls, value: str) -> str: """ From 51332a3b3caec592adbe375cdab6c6c7be24c0a4 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:37:18 -0500 Subject: [PATCH 05/49] Add missing field_validator, model_validator, and ConfigDict imports. --- src/fideslang/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 1d71eb9c..6a2cdea3 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -10,6 +10,9 @@ from typing import Any, Dict, List, Optional, Union from pydantic import ( + field_validator, + model_validator, + ConfigDict, AnyUrl, BaseModel, StringConstraints, From d7323eb489912891795260e96def76ad13ec2f76 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:40:55 -0500 Subject: [PATCH 06/49] Update some @validator -> @field_validator changes that bump-pydantic couldn't make automatically. Includes: DefaultModel.validate_verion_added, DefaultModel.validate_version_deprecated, DatasetField.validate_object_fields --- src/fideslang/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 6a2cdea3..8e1894cf 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -107,7 +107,7 @@ class DefaultModel(BaseModel): ) _is_deprecated_if_replaced: classmethod = is_deprecated_if_replaced_validator - @validator("version_added") + @field_validator("version_added") @classmethod def validate_verion_added( cls, version_added: Optional[str], values: Dict @@ -121,7 +121,7 @@ def validate_verion_added( FidesVersion.validate(version_added) return version_added - @validator("version_deprecated") + @field_validator("version_deprecated") @classmethod def validate_version_deprecated( cls, version_deprecated: Optional[str], values: Dict @@ -434,7 +434,7 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: ) return meta_values - @validator("fields") + @field_validator("fields") @classmethod def validate_object_fields( # type: ignore cls, From 0c06a15dc690e10342fa470fda349527318ff57c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:50:06 -0500 Subject: [PATCH 07/49] Adjust DefaultModel.validate_version_added and validate_version_deprecated to not have values keyword. --- src/fideslang/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 8e1894cf..b32aa6b5 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -109,8 +109,8 @@ class DefaultModel(BaseModel): @field_validator("version_added") @classmethod - def validate_verion_added( - cls, version_added: Optional[str], values: Dict + def validate_version_added( + cls, version_added: Optional[str] ) -> Optional[str]: """ Validate that the `version_added` field is a proper FidesVersion @@ -124,7 +124,7 @@ def validate_verion_added( @field_validator("version_deprecated") @classmethod def validate_version_deprecated( - cls, version_deprecated: Optional[str], values: Dict + cls, version_deprecated: Optional[str] ) -> Optional[str]: """ Validate that the `version_deprecated` is a proper FidesVersion From 4bbd059cbd98944da78a268073ffa0be9f3d5986 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:52:59 -0500 Subject: [PATCH 08/49] Convert DatasetField.validate_object_fields from a field_validator to a root_validator which is better when performing validation using multiple field values --- src/fideslang/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index b32aa6b5..4471766a 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -434,17 +434,17 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: ) return meta_values - @field_validator("fields") + @model_validator() @classmethod def validate_object_fields( # type: ignore cls, - fields: Optional[List["DatasetField"]], values: Dict[str, Any], ) -> Optional[List["DatasetField"]]: """Two validation checks for object fields: - If there are sub-fields specified, type should be either empty or 'object' - Additionally object fields cannot have data_categories. """ + fields = values.get("fields") declared_data_type = None field_name: str = values.get("name") # type: ignore @@ -463,7 +463,7 @@ def validate_object_fields( # type: ignore f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead" ) - return fields + return values # this is required for the recursive reference in the pydantic model: @@ -1056,6 +1056,7 @@ def privacy_declarations_reference_data_flows( ], f"PrivacyDeclaration '{value.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource." return value + model_config = ConfigDict(use_enum_values=True) From 20081c1c3850faa665d358326f975668c7de80b5 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 14:58:19 -0500 Subject: [PATCH 09/49] Transform System.privacy_declarations_reference_data_flows from a field_validator into a model_validator to better be able to work with privacy declaration and system ingresses and egresses together, since "each_item" has been deprecated. --- src/fideslang/models.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 4471766a..c9e53796 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -1029,33 +1029,33 @@ class System(FidesModel): "privacy_declarations", allow_reuse=True )(sort_list_objects_by_name) - @validator("privacy_declarations", each_item=True) + @model_validator() @classmethod def privacy_declarations_reference_data_flows( cls, - value: PrivacyDeclaration, values: Dict, ) -> PrivacyDeclaration: """ Any `PrivacyDeclaration`s which include `egress` and/or `ingress` fields must only reference the `fides_key`s of defined `DataFlow`s in said field(s). """ + privacy_declarations = values.get("privacy_declarations") or [] + for privacy_declaration in privacy_declarations: + for direction in ["egress", "ingress"]: + fides_keys = getattr(privacy_declaration, direction, None) + if fides_keys is not None: + data_flows = values[direction] + system = values["fides_key"] + assert ( + data_flows is not None and len(data_flows) > 0 + ), f"PrivacyDeclaration '{privacy_declaration.name}' defines {direction} with one or more resources and is applied to the System '{system}', which does not itself define any {direction}." + + for fides_key in fides_keys: + assert fides_key in [ + data_flow.fides_key for data_flow in data_flows + ], f"PrivacyDeclaration '{privacy_declaration.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource." - for direction in ["egress", "ingress"]: - fides_keys = getattr(value, direction, None) - if fides_keys is not None: - data_flows = values[direction] - system = values["fides_key"] - assert ( - data_flows is not None and len(data_flows) > 0 - ), f"PrivacyDeclaration '{value.name}' defines {direction} with one or more resources and is applied to the System '{system}', which does not itself define any {direction}." - - for fides_key in fides_keys: - assert fides_key in [ - data_flow.fides_key for data_flow in data_flows - ], f"PrivacyDeclaration '{value.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource." - - return value + return values model_config = ConfigDict(use_enum_values=True) From 912b9a94b4636cd5cc5532a43eefc5a50100cab5 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 15:05:49 -0500 Subject: [PATCH 10/49] Fields that were marked Optional used to not be required. Now this just means that None is an allowed value, so you need to specify a default of None to match existing behavior. --- src/fideslang/models.py | 107 ++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index c9e53796..3c59558d 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -254,9 +254,8 @@ class Cookies(BaseModel): """The Cookies resource model""" name: str - path: Optional[str] - domain: Optional[str] - + path: Optional[str] = None + domain: Optional[str] = None model_config = ConfigDict(from_attributes=True) @@ -273,7 +272,7 @@ class DataSubjectRights(BaseModel): description="Defines the strategy used when mapping data rights to a data subject.", ) values: Optional[List[DataSubjectRightsEnum]] = Field( - description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", + default=None, description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) @model_validator() @@ -294,7 +293,7 @@ def include_exclude_has_values(cls, values: Dict) -> Dict: class DataSubject(FidesModel, DefaultModel): """The DataSubject resource model.""" - rights: Optional[DataSubjectRights] = Field(description=DataSubjectRights.__doc__) + rights: Optional[DataSubjectRights] = Field(default=None, description=DataSubjectRights.__doc__) automated_decisions_or_profiling: Optional[bool] = Field( default=False, description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.", @@ -335,7 +334,7 @@ class MyDatasetField(DatasetFieldBase): name: str = name_field description: Optional[str] = description_field data_categories: Optional[List[FidesKey]] = Field( - description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.", + default=None, description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.", ) @@ -351,7 +350,7 @@ class FidesDatasetReference(BaseModel): dataset: FidesKey field: str - direction: Optional[EdgeDirection] + direction: Optional[EdgeDirection] = None class FidesMeta(BaseModel): @@ -362,17 +361,17 @@ class FidesMeta(BaseModel): default=None, ) identity: Optional[str] = Field( - description="The type of the identity data that should be used to query this collection for a DSR." + default=None, description="The type of the identity data that should be used to query this collection for a DSR." ) primary_key: Optional[bool] = Field( default=False, description="Whether the current field can be considered a primary key of the current collection" ) data_type: Optional[str] = Field( - description="Optionally specify the data type. Fides will attempt to cast values to this type when querying." + default=None, description="Optionally specify the data type. Fides will attempt to cast values to this type when querying." ) length: Optional[PositiveInt] = Field( - description="Optionally specify the allowable field length. Fides will not generate values that exceed this size." + default=None, description="Optionally specify the allowable field length. Fides will not generate values that exceed this size." ) return_all_elements: Optional[bool] = Field( default=False, @@ -415,7 +414,7 @@ class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat): fides_meta: Optional[FidesMeta] = None fields: Optional[List[DatasetField]] = Field( - description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", + default=None, description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", ) @field_validator("fides_meta") @@ -494,7 +493,7 @@ def validate(cls, value: str) -> str: class CollectionMeta(BaseModel): """Collection-level specific annotations used for query traversal""" - after: Optional[List[FidesCollectionKey]] + after: Optional[List[FidesCollectionKey]] = None skip_processing: Optional[bool] = False @@ -508,7 +507,7 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): name: str = name_field description: Optional[str] = description_field data_categories: Optional[List[FidesKey]] = Field( - description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.", + default=None, description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.", ) fields: List[DatasetField] = Field( description="An array of objects that describe the collection's fields.", @@ -560,8 +559,8 @@ class DatasetMetadata(BaseModel): Object used to hold application specific metadata for a dataset """ - resource_id: Optional[str] - after: Optional[List[FidesKey]] + resource_id: Optional[str] = None + after: Optional[List[FidesKey]] = None class Dataset(FidesModel, FidesopsMetaBackwardsCompat): @@ -569,7 +568,7 @@ class Dataset(FidesModel, FidesopsMetaBackwardsCompat): meta: Optional[Dict] = meta_field data_categories: Optional[List[FidesKey]] = Field( - description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.", + default=None, description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.", ) fides_meta: Optional[DatasetMetadata] = Field( description=DatasetMetadata.__doc__, default=None @@ -663,7 +662,7 @@ class OrganizationMetadata(BaseModel): """ resource_filters: Optional[List[ResourceFilter]] = Field( - description="A list of filters that can be used when generating or scanning systems." + default=None, description="A list of filters that can be used when generating or scanning systems." ) @@ -680,19 +679,19 @@ class Organization(FidesModel): description="An inherited field from the FidesModel that is unused with an Organization.", ) controller: Optional[ContactDetails] = Field( - description=ContactDetails.__doc__, + default=None, description=ContactDetails.__doc__, ) data_protection_officer: Optional[ContactDetails] = Field( - description=ContactDetails.__doc__, + default=None, description=ContactDetails.__doc__, ) fidesctl_meta: Optional[OrganizationMetadata] = Field( - description=OrganizationMetadata.__doc__, + default=None, description=OrganizationMetadata.__doc__, ) representative: Optional[ContactDetails] = Field( - description=ContactDetails.__doc__, + default=None, description=ContactDetails.__doc__, ) security_policy: Optional[HttpUrl] = Field( - description="Am optional URL to the organization security policy." + default=None, description="Am optional URL to the organization security policy." ) @@ -769,7 +768,7 @@ class PrivacyDeclaration(BaseModel): """ name: Optional[str] = Field( - description="The name of the privacy declaration on the system.", + default=None, description="The name of the privacy declaration on the system.", ) data_categories: List[FidesKey] = Field( description="An array of data categories describing a system in a privacy declaration.", @@ -782,13 +781,13 @@ class PrivacyDeclaration(BaseModel): description="An array of data subjects describing a system in a privacy declaration.", ) dataset_references: Optional[List[FidesKey]] = Field( - description="Referenced Dataset fides keys used by the system.", + default=None, description="Referenced Dataset fides keys used by the system.", ) egress: Optional[List[FidesKey]] = Field( - description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied." + default=None, description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied." ) ingress: Optional[List[FidesKey]] = Field( - description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied." + default=None, description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied." ) features: List[str] = Field( default_factory=list, description="The features of processing personal data." @@ -798,34 +797,34 @@ class PrivacyDeclaration(BaseModel): default=True, ) legal_basis_for_processing: Optional[LegalBasisForProcessingEnum] = Field( - description="The legal basis under which personal data is processed for this purpose." + default=None, description="The legal basis under which personal data is processed for this purpose." ) impact_assessment_location: Optional[str] = Field( - description="Where the legitimate interest impact assessment is stored" + default=None, description="Where the legitimate interest impact assessment is stored" ) retention_period: Optional[str] = Field( - description="An optional string to describe the time period for which data is retained for this purpose." + default=None, description="An optional string to describe the time period for which data is retained for this purpose." ) processes_special_category_data: bool = Field( default=False, description="This system processes special category data", ) special_category_legal_basis: Optional[SpecialCategoryLegalBasisEnum] = Field( - description="The legal basis under which the special category data is processed.", + default=None, description="The legal basis under which the special category data is processed.", ) data_shared_with_third_parties: bool = Field( default=False, description="This system shares data with third parties for this purpose.", ) third_parties: Optional[str] = Field( - description="The types of third parties the data is shared with.", + default=None, description="The types of third parties the data is shared with.", ) shared_categories: List[str] = Field( default_factory=list, description="The categories of personal data that this system shares with third parties.", ) cookies: Optional[List[Cookies]] = Field( - description="Cookies associated with this data use to deliver services and functionality", + default=None, description="Cookies associated with this data use to deliver services and functionality", ) model_config = ConfigDict(from_attributes=True) @@ -838,13 +837,13 @@ class SystemMetadata(BaseModel): """ resource_id: Optional[str] = Field( - description="The external resource id for the system being modeled." + default=None, description="The external resource id for the system being modeled." ) endpoint_address: Optional[str] = Field( - description="The host of the external resource for the system being modeled." + default=None, description="The host of the external resource for the system being modeled." ) endpoint_port: Optional[str] = Field( - description="The port of the external resource for the system being modeled." + default=None, description="The port of the external resource for the system being modeled." ) @@ -874,7 +873,7 @@ class DataFlow(BaseModel): description=f"Specifies the resource model class for which the `fides_key` applies. May be any of {', '.join([member.value for member in FlowableResources])}.", ) data_categories: Optional[List[FidesKey]] = Field( - description="An array of data categories describing the data in transit.", + default=None, description="An array of data categories describing the data in transit.", ) @model_validator(skip_on_failure=True) @@ -914,16 +913,16 @@ class System(FidesModel): meta: Optional[Dict] = meta_field fidesctl_meta: Optional[SystemMetadata] = Field( - description=SystemMetadata.__doc__, + default=None, description=SystemMetadata.__doc__, ) system_type: str = Field( description="A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc.", ) egress: Optional[List[DataFlow]] = Field( - description="The resources to which the system sends data." + default=None, description="The resources to which the system sends data." ) ingress: Optional[List[DataFlow]] = Field( - description="The resources from which the system receives data." + default=None, description="The resources from which the system receives data." ) privacy_declarations: List[PrivacyDeclaration] = Field( description=PrivacyDeclaration.__doc__, @@ -933,13 +932,13 @@ class System(FidesModel): description="An optional value to identify the owning department or group of the system within your organization", ) vendor_id: Optional[str] = Field( - description="The unique identifier for the vendor that's associated with this system." + default=None, description="The unique identifier for the vendor that's associated with this system." ) previous_vendor_id: Optional[str] = Field( - description="If specified, the unique identifier for the vendor that was previously associated with this system." + default=None, description="If specified, the unique identifier for the vendor that was previously associated with this system." ) vendor_deleted_date: Optional[datetime] = Field( - description="The deleted date of the vendor that's associated with this system." + default=None, description="The deleted date of the vendor that's associated with this system." ) dataset_references: List[FidesKey] = Field( default_factory=list, @@ -954,7 +953,7 @@ class System(FidesModel): description="This toggle indicates whether the system is exempt from privacy regulation if they do process personal data.", ) reason_for_exemption: Optional[str] = Field( - description="The reason that the system is exempt from privacy regulation." + default=None, description="The reason that the system is exempt from privacy regulation." ) uses_profiling: bool = Field( default=False, @@ -977,35 +976,35 @@ class System(FidesModel): description="Whether this system requires data protection impact assessments.", ) dpa_location: Optional[str] = Field( - description="Location where the DPAs or DIPAs can be found." + default=None, description="Location where the DPAs or DIPAs can be found." ) dpa_progress: Optional[str] = Field( - description="The optional status of a Data Protection Impact Assessment" + default=None, description="The optional status of a Data Protection Impact Assessment" ) privacy_policy: Optional[AnyUrl] = Field( - description="A URL that points to the system's publicly accessible privacy policy." + default=None, description="A URL that points to the system's publicly accessible privacy policy." ) legal_name: Optional[str] = Field( - description="The legal name for the business represented by the system." + default=None, description="The legal name for the business represented by the system." ) legal_address: Optional[str] = Field( - description="The legal address for the business represented by the system." + default=None, description="The legal address for the business represented by the system." ) responsibility: List[DataResponsibilityTitle] = Field( default_factory=list, description=DataResponsibilityTitle.__doc__, ) dpo: Optional[str] = Field( - description="The official privacy contact address or DPO." + default=None, description="The official privacy contact address or DPO." ) joint_controller_info: Optional[str] = Field( - description="The party or parties that share the responsibility for processing personal data." + default=None, description="The party or parties that share the responsibility for processing personal data." ) data_security_practices: Optional[str] = Field( - description="The data security practices employed by this system." + default=None, description="The data security practices employed by this system." ) cookie_max_age_seconds: Optional[int] = Field( - description="The maximum storage duration, in seconds, for cookies used by this system." + default=None, description="The maximum storage duration, in seconds, for cookies used by this system." ) uses_cookies: bool = Field( default=False, description="Whether this system uses cookie storage." @@ -1019,10 +1018,10 @@ class System(FidesModel): description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.", ) legitimate_interest_disclosure_url: Optional[AnyUrl] = Field( - description="A URL that points to the system's publicly accessible legitimate interest disclosure." + default=None, description="A URL that points to the system's publicly accessible legitimate interest disclosure." ) cookies: Optional[List[Cookies]] = Field( - description="System-level cookies unassociated with a data use to deliver services and functionality", + default=None, description="System-level cookies unassociated with a data use to deliver services and functionality", ) _sort_privacy_declarations: classmethod = validator( From af4aef3d7847850f82b14df6b67ba472a88b41d2 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 15:18:53 -0500 Subject: [PATCH 11/49] First attempt at replacing deprecated FidesVersion.__get_validators__ with __get_pydantic_core_schema__ --- src/fideslang/validation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 97edfae7..48b2f389 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -7,6 +7,7 @@ from packaging.version import Version from pydantic import StringConstraints +from pydantic_core import core_schema class FidesValidationError(ValueError): @@ -17,12 +18,14 @@ class FidesVersion(Version): """Validate strings as proper semantic versions.""" @classmethod - def __get_validators__(cls) -> Generator: - yield cls.validate + def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: + return core_schema.with_info_before_validator_function( + cls.validate, handler(date), field_name=handler.field_name + ) @classmethod def validate(cls, value: str) -> Version: - """Validates that the provided string is a valid Semantic Version.""" + """Validates that the provided string is a valid Seman. tic Version.""" return Version(value) From 9b1ce39da130ea69ced5f0cb92b7e7785e90bf0f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 15:19:28 -0500 Subject: [PATCH 12/49] Add required field mode="after" to model_validator. --- src/fideslang/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 3c59558d..380ebde8 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -275,7 +275,7 @@ class DataSubjectRights(BaseModel): default=None, description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) - @model_validator() + @model_validator(mode="after") @classmethod def include_exclude_has_values(cls, values: Dict) -> Dict: """ @@ -433,7 +433,7 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: ) return meta_values - @model_validator() + @model_validator(mode="after") @classmethod def validate_object_fields( # type: ignore cls, @@ -876,7 +876,7 @@ class DataFlow(BaseModel): default=None, description="An array of data categories describing the data in transit.", ) - @model_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True, mode="after") @classmethod def user_special_case(cls, values: Dict) -> Dict: """ @@ -1028,7 +1028,7 @@ class System(FidesModel): "privacy_declarations", allow_reuse=True )(sort_list_objects_by_name) - @model_validator() + @model_validator(mode="after") @classmethod def privacy_declarations_reference_data_flows( cls, From 9531c9a436e5c728c2426a0e532d61e1f9e74f31 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 15:26:09 -0500 Subject: [PATCH 13/49] =?UTF-8?q?Remove=20skip=5Fon=5Ffailure=3DTrue=20fro?= =?UTF-8?q?m=20model=5Fvalidator,=20which=20is=20an=20unexpected=20keyword?= =?UTF-8?q?=20argument.=20I=20think=20skip=5Fon=5Ffailure=3Dtrue=20just=20?= =?UTF-8?q?didn=E2=80=99t=20call=20the=20root=5Fvalidator=20if=20other=20v?= =?UTF-8?q?alidators=20failed.=20=20I=20am=20going=20to=20remove=20this=20?= =?UTF-8?q?instance.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fideslang/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 380ebde8..02eee015 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -876,7 +876,7 @@ class DataFlow(BaseModel): default=None, description="An array of data categories describing the data in transit.", ) - @model_validator(skip_on_failure=True, mode="after") + @model_validator(mode="after") @classmethod def user_special_case(cls, values: Dict) -> Dict: """ From c49f7af206d9a250af8950c9f231f04b897a6da4 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 16:01:16 -0500 Subject: [PATCH 14/49] Redefine FidesKey using Annotated[Str, StringConstraints(pattern=...) --- src/fideslang/validation.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 48b2f389..1924fc80 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -4,6 +4,7 @@ import re from collections import Counter from typing import Dict, Generator, List, Optional, Pattern, Set, Tuple +from typing_extensions import Annotated from packaging.version import Version from pydantic import StringConstraints @@ -29,23 +30,7 @@ def validate(cls, value: str) -> Version: return Version(value) -class FidesKey(StringConstraints): - """ - A FidesKey type that creates a custom constrained string. - """ - - regex: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.<>-]+$") - - @classmethod # This overrides the default method to throw the custom FidesValidationError - def validate(cls, value: str) -> str: - """Throws ValueError if val is not a valid FidesKey""" - - if not cls.regex.match(value): - raise FidesValidationError( - f"FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: {value}" - ) - - return value +FidesKey = Annotated[str, StringConstraints(pattern="^[a-zA-Z0-9_.<>-]+$")] def sort_list_objects_by_name(values: List) -> List: @@ -83,7 +68,7 @@ def no_self_reference(value: FidesKey, values: Dict) -> FidesKey: i.e. DataCategory.parent_key != DataCategory.fides_key """ - fides_key = FidesKey.validate(values.get("fides_key", "")) + fides_key = FidesKey(values.get("fides_key", "")) if value == fides_key: raise FidesValidationError("FidesKey can not self-reference!") return value @@ -155,7 +140,7 @@ def matching_parent_key(parent_key: FidesKey, values: Dict) -> FidesKey: Confirm that the parent_key matches the parent parsed from the FidesKey. """ - fides_key = FidesKey.validate(values.get("fides_key", "")) + fides_key = FidesKey(values.get("fides_key", "")) split_fides_key = fides_key.split(".") # Check if it is a top-level resource From 48e3bbfba2297e91ce372181e319f4d813405ac7 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 16:03:06 -0500 Subject: [PATCH 15/49] Organization tests failed because name_field and description_field don't have default values of None, and are often called in a optional context. --- src/fideslang/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 02eee015..0648abeb 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -53,9 +53,9 @@ ) # Reusable Fields -name_field = Field(description="Human-Readable name for this resource.") +name_field = Field(default=None, description="Human-Readable name for this resource.") description_field = Field( - description="A detailed description of what this resource is." + default=None, description="A detailed description of what this resource is." ) meta_field = Field( default=None, From 07def94f3231c9ca21c2c7bd5d9cd28dba915033 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 16:55:45 -0500 Subject: [PATCH 16/49] Update all instances of model_validator(mode="after") to mode=before to better match what we had before, so the "values" are a dictionary and not an instance of an object. This is to avoid a lot of errors like "DataFlow" object is not subscriptable that I newly created. - Update reusable validators to be f"field_validators" instead of "validators". - Remove allow_reuse=True from all field_validators. This is not a supported keyword and can be removed entirely. Set validate=default on the fields that were previously setting always=True, as this keyword is also not supported for field validators. This mimics the behavior of always=True in v1. - Data type in these field validators is a ValidationInfo not a dictionary. The data can be accessed via values.data.get(""_ --- src/fideslang/models.py | 41 +++++++++++++++++++------------------ src/fideslang/validation.py | 30 +++++++++++++-------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 0648abeb..a66f0ca2 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -36,19 +36,19 @@ valid_data_type, ) -matching_parent_key_validator = validator("parent_key", allow_reuse=True, always=True)( +matching_parent_key_validator = field_validator("parent_key")( matching_parent_key ) -no_self_reference_validator = validator("parent_key", allow_reuse=True)( +no_self_reference_validator = field_validator("parent_key")( no_self_reference ) -has_versioning_if_default_validator = validator( - "is_default", allow_reuse=True, always=True +has_versioning_if_default_validator = field_validator( + "is_default" )(has_versioning_if_default) -deprecated_version_later_than_added_validator = validator( - "version_deprecated", allow_reuse=True +deprecated_version_later_than_added_validator = field_validator( + "version_deprecated", )(deprecated_version_later_than_added) -is_deprecated_if_replaced_validator = validator("replaced_by", allow_reuse=True)( +is_deprecated_if_replaced_validator = field_validator("replaced_by")( is_deprecated_if_replaced ) @@ -98,6 +98,7 @@ class DefaultModel(BaseModel): ) is_default: bool = Field( default=False, + validate_default=True, description="Denotes whether the resource is part of the default taxonomy or not.", ) @@ -244,7 +245,7 @@ class SpecialCategoryLegalBasisEnum(str, Enum): class DataCategory(FidesModel, DefaultModel): """The DataCategory resource model.""" - parent_key: Optional[FidesKey] = None + parent_key: Optional[FidesKey] = Field(default=None, validate_default=True) _matching_parent_key: classmethod = matching_parent_key_validator _no_self_reference: classmethod = no_self_reference_validator @@ -275,7 +276,7 @@ class DataSubjectRights(BaseModel): default=None, description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) - @model_validator(mode="after") + @model_validator(mode="before") @classmethod def include_exclude_has_values(cls, values: Dict) -> Dict: """ @@ -303,7 +304,7 @@ class DataSubject(FidesModel, DefaultModel): class DataUse(FidesModel, DefaultModel): """The DataUse resource model.""" - parent_key: Optional[FidesKey] = None + parent_key: Optional[FidesKey] = Field(default=None, validate_default=True) _matching_parent_key: classmethod = matching_parent_key_validator _no_self_reference: classmethod = no_self_reference_validator @@ -433,7 +434,7 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: ) return meta_values - @model_validator(mode="after") + @model_validator(mode="before") @classmethod def validate_object_fields( # type: ignore cls, @@ -515,10 +516,10 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): fides_meta: Optional[CollectionMeta] = None - _sort_fields: classmethod = validator("fields", allow_reuse=True)( + _sort_fields: classmethod = field_validator("fields")( sort_list_objects_by_name ) - _unique_items_in_list: classmethod = validator("fields", allow_reuse=True)( + _unique_items_in_list: classmethod = field_validator("fields")( unique_items_in_list ) @@ -577,10 +578,10 @@ class Dataset(FidesModel, FidesopsMetaBackwardsCompat): description="An array of objects that describe the Dataset's collections.", ) - _sort_collections: classmethod = validator("collections", allow_reuse=True)( + _sort_collections: classmethod = field_validator("collections")( sort_list_objects_by_name ) - _unique_items_in_list: classmethod = validator("collections", allow_reuse=True)( + _unique_items_in_list: classmethod = field_validator("collections")( unique_items_in_list ) @@ -754,7 +755,7 @@ class Policy(FidesModel): description=PolicyRule.__doc__, ) - _sort_rules: classmethod = validator("rules", allow_reuse=True)( + _sort_rules: classmethod = field_validator("rules")( sort_list_objects_by_name ) @@ -876,7 +877,7 @@ class DataFlow(BaseModel): default=None, description="An array of data categories describing the data in transit.", ) - @model_validator(mode="after") + @model_validator(mode="before") @classmethod def user_special_case(cls, values: Dict) -> Dict: """ @@ -1024,11 +1025,11 @@ class System(FidesModel): default=None, description="System-level cookies unassociated with a data use to deliver services and functionality", ) - _sort_privacy_declarations: classmethod = validator( - "privacy_declarations", allow_reuse=True + _sort_privacy_declarations: classmethod = field_validator( + "privacy_declarations" )(sort_list_objects_by_name) - @model_validator(mode="after") + @model_validator(mode="before") @classmethod def privacy_declarations_reference_data_flows( cls, diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 1924fc80..7a31ee45 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -7,7 +7,7 @@ from typing_extensions import Annotated from packaging.version import Version -from pydantic import StringConstraints +from pydantic import StringConstraints, ValidationInfo from pydantic_core import core_schema @@ -61,21 +61,21 @@ def unique_items_in_list(values: List) -> List: return values -def no_self_reference(value: FidesKey, values: Dict) -> FidesKey: +def no_self_reference(value: FidesKey, values: ValidationInfo) -> FidesKey: """ Check to make sure that the fides_key doesn't match other fides_key references within an object. i.e. DataCategory.parent_key != DataCategory.fides_key """ - fides_key = FidesKey(values.get("fides_key", "")) + fides_key = FidesKey(values.data.get("fides_key", "")) if value == fides_key: raise FidesValidationError("FidesKey can not self-reference!") return value def deprecated_version_later_than_added( - version_deprecated: Optional[FidesVersion], values: Dict + version_deprecated: Optional[FidesVersion], values: ValidationInfo ) -> Optional[FidesVersion]: """ Check to make sure that the deprecated version is later than the added version. @@ -87,19 +87,19 @@ def deprecated_version_later_than_added( if not version_deprecated: return None - if version_deprecated < values.get("version_added", Version("0")): + if version_deprecated < values.data.get("version_added", Version("0")): raise FidesValidationError( "Deprecated version number can't be earlier than version added!" ) - if version_deprecated == values.get("version_added", Version("0")): + if version_deprecated == values.data.get("version_added", Version("0")): raise FidesValidationError( "Deprecated version number can't be the same as the version added!" ) return version_deprecated -def has_versioning_if_default(is_default: bool, values: Dict) -> bool: +def has_versioning_if_default(is_default: bool, values: ValidationInfo) -> bool: """ Check to make sure that version fields are set for default items. """ @@ -107,15 +107,15 @@ def has_versioning_if_default(is_default: bool, values: Dict) -> bool: # If it's a default item, it at least needs a starting version if is_default: try: - assert values.get("version_added") + assert values.data.get("version_added") except AssertionError: raise FidesValidationError("Default items must have version information!") # If it's not default, it shouldn't have version info else: try: - assert not values.get("version_added") - assert not values.get("version_deprecated") - assert not values.get("replaced_by") + assert not values.data.get("version_added") + assert not values.data.get("version_deprecated") + assert not values.data.get("replaced_by") except AssertionError: raise FidesValidationError( "Non-default items can't have version information!" @@ -124,23 +124,23 @@ def has_versioning_if_default(is_default: bool, values: Dict) -> bool: return is_default -def is_deprecated_if_replaced(replaced_by: str, values: Dict) -> str: +def is_deprecated_if_replaced(replaced_by: str, values: ValidationInfo) -> str: """ Check to make sure that the item has been deprecated if there is a replacement. """ - if replaced_by and not values.get("version_deprecated"): + if replaced_by and not values.data.get("version_deprecated"): raise FidesValidationError("Cannot be replaced without deprecation!") return replaced_by -def matching_parent_key(parent_key: FidesKey, values: Dict) -> FidesKey: +def matching_parent_key(parent_key: FidesKey, values: ValidationInfo) -> FidesKey: """ Confirm that the parent_key matches the parent parsed from the FidesKey. """ - fides_key = FidesKey(values.get("fides_key", "")) + fides_key = FidesKey(values.data.get("fides_key", "")) split_fides_key = fides_key.split(".") # Check if it is a top-level resource From c1f407eec49cf075dc374fbf0eb70fa32be1aeae Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 17:53:39 -0500 Subject: [PATCH 17/49] Continued changes to get pytest tests/fideslang/test_models.py running. - Types are not coerced any longer. We were passing in an organization_fides_key of integer 1, but it no longer gets coerced to a string. It either needs to be passed in as a string, or we need to update FidesKey itself to coerce it. - Return all elements has other validation so it should be null if it's not a list field - Update the definitions of FidesKey and FidesCollectionKey to be subclassing StringConstraints to try to maintain original validation. Override __get_pydantic_core_schema__ - Due to other arrangements around when validation is running (mode=before or after) egress key/ingress key does not necessarily exist when validating privacy declaration egress and egress. --- src/fideslang/models.py | 17 ++++++++--- src/fideslang/validation.py | 30 ++++++++++++++++--- tests/conftest.py | 12 ++++---- tests/fideslang/test_models.py | 16 +++++------ tests/fideslang/test_parse.py | 2 +- tests/fideslang/test_validation.py | 46 +++++++++++++++--------------- 6 files changed, 77 insertions(+), 46 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index a66f0ca2..26babcc3 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -4,6 +4,7 @@ Contains all of the Fides resources modeled as Pydantic models. """ from __future__ import annotations +from pydantic_core import core_schema from datetime import datetime from enum import Enum @@ -375,7 +376,7 @@ class FidesMeta(BaseModel): default=None, description="Optionally specify the allowable field length. Fides will not generate values that exceed this size." ) return_all_elements: Optional[bool] = Field( - default=False, + default=None, description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False." ) read_only: Optional[bool] = Field( @@ -449,7 +450,7 @@ def validate_object_fields( # type: ignore field_name: str = values.get("name") # type: ignore if values.get("fides_meta"): - declared_data_type = values["fides_meta"].data_type + declared_data_type = values["fides_meta"].get("data_type") if fields and declared_data_type: data_type, _ = parse_data_type_string(declared_data_type) @@ -475,8 +476,16 @@ class FidesCollectionKey(StringConstraints): Dataset.Collection name where both dataset and collection names are valid FidesKeys """ + """Validate strings as proper semantic versions.""" + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: + return core_schema.with_info_before_validator_function( + cls.validate, handler(str), field_name=handler.field_name + ) + @classmethod - def validate(cls, value: str) -> str: + def validate(cls, value: str, _: Optional[ValidationInfo] = None) -> str: """ Overrides validation to check FidesCollectionKey format, and that both the dataset and collection names have the FidesKey format. @@ -1044,7 +1053,7 @@ def privacy_declarations_reference_data_flows( for direction in ["egress", "ingress"]: fides_keys = getattr(privacy_declaration, direction, None) if fides_keys is not None: - data_flows = values[direction] + data_flows = values.get(direction) system = values["fides_key"] assert ( data_flows is not None and len(data_flows) > 0 diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 7a31ee45..ed6782db 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -21,7 +21,7 @@ class FidesVersion(Version): @classmethod def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: return core_schema.with_info_before_validator_function( - cls.validate, handler(date), field_name=handler.field_name + cls.validate, handler(str), field_name=handler.field_name ) @classmethod @@ -30,7 +30,29 @@ def validate(cls, value: str) -> Version: return Version(value) -FidesKey = Annotated[str, StringConstraints(pattern="^[a-zA-Z0-9_.<>-]+$")] +class FidesKey(StringConstraints): + """ + A FidesKey type that creates a custom constrained string. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: + return core_schema.with_info_before_validator_function( + cls.validate, handler(str), field_name=handler.field_name + ) + + regex: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.<>-]+$") + + @classmethod # This overrides the default method to throw the custom FidesValidationError + def validate(cls, value: str, _: Optional[ValidationInfo] = None) -> str: + """Throws ValueError if val is not a valid FidesKey""" + + if not cls.regex.match(value): + raise FidesValidationError( + f"FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: {value}" + ) + + return value def sort_list_objects_by_name(values: List) -> List: @@ -68,7 +90,7 @@ def no_self_reference(value: FidesKey, values: ValidationInfo) -> FidesKey: i.e. DataCategory.parent_key != DataCategory.fides_key """ - fides_key = FidesKey(values.data.get("fides_key", "")) + fides_key = FidesKey.validate(values.data.get("fides_key", "")) if value == fides_key: raise FidesValidationError("FidesKey can not self-reference!") return value @@ -140,7 +162,7 @@ def matching_parent_key(parent_key: FidesKey, values: ValidationInfo) -> FidesKe Confirm that the parent_key matches the parent parsed from the FidesKey. """ - fides_key = FidesKey(values.data.get("fides_key", "")) + fides_key = FidesKey.validate(values.data.get("fides_key", "")) split_fides_key = fides_key.split(".") # Check if it is a top-level resource diff --git a/tests/conftest.py b/tests/conftest.py index 1d2b15ab..a290e199 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,14 +15,14 @@ def resources_dict(): """ resources_dict: Dict[str, Any] = { "data_category": models.DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user.custom", parent_key="user", name="Custom Data Category", description="Custom Data Category", ), "dataset": models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_sample_db_dataset", name="Sample DB Dataset", description="This is a Sample Database Dataset", @@ -52,13 +52,13 @@ def resources_dict(): ], ), "data_subject": models.DataSubject( - organization_fides_key=1, + organization_fides_key="1", fides_key="custom_subject", name="Custom Data Subject", description="Custom Data Subject", ), "data_use": models.DataUse( - organization_fides_key=1, + organization_fides_key="1", fides_key="custom_data_use", name="Custom Data Use", description="Custom Data Use", @@ -72,7 +72,7 @@ def resources_dict(): description="Test Organization", ), "policy": models.Policy( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_policy", name="Test Policy", version="1.3", @@ -86,7 +86,7 @@ def resources_dict(): data_subjects=models.PrivacyRule(matches="ANY", values=[]), ), "system": models.System( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_system", system_type="SYSTEM", name="Test System", diff --git a/tests/fideslang/test_models.py b/tests/fideslang/test_models.py index 249ef012..e4d0242b 100644 --- a/tests/fideslang/test_models.py +++ b/tests/fideslang/test_models.py @@ -104,7 +104,7 @@ def test_system_valid(self) -> None: ], meta={"some": "meta stuff"}, name="Test System", - organization_fides_key=1, + organization_fides_key="1", cookies=[{"name": "test_cookie"}], privacy_declarations=[ PrivacyDeclaration( @@ -154,7 +154,7 @@ def test_system_valid_nested_meta(self) -> None: }, }, name="Test System", - organization_fides_key=1, + organization_fides_key="1", privacy_declarations=[ PrivacyDeclaration( data_categories=[], @@ -189,7 +189,7 @@ def test_system_valid_no_meta(self) -> None: ], # purposefully omitting the `meta` property to ensure it's effectively optional name="Test System", - organization_fides_key=1, + organization_fides_key="1", privacy_declarations=[ PrivacyDeclaration( data_categories=[], @@ -211,7 +211,7 @@ def test_system_valid_no_egress_or_ingress(self) -> None: fides_key="test_system", meta={"some": "meta stuff"}, name="Test System", - organization_fides_key=1, + organization_fides_key="1", privacy_declarations=[ PrivacyDeclaration( data_categories=[], @@ -238,7 +238,7 @@ def test_system_no_egress(self) -> None: ], meta={"some": "meta stuff"}, name="Test System", - organization_fides_key=1, + organization_fides_key="1", privacy_declarations=[ PrivacyDeclaration( data_categories=[], @@ -267,7 +267,7 @@ def test_system_no_ingress(self) -> None: fides_key="test_system", meta={"some": "meta stuff"}, name="Test System", - organization_fides_key=1, + organization_fides_key="1", privacy_declarations=[ PrivacyDeclaration( data_categories=[], @@ -295,7 +295,7 @@ def test_system_user_ingress_valid(self) -> None: ], meta={"some": "meta stuff"}, name="Test System", - organization_fides_key=1, + organization_fides_key="1", privacy_declarations=[ PrivacyDeclaration( data_categories=[], @@ -312,7 +312,7 @@ def test_system_user_ingress_valid(self) -> None: def test_expanded_system(self): assert System( fides_key="test_system", - organization_fides_key=1, + organization_fides_key="1", tags=["some", "tags"], name="Exponential Interactive, Inc d/b/a VDX.tv", description="My system test", diff --git a/tests/fideslang/test_parse.py b/tests/fideslang/test_parse.py index b94e752b..af42aff9 100644 --- a/tests/fideslang/test_parse.py +++ b/tests/fideslang/test_parse.py @@ -7,7 +7,7 @@ @pytest.mark.unit def test_parse_manifest(): expected_result = models.DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="some_resource", name="Test resource 1", description="Test Description", diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 2131083f..594781e2 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -35,7 +35,7 @@ def test_default_no_versions_error(self, TaxonomyClass): """There should be version info for default items.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -47,7 +47,7 @@ def test_not_default_no_versions_error(self, TaxonomyClass): """There shouldn't be version info on a non-default item.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -59,7 +59,7 @@ def test_deprecated_when_added(self, TaxonomyClass): """Item can't be deprecated in a version earlier than it was added.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -73,7 +73,7 @@ def test_deprecated_after_added(self, TaxonomyClass): """Item can't be deprecated in a version earlier than it was added.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -102,7 +102,7 @@ def test_built_from_dict_with_empty_versions(self, TaxonomyClass) -> None: def test_built_with_empty_versions(self, TaxonomyClass) -> None: """Try building directly with explicit None values.""" TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -117,7 +117,7 @@ def test_deprecated_not_added(self, TaxonomyClass): """Can't be deprecated without being added in an earlier version.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -130,7 +130,7 @@ def test_replaced_not_deprecated(self, TaxonomyClass): """If the field is replaced, it must also be deprecated.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -143,7 +143,7 @@ def test_replaced_not_deprecated(self, TaxonomyClass): def test_replaced_and_deprecated(self, TaxonomyClass): """If the field is replaced, it must also be deprecated.""" assert TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -158,7 +158,7 @@ def test_version_error(self, TaxonomyClass): """Check that versions are validated.""" with pytest.raises(ValidationError): TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -170,7 +170,7 @@ def test_version_error(self, TaxonomyClass): def test_versions_valid(self, TaxonomyClass): """Check that versions are validated.""" assert TaxonomyClass( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -246,7 +246,7 @@ def test_dataset_duplicate_collections_error(): @pytest.mark.unit def test_top_level_resource(): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="Custom Test Data", description="Custom Test Data Category", @@ -258,7 +258,7 @@ def test_top_level_resource(): def test_fides_key_doesnt_match_stated_parent_key(): with pytest.raises(ValidationError): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", @@ -270,7 +270,7 @@ def test_fides_key_doesnt_match_stated_parent_key(): @pytest.mark.unit def test_fides_key_matches_stated_parent_key(): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user.account.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", @@ -283,7 +283,7 @@ def test_fides_key_matches_stated_parent_key(): def test_no_parent_key_but_fides_key_contains_parent_key(): with pytest.raises(ValidationError): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", @@ -294,7 +294,7 @@ def test_no_parent_key_but_fides_key_contains_parent_key(): @pytest.mark.unit def test_fides_key_with_carets(): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="", name="Example valid key with brackets", description="This key contains a <> which is valid", @@ -306,7 +306,7 @@ def test_fides_key_with_carets(): def test_invalid_chars_in_fides_key(): with pytest.raises(ValidationError): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="!", name="Example invalid key", description="This key contains a ! so it is invalid", @@ -317,7 +317,7 @@ def test_invalid_chars_in_fides_key(): @pytest.mark.unit def test_create_valid_data_category(): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", @@ -330,7 +330,7 @@ def test_create_valid_data_category(): def test_circular_dependency_data_category(): with pytest.raises(ValidationError): DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user", name="User Data", description="Test Data Category", @@ -342,7 +342,7 @@ def test_circular_dependency_data_category(): @pytest.mark.unit def test_create_valid_data_use(): DataUse( - organization_fides_key=1, + organization_fides_key="1", fides_key="provide.service", name="Provide the Product or Service", parent_key="provide", @@ -355,7 +355,7 @@ def test_create_valid_data_use(): def test_circular_dependency_data_use(): with pytest.raises(ValidationError): DataUse( - organization_fides_key=1, + organization_fides_key="1", fides_key="provide.service", name="Provide the Product or Service", description="Test Data Use", @@ -402,7 +402,7 @@ def test_invalid_matches_privacy_rule(): @pytest.mark.unit def test_valid_policy_rule(): assert PolicyRule( - organization_fides_key=1, + organization_fides_key="1", policyId=1, fides_key="test_policy", name="Test Policy", @@ -416,7 +416,7 @@ def test_valid_policy_rule(): @pytest.mark.unit def test_valid_policy(): Policy( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_policy", name="Test Policy", version="1.3", @@ -429,7 +429,7 @@ def test_valid_policy(): @pytest.mark.unit def test_create_valid_system(): System( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_system", system_type="SYSTEM", name="Test System", From 171307cbcbc40b2ee6e9cb08f6cc9df7964d976f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 17:55:38 -0500 Subject: [PATCH 18/49] Revert some of the defaults for Optional[bool] fields, making them None to match previous behavior. --- src/fideslang/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 26babcc3..1e8c2714 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -297,7 +297,7 @@ class DataSubject(FidesModel, DefaultModel): rights: Optional[DataSubjectRights] = Field(default=None, description=DataSubjectRights.__doc__) automated_decisions_or_profiling: Optional[bool] = Field( - default=False, + default=None, description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.", ) @@ -366,7 +366,7 @@ class FidesMeta(BaseModel): default=None, description="The type of the identity data that should be used to query this collection for a DSR." ) primary_key: Optional[bool] = Field( - default=False, + default=None, description="Whether the current field can be considered a primary key of the current collection" ) data_type: Optional[str] = Field( @@ -380,7 +380,7 @@ class FidesMeta(BaseModel): description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False." ) read_only: Optional[bool] = Field( - default=False, + default=None, description="Optionally specify if a field is read-only, meaning it can't be updated or deleted." ) From 69bc988de28379c2bf5a832e6d2f2bbfebb1b41d Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 23:13:43 -0500 Subject: [PATCH 19/49] Very shaky changes. FidesVersion is not always the type I expect it to be when validating. Similarly, likely not defining FidesKey and FidesCollectionKey in the proper ways. - Have FidesKey and FidesCollectionKey extend str - instead of StringConstraint - still defining a __get_pydantic_core_schema__ to override how it gets validated --- src/fideslang/models.py | 14 +++++++------- src/fideslang/validation.py | 29 ++++++++++++++++++++--------- tests/fideslang/test_validation.py | 5 +---- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 1e8c2714..462ff792 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -435,7 +435,7 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: ) return meta_values - @model_validator(mode="before") + @model_validator(mode="after") @classmethod def validate_object_fields( # type: ignore cls, @@ -445,12 +445,12 @@ def validate_object_fields( # type: ignore - If there are sub-fields specified, type should be either empty or 'object' - Additionally object fields cannot have data_categories. """ - fields = values.get("fields") + fields = values.fields declared_data_type = None - field_name: str = values.get("name") # type: ignore + field_name: str = values.name # type: ignore - if values.get("fides_meta"): - declared_data_type = values["fides_meta"].get("data_type") + if values.fides_meta: + declared_data_type = values.fides_meta.data_type if fields and declared_data_type: data_type, _ = parse_data_type_string(declared_data_type) @@ -459,7 +459,7 @@ def validate_object_fields( # type: ignore f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field." ) - if (fields or declared_data_type == "object") and values.get("data_categories"): + if (fields or declared_data_type == "object") and values.data_categories: raise ValueError( f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead" ) @@ -471,7 +471,7 @@ def validate_object_fields( # type: ignore DatasetField.update_forward_refs() -class FidesCollectionKey(StringConstraints): +class FidesCollectionKey(str): # TODO what is the best way to define this custom string type? """ Dataset.Collection name where both dataset and collection names are valid FidesKeys """ diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index ed6782db..89d61cd8 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -15,22 +15,22 @@ class FidesValidationError(ValueError): """Custom exception for when the pydantic ValidationError can't be used.""" -class FidesVersion(Version): +class FidesVersion(Version): # TODO what is the best way to define this class? """Validate strings as proper semantic versions.""" @classmethod def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: - return core_schema.with_info_before_validator_function( - cls.validate, handler(str), field_name=handler.field_name - ) + return core_schema.no_info_after_validator_function(cls.validate) @classmethod def validate(cls, value: str) -> Version: - """Validates that the provided string is a valid Seman. tic Version.""" - return Version(value) + """Validates that the provided string is a valid Semantic Version.""" + if isinstance(value, str): + return Version(value) + return value -class FidesKey(StringConstraints): +class FidesKey(str): # TODO - what is the best way to create this custom str type? """ A FidesKey type that creates a custom constrained string. """ @@ -46,6 +46,12 @@ def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema @classmethod # This overrides the default method to throw the custom FidesValidationError def validate(cls, value: str, _: Optional[ValidationInfo] = None) -> str: """Throws ValueError if val is not a valid FidesKey""" + # TODO previously FidesKey would coerce to a string if it could, but with Pydantic, + # that's not the pattern. How should we approach here? + try: + value = str(value) + except ValueError: + raise Exception("Cannot coerce to string") if not cls.regex.match(value): raise FidesValidationError( @@ -109,12 +115,17 @@ def deprecated_version_later_than_added( if not version_deprecated: return None - if version_deprecated < values.data.get("version_added", Version("0")): + version_added = values.data.get("version_added") + version_added = FidesVersion(version_added) if version_added else Version("0") + # Why is version_deprecated a string here and not already a FidesVersion? + version_deprecated = FidesVersion(version_deprecated) + + if version_deprecated < version_added: raise FidesValidationError( "Deprecated version number can't be earlier than version added!" ) - if version_deprecated == values.data.get("version_added", Version("0")): + if version_deprecated == version_added: raise FidesValidationError( "Deprecated version number can't be the same as the version added!" ) diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 594781e2..3ef4a571 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -453,9 +453,6 @@ def test_create_valid_system(): assert True - - - @pytest.mark.unit def test_fides_key_validate_bad_key(): with pytest.raises(FidesValidationError): @@ -683,7 +680,7 @@ def test_object_field_conflicting_types(self): fields=[DatasetField(name="nested_field")], ) assert ( - "The data type 'string' on field 'test_field' is not compatible with specified sub-fields." + "The data type 'string' on field 'test_field' is not compatible with" in str(exc) ) From d194ba6bce2d07e1c737621afcd40483e311ad61 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 14 Jun 2024 23:19:07 -0500 Subject: [PATCH 20/49] Update deprecated methods that won't be removed until v3 parse_obj -> model_validate update_forward_refs -> model_rebuild __fields_set__ -> model_fields_set construct -> model_construct --- src/fideslang/default_taxonomy/utils.py | 2 +- src/fideslang/gvl/__init__.py | 16 ++++++++-------- src/fideslang/models.py | 2 +- src/fideslang/parse.py | 4 ++-- src/fideslang/relationships.py | 2 +- src/fideslang/utils.py | 2 +- tests/fideslang/test_relationships.py | 4 ++-- tests/fideslang/test_validation.py | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/fideslang/default_taxonomy/utils.py b/src/fideslang/default_taxonomy/utils.py index f97cc94f..39133a6e 100644 --- a/src/fideslang/default_taxonomy/utils.py +++ b/src/fideslang/default_taxonomy/utils.py @@ -18,5 +18,5 @@ def default_factory(taxonomy_class: CustomType, **kwargs: Dict) -> CustomType: # This is the version where we started tracking from, so # we use it as the default starting point. kwargs["version_added"] = "2.0.0" # type: ignore[assignment] - item = taxonomy_class.parse_obj(kwargs) + item = taxonomy_class.model_validate(kwargs) return item diff --git a/src/fideslang/gvl/__init__.py b/src/fideslang/gvl/__init__.py index 2a298eab..ec1c7d0a 100644 --- a/src/fideslang/gvl/__init__.py +++ b/src/fideslang/gvl/__init__.py @@ -50,16 +50,16 @@ def _load_data() -> None: ) as mapping_file: data = load(mapping_file) for raw_purpose in data["purposes"].values(): - purpose = Purpose.parse_obj(raw_purpose) - mapped_purpose = MappedPurpose.parse_obj(raw_purpose) + purpose = Purpose.model_validate(raw_purpose) + mapped_purpose = MappedPurpose.model_validate(raw_purpose) GVL_PURPOSES[purpose.id] = purpose MAPPED_PURPOSES[mapped_purpose.id] = mapped_purpose for data_use in mapped_purpose.data_uses: MAPPED_PURPOSES_BY_DATA_USE[data_use] = mapped_purpose for raw_special_purpose in data["specialPurposes"].values(): - special_purpose = Purpose.parse_obj(raw_special_purpose) - mapped_special_purpose = MappedPurpose.parse_obj(raw_special_purpose) + special_purpose = Purpose.model_validate(raw_special_purpose) + mapped_special_purpose = MappedPurpose.model_validate(raw_special_purpose) GVL_SPECIAL_PURPOSES[special_purpose.id] = special_purpose MAPPED_SPECIAL_PURPOSES[mapped_special_purpose.id] = mapped_special_purpose for data_use in mapped_special_purpose.data_uses: @@ -71,12 +71,12 @@ def _load_data() -> None: feature_data = load(feature_mapping_file) for raw_feature in feature_data["features"].values(): - feature = Feature.parse_obj(raw_feature) + feature = Feature.model_validate(raw_feature) GVL_FEATURES[feature.id] = feature FEATURES_BY_NAME[feature.name] = feature for raw_special_feature in feature_data["specialFeatures"].values(): - special_feature = Feature.parse_obj(raw_special_feature) + special_feature = Feature.model_validate(raw_special_feature) GVL_SPECIAL_FEATURES[special_feature.id] = special_feature FEATURES_BY_NAME[special_feature.name] = special_feature @@ -86,8 +86,8 @@ def _load_data() -> None: data_category_data = load(data_category_mapping_file) for raw_data_category in data_category_data.values(): - data_category = GVLDataCategory.parse_obj(raw_data_category) - mapped_data_category = MappedDataCategory.parse_obj(raw_data_category) + data_category = GVLDataCategory.model_validate(raw_data_category) + mapped_data_category = MappedDataCategory.model_validate(raw_data_category) GVL_DATA_CATEGORIES[data_category.id] = data_category MAPPED_GVL_DATA_CATEGORIES[mapped_data_category.id] = mapped_data_category diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 462ff792..613404b9 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -468,7 +468,7 @@ def validate_object_fields( # type: ignore # this is required for the recursive reference in the pydantic model: -DatasetField.update_forward_refs() +DatasetField.model_rebuild() class FidesCollectionKey(str): # TODO what is the best way to define this custom string type? diff --git a/src/fideslang/parse.py b/src/fideslang/parse.py index 94462d94..9c1a25c5 100644 --- a/src/fideslang/parse.py +++ b/src/fideslang/parse.py @@ -19,7 +19,7 @@ def parse_dict( raise SystemExit(1) try: - parsed_manifest = model_map[resource_type].parse_obj(resource) + parsed_manifest = model_map[resource_type].model_validate(resource) except Exception as err: print( "Failed to parse {} from {}:\n{}".format( @@ -34,7 +34,7 @@ def load_manifests_into_taxonomy(raw_manifests: Dict[str, List[Dict]]) -> Taxono """ Parse the raw resource manifests into resource resources. """ - taxonomy = Taxonomy.parse_obj( + taxonomy = Taxonomy.model_validate( { resource_type: [ parse_dict(resource_type, resource) for resource in resource_list diff --git a/src/fideslang/relationships.py b/src/fideslang/relationships.py index b238a226..8840bbdd 100644 --- a/src/fideslang/relationships.py +++ b/src/fideslang/relationships.py @@ -75,7 +75,7 @@ def get_referenced_missing_keys(taxonomy: Taxonomy) -> Set[FidesKey]: """ referenced_keys: List[Set[FidesKey]] = [ find_referenced_fides_keys(resource) - for resource_type in taxonomy.__fields_set__ + for resource_type in taxonomy.model_fields_set for resource in getattr(taxonomy, resource_type) ] key_set: Set[FidesKey] = set( diff --git a/src/fideslang/utils.py b/src/fideslang/utils.py index 5b64dbcb..e2c490bc 100644 --- a/src/fideslang/utils.py +++ b/src/fideslang/utils.py @@ -16,7 +16,7 @@ def get_resource_by_fides_key( return { resource_type: resource - for resource_type in taxonomy.__fields_set__ + for resource_type in taxonomy.model_fields_set for resource in getattr(taxonomy, resource_type) if resource.fides_key == fides_key } or None diff --git a/tests/fideslang/test_relationships.py b/tests/fideslang/test_relationships.py index 2e37dfaa..bd0381a3 100644 --- a/tests/fideslang/test_relationships.py +++ b/tests/fideslang/test_relationships.py @@ -103,7 +103,7 @@ def test_find_referenced_fides_keys_1(self) -> None: assert referenced_keys == set(expected_referenced_key) def test_find_referenced_fides_keys_2(self) -> None: - test_system = System.construct( + test_system = System.model_construct( name="test_dc", fides_key="test_dc", description="test description", @@ -149,7 +149,7 @@ def test_get_referenced_missing_keys(self): ), ], system=[ - System.construct( + System.model_construct( name="test_system", fides_key="test_system", description="test description", diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 3ef4a571..580106e8 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -85,7 +85,7 @@ def test_deprecated_after_added(self, TaxonomyClass): @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_built_from_dict_with_empty_versions(self, TaxonomyClass) -> None: """Try building from a dictionary with explicit None values.""" - TaxonomyClass.parse_obj( + TaxonomyClass.model_validate( { "organization_fides_key": 1, "fides_key": "user", From 7f03b223fa5458bf057be7214b2b8c4352cf068f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:00:36 -0500 Subject: [PATCH 21/49] Update how FidesKey is to defined to use Annotated and a BeforeValidator. --- src/fideslang/models.py | 6 +-- src/fideslang/validation.py | 70 ++++++++++-------------------- tests/conftest.py | 8 ++-- tests/fideslang/test_manifests.py | 16 +++---- tests/fideslang/test_parse.py | 6 +-- tests/fideslang/test_validation.py | 11 +++-- 6 files changed, 47 insertions(+), 70 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 613404b9..63bab375 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -34,7 +34,7 @@ parse_data_type_string, sort_list_objects_by_name, unique_items_in_list, - valid_data_type, + valid_data_type, validate_fides_key, ) matching_parent_key_validator = field_validator("parent_key")( @@ -492,8 +492,8 @@ def validate(cls, value: str, _: Optional[ValidationInfo] = None) -> str: """ values = value.split(".") if len(values) == 2: - FidesKey.validate(values[0]) - FidesKey.validate(values[1]) + validate_fides_key(values[0]) + validate_fides_key(values[1]) return value raise ValueError( "FidesCollection must be specified in the form 'FidesKey.FidesKey'" diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 89d61cd8..f753c49d 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -7,59 +7,36 @@ from typing_extensions import Annotated from packaging.version import Version -from pydantic import StringConstraints, ValidationInfo -from pydantic_core import core_schema +from pydantic import StringConstraints, ValidationInfo, BeforeValidator + +FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" class FidesValidationError(ValueError): """Custom exception for when the pydantic ValidationError can't be used.""" -class FidesVersion(Version): # TODO what is the best way to define this class? +class FidesVersion(Version): """Validate strings as proper semantic versions.""" - @classmethod - def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: - return core_schema.no_info_after_validator_function(cls.validate) - @classmethod def validate(cls, value: str) -> Version: """Validates that the provided string is a valid Semantic Version.""" - if isinstance(value, str): - return Version(value) - return value + return Version(value) -class FidesKey(str): # TODO - what is the best way to create this custom str type? - """ - A FidesKey type that creates a custom constrained string. - """ +def validate_fides_key(value: str) -> str: + """Throws ValueError if val is not a valid FidesKey""" - @classmethod - def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: - return core_schema.with_info_before_validator_function( - cls.validate, handler(str), field_name=handler.field_name + regex: Pattern[str] = re.compile(FIDES_KEY_PATTERN) + if not regex.match(value): + raise FidesValidationError( + f"FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: {value}" ) + return value - regex: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.<>-]+$") - - @classmethod # This overrides the default method to throw the custom FidesValidationError - def validate(cls, value: str, _: Optional[ValidationInfo] = None) -> str: - """Throws ValueError if val is not a valid FidesKey""" - # TODO previously FidesKey would coerce to a string if it could, but with Pydantic, - # that's not the pattern. How should we approach here? - try: - value = str(value) - except ValueError: - raise Exception("Cannot coerce to string") - - if not cls.regex.match(value): - raise FidesValidationError( - f"FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: {value}" - ) - - return value +FidesKey = Annotated[str, BeforeValidator(validate_fides_key)] def sort_list_objects_by_name(values: List) -> List: """ @@ -96,15 +73,15 @@ def no_self_reference(value: FidesKey, values: ValidationInfo) -> FidesKey: i.e. DataCategory.parent_key != DataCategory.fides_key """ - fides_key = FidesKey.validate(values.data.get("fides_key", "")) + fides_key = validate_fides_key(values.data.get("fides_key", "")) if value == fides_key: raise FidesValidationError("FidesKey can not self-reference!") return value def deprecated_version_later_than_added( - version_deprecated: Optional[FidesVersion], values: ValidationInfo -) -> Optional[FidesVersion]: + version_deprecated: Optional[str], values: ValidationInfo +) -> Optional[str]: """ Check to make sure that the deprecated version is later than the added version. @@ -115,17 +92,18 @@ def deprecated_version_later_than_added( if not version_deprecated: return None - version_added = values.data.get("version_added") - version_added = FidesVersion(version_added) if version_added else Version("0") - # Why is version_deprecated a string here and not already a FidesVersion? - version_deprecated = FidesVersion(version_deprecated) + version_added: Optional[str] = values.data.get("version_added") + + # Convert into Versions + transformed_version_added = FidesVersion(version_added) if version_added else Version("0") + transformed_version_deprecated = FidesVersion(version_deprecated) - if version_deprecated < version_added: + if transformed_version_deprecated < transformed_version_added: raise FidesValidationError( "Deprecated version number can't be earlier than version added!" ) - if version_deprecated == version_added: + if transformed_version_deprecated == transformed_version_added: raise FidesValidationError( "Deprecated version number can't be the same as the version added!" ) @@ -173,7 +151,7 @@ def matching_parent_key(parent_key: FidesKey, values: ValidationInfo) -> FidesKe Confirm that the parent_key matches the parent parsed from the FidesKey. """ - fides_key = FidesKey.validate(values.data.get("fides_key", "")) + fides_key = validate_fides_key(values.data.get("fides_key", "")) split_fides_key = fides_key.split(".") # Check if it is a top-level resource diff --git a/tests/conftest.py b/tests/conftest.py index a290e199..8994e3dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -111,7 +111,7 @@ def test_manifests(): "dataset": [ { "name": "Test Dataset 1", - "organization_fides_key": 1, + "organization_fides_key": "1", "datasetType": {}, "datasetLocation": "somedb:3306", "description": "Test Dataset 1", @@ -122,7 +122,7 @@ def test_manifests(): "system": [ { "name": "Test System 1", - "organization_fides_key": 1, + "organization_fides_key": "1", "systemType": "mysql", "description": "Test System 1", "fides_key": "some_system", @@ -134,7 +134,7 @@ def test_manifests(): { "name": "Test Dataset 2", "description": "Test Dataset 2", - "organization_fides_key": 1, + "organization_fides_key": "1", "datasetType": {}, "datasetLocation": "somedb:3306", "fides_key": "another_dataset", @@ -144,7 +144,7 @@ def test_manifests(): "system": [ { "name": "Test System 2", - "organization_fides_key": 1, + "organization_fides_key": "1", "systemType": "mysql", "description": "Test System 2", "fides_key": "another_system", diff --git a/tests/fideslang/test_manifests.py b/tests/fideslang/test_manifests.py index 5310624c..9fab7e04 100644 --- a/tests/fideslang/test_manifests.py +++ b/tests/fideslang/test_manifests.py @@ -68,7 +68,7 @@ def test_union_manifests(test_manifests): "name": "Test Dataset 1", "description": "Test Dataset 1", "fides_key": "some_dataset", - "organization_fides_key": 1, + "organization_fides_key": "1", "datasetType": {}, "datasetLocation": "somedb:3306", "datasetTables": [], @@ -77,7 +77,7 @@ def test_union_manifests(test_manifests): "name": "Test Dataset 2", "description": "Test Dataset 2", "fides_key": "another_dataset", - "organization_fides_key": 1, + "organization_fides_key": "1", "datasetType": {}, "datasetLocation": "somedb:3306", "datasetTables": [], @@ -86,14 +86,14 @@ def test_union_manifests(test_manifests): "system": [ { "name": "Test System 1", - "organization_fides_key": 1, + "organization_fides_key": "1", "systemType": "mysql", "description": "Test System 1", "fides_key": "some_system", }, { "name": "Test System 2", - "organization_fides_key": 1, + "organization_fides_key": "1", "systemType": "mysql", "description": "Test System 2", "fides_key": "another_system", @@ -122,7 +122,7 @@ def test_ingest_manifests(ingestion_manifest_directory): assert sorted(actual_result["dataset"], key=lambda x: x["name"]) == [ { "name": "Test Dataset 1", - "organization_fides_key": 1, + "organization_fides_key": "1", "datasetType": {}, "datasetLocation": "somedb:3306", "description": "Test Dataset 1", @@ -132,7 +132,7 @@ def test_ingest_manifests(ingestion_manifest_directory): { "name": "Test Dataset 2", "description": "Test Dataset 2", - "organization_fides_key": 1, + "organization_fides_key": "1", "datasetType": {}, "datasetLocation": "somedb:3306", "fides_key": "another_dataset", @@ -142,14 +142,14 @@ def test_ingest_manifests(ingestion_manifest_directory): assert sorted(actual_result["system"], key=lambda x: x["name"]) == [ { "name": "Test System 1", - "organization_fides_key": 1, + "organization_fides_key": "1", "systemType": "mysql", "description": "Test System 1", "fides_key": "some_system", }, { "name": "Test System 2", - "organization_fides_key": 1, + "organization_fides_key": "1", "systemType": "mysql", "description": "Test System 2", "fides_key": "another_system", diff --git a/tests/fideslang/test_parse.py b/tests/fideslang/test_parse.py index af42aff9..d8e75171 100644 --- a/tests/fideslang/test_parse.py +++ b/tests/fideslang/test_parse.py @@ -13,7 +13,7 @@ def test_parse_manifest(): description="Test Description", ) test_dict = { - "organization_fides_key": 1, + "organization_fides_key": "1", "fides_key": "some_resource", "name": "Test resource 1", "description": "Test Description", @@ -26,7 +26,7 @@ def test_parse_manifest(): def test_parse_manifest_no_fides_key_validation_error(): with pytest.raises(SystemExit): test_dict = { - "organization_fides_key": 1, + "organization_fides_key": "1", "name": "Test resource 1", "description": "Test Description", } @@ -38,7 +38,7 @@ def test_parse_manifest_no_fides_key_validation_error(): def test_parse_manifest_resource_type_error(): with pytest.raises(SystemExit): test_dict = { - "organization_fides_key": 1, + "organization_fides_key": "1", "fides_key": "some_resource", "name": "Test resource 1", "description": "Test Description", diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 580106e8..c17d7d76 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -21,7 +21,7 @@ PrivacyRule, System, ) -from fideslang.validation import FidesKey, FidesValidationError, valid_data_type +from fideslang.validation import FidesKey, FidesValidationError, valid_data_type, validate_fides_key DEFAULT_TAXONOMY_CLASSES = [DataCategory, DataUse, DataSubject] @@ -87,7 +87,7 @@ def test_built_from_dict_with_empty_versions(self, TaxonomyClass) -> None: """Try building from a dictionary with explicit None values.""" TaxonomyClass.model_validate( { - "organization_fides_key": 1, + "organization_fides_key": "1", "fides_key": "user", "name": "Custom Test Data", "description": "Custom Test Data Category", @@ -156,7 +156,7 @@ def test_replaced_and_deprecated(self, TaxonomyClass): @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_version_error(self, TaxonomyClass): """Check that versions are validated.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -456,12 +456,11 @@ def test_create_valid_system(): @pytest.mark.unit def test_fides_key_validate_bad_key(): with pytest.raises(FidesValidationError): - FidesKey.validate("hi!") - + validate_fides_key("hi!") @pytest.mark.unit def test_fides_key_validate_good_key(): - FidesKey.validate("hello_test_file.txt") + validate_fides_key("hello_test_file.txt") @pytest.mark.unit From a0bc3ecec71f123aeecac2ae3eb2bbeae61a215e Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:04:08 -0500 Subject: [PATCH 22/49] Likewise, update definition of FidesCollectionKey be a custom type using Annotated. --- src/fideslang/models.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 63bab375..d6b6583a 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -4,7 +4,10 @@ Contains all of the Fides resources modeled as Pydantic models. """ from __future__ import annotations +from typing_extensions import Annotated + from pydantic_core import core_schema +from pydantic import StringConstraints, ValidationInfo, BeforeValidator from datetime import datetime from enum import Enum @@ -471,33 +474,22 @@ def validate_object_fields( # type: ignore DatasetField.model_rebuild() -class FidesCollectionKey(str): # TODO what is the best way to define this custom string type? +def validate_fides_collection_key(value: str) -> str: """ - Dataset.Collection name where both dataset and collection names are valid FidesKeys + Overrides validation to check FidesCollectionKey format, and that both the dataset + and collection names have the FidesKey format. """ + values = value.split(".") + if len(values) == 2: + validate_fides_key(values[0]) + validate_fides_key(values[1]) + return value + raise ValueError( + "FidesCollection must be specified in the form 'FidesKey.FidesKey'" + ) - """Validate strings as proper semantic versions.""" - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler) -> core_schema.CoreSchema: - return core_schema.with_info_before_validator_function( - cls.validate, handler(str), field_name=handler.field_name - ) - @classmethod - def validate(cls, value: str, _: Optional[ValidationInfo] = None) -> str: - """ - Overrides validation to check FidesCollectionKey format, and that both the dataset - and collection names have the FidesKey format. - """ - values = value.split(".") - if len(values) == 2: - validate_fides_key(values[0]) - validate_fides_key(values[1]) - return value - raise ValueError( - "FidesCollection must be specified in the form 'FidesKey.FidesKey'" - ) +FidesCollectionKey = Annotated[str, BeforeValidator(validate_fides_collection_key)] class CollectionMeta(BaseModel): From 21eb4b2abdbbbea45017bb4e4e5b1e34565c1bd9 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:06:54 -0500 Subject: [PATCH 23/49] Get rid of FidesVersion entirely - it's not being used as a pydantic type - just use Version directly. --- src/fideslang/models.py | 10 +++++----- src/fideslang/validation.py | 14 +++----------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index d6b6583a..2a0ec021 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing_extensions import Annotated +from packaging.version import Version from pydantic_core import core_schema from pydantic import StringConstraints, ValidationInfo, BeforeValidator @@ -28,7 +29,6 @@ from fideslang.validation import ( FidesKey, - FidesVersion, deprecated_version_later_than_added, has_versioning_if_default, is_deprecated_if_replaced, @@ -118,12 +118,12 @@ def validate_version_added( cls, version_added: Optional[str] ) -> Optional[str]: """ - Validate that the `version_added` field is a proper FidesVersion + Validate that the `version_added` field is a proper Version """ if not version_added: return None - FidesVersion.validate(version_added) + Version(version_added) return version_added @field_validator("version_deprecated") @@ -132,12 +132,12 @@ def validate_version_deprecated( cls, version_deprecated: Optional[str] ) -> Optional[str]: """ - Validate that the `version_deprecated` is a proper FidesVersion + Validate that the `version_deprecated` is a proper Version """ if not version_deprecated: return None - FidesVersion.validate(version_deprecated) + Version(version_deprecated) return version_deprecated diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index f753c49d..cfd8c3bf 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -16,15 +16,6 @@ class FidesValidationError(ValueError): """Custom exception for when the pydantic ValidationError can't be used.""" -class FidesVersion(Version): - """Validate strings as proper semantic versions.""" - - @classmethod - def validate(cls, value: str) -> Version: - """Validates that the provided string is a valid Semantic Version.""" - return Version(value) - - def validate_fides_key(value: str) -> str: """Throws ValueError if val is not a valid FidesKey""" @@ -38,6 +29,7 @@ def validate_fides_key(value: str) -> str: FidesKey = Annotated[str, BeforeValidator(validate_fides_key)] + def sort_list_objects_by_name(values: List) -> List: """ Sort objects in a list by their name. @@ -95,8 +87,8 @@ def deprecated_version_later_than_added( version_added: Optional[str] = values.data.get("version_added") # Convert into Versions - transformed_version_added = FidesVersion(version_added) if version_added else Version("0") - transformed_version_deprecated = FidesVersion(version_deprecated) + transformed_version_added: Version = Version(version_added) if version_added else Version("0") + transformed_version_deprecated: Version = Version(version_deprecated) if transformed_version_deprecated < transformed_version_added: raise FidesValidationError( From 3fa9cb77786f3fc7e67828ddd8f905826e46b0ff Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 14 Nov 2023 15:30:11 +0800 Subject: [PATCH 24/49] update requirements file, remove python 3.8 and add 3.12 to supported versions --- .github/workflows/pr_checks.yml | 4 ++-- Dockerfile | 2 +- noxfile.py | 5 +++-- requirements.txt | 5 ++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index d6719934..bad5ab3c 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -93,8 +93,8 @@ jobs: Pytest-Matrix: strategy: matrix: - python_version: ["3.8", "3.9", "3.10", "3.11"] - pydantic_version: ["1.8.2", "1.9.2", "1.10.9"] + python_version: ["3.9", "3.10", "3.11", "3.12"] + pydantic_version: ["2.2.1", "2.3.0", "2.4.2", "2.5.0"] pyyaml_version: ["5.4.1", "6.0"] runs-on: ubuntu-latest continue-on-error: true diff --git a/Dockerfile b/Dockerfile index 87c40171..9984e581 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bullseye as base +FROM python:3.10-slim-bullseye as base # Update pip in the base image since we'll use it everywhere RUN pip install -U pip diff --git a/noxfile.py b/noxfile.py index 2b0dd342..95bc8217 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,8 +3,9 @@ nox.options.sessions = [] nox.options.reuse_existing_virtualenvs = True -TESTED_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"] -TESTED_PYDANTIC_VERSIONS = ["1.8.2", "1.9.2", "1.10.9"] +# These should match what is in the `pr_checks.yml` file for CI runs +TESTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] +TESTED_PYDANTIC_VERSIONS = ["2.2.1", "2.3.0", "2.4.2", "2.5.0"] TESTED_PYYAML_VERSIONS = ["5.4.1", "6.0"] diff --git a/requirements.txt b/requirements.txt index 64279819..a9e06078 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -pydantic==2.7.1 +pydantic>=2.2.1,<=2.6.0 pyyaml>=5,<7 packaging>=20.0 -bump-pydantic==0.8.0 -eval_type_backport==0.2.0 \ No newline at end of file +bump-pydantic==0.8.0 \ No newline at end of file From abc66ec2eeded17634ef51cec349322719304f45 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:38:20 -0500 Subject: [PATCH 25/49] Add 2.7.1 to pydantic matrix - - Bump xenon and pre-commit in dev-requirements --- .github/workflows/pr_checks.yml | 4 ++-- dev-requirements.txt | 4 ++-- noxfile.py | 4 ++-- pyproject.toml | 3 +-- requirements.txt | 2 +- src/fideslang/validation.py | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index bad5ab3c..766e58aa 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -94,8 +94,8 @@ jobs: strategy: matrix: python_version: ["3.9", "3.10", "3.11", "3.12"] - pydantic_version: ["2.2.1", "2.3.0", "2.4.2", "2.5.0"] - pyyaml_version: ["5.4.1", "6.0"] + pydantic_version: ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] + pyyaml_version: ["6.0.1"] runs-on: ubuntu-latest continue-on-error: true steps: diff --git a/dev-requirements.txt b/dev-requirements.txt index f91fc28a..0d0d24b3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,11 +2,11 @@ black==23.3.0 mypy==1.4.0 nox>=2023 packaging>=22.0 -pre-commit==2.9.3 +pre-commit==3.7.1 pylint==2.10.0 pytest==7.3.1 pytest-cov==2.11.1 requests-mock==1.8.0 setuptools>=64.0.2 types-PyYAML -xenon==0.7.3 +xenon==0.9.1 diff --git a/noxfile.py b/noxfile.py index 95bc8217..39c96422 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,8 +5,8 @@ # These should match what is in the `pr_checks.yml` file for CI runs TESTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] -TESTED_PYDANTIC_VERSIONS = ["2.2.1", "2.3.0", "2.4.2", "2.5.0"] -TESTED_PYYAML_VERSIONS = ["5.4.1", "6.0"] +TESTED_PYDANTIC_VERSIONS = ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] +TESTED_PYYAML_VERSIONS = ["5.5.1", "6.0.1"] def install_requirements(session: nox.Session) -> None: diff --git a/pyproject.toml b/pyproject.toml index 47a26082..c9befed7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,13 +9,12 @@ name = "fideslang" description = "Fides Taxonomy Language" dynamic = ["dependencies", "version"] readme = "README.md" -requires-python = ">=3.8, <4" +requires-python = ">=3.9, <4" authors = [{ name = "Ethyca, Inc.", email = "fidesteam@ethyca.com" }] license = { text = "Apache License 2.0" } classifiers = [ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/requirements.txt b/requirements.txt index a9e06078..a7e50780 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pydantic>=2.2.1,<=2.6.0 +pydantic>=2.2.1,<=2.7.1 pyyaml>=5,<7 packaging>=20.0 bump-pydantic==0.8.0 \ No newline at end of file diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index cfd8c3bf..007e2383 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -3,7 +3,7 @@ """ import re from collections import Counter -from typing import Dict, Generator, List, Optional, Pattern, Set, Tuple +from typing import Dict, List, Optional, Pattern, Set, Tuple from typing_extensions import Annotated from packaging.version import Version From 51af7ddd155ad19c8287a6d5fabc41dfa6e620bc Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:41:49 -0500 Subject: [PATCH 26/49] Remove python 3.12 support --- .github/workflows/pr_checks.yml | 4 ++-- noxfile.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 766e58aa..bf7f7365 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -93,9 +93,9 @@ jobs: Pytest-Matrix: strategy: matrix: - python_version: ["3.9", "3.10", "3.11", "3.12"] + python_version: ["3.9", "3.10", "3.11"] pydantic_version: ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] - pyyaml_version: ["6.0.1"] + pyyaml_version: ["5.5.1", "6.0.1"] runs-on: ubuntu-latest continue-on-error: true steps: diff --git a/noxfile.py b/noxfile.py index 39c96422..62cfdfbf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,7 +4,7 @@ nox.options.reuse_existing_virtualenvs = True # These should match what is in the `pr_checks.yml` file for CI runs -TESTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] +TESTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] TESTED_PYDANTIC_VERSIONS = ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] TESTED_PYYAML_VERSIONS = ["5.5.1", "6.0.1"] From b35343fda067a32ffb32e764cfb560a467bd70cc Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:44:09 -0500 Subject: [PATCH 27/49] Use Python 3.9 in the base dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9984e581..5b0aa15c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-bullseye as base +FROM python:3.9-slim-bullseye as base # Update pip in the base image since we'll use it everywhere RUN pip install -U pip From 3fb3d438552374ad03df5689e65cb775cd8b6054 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 14:48:42 -0500 Subject: [PATCH 28/49] Uninstall bump-pydantic - no longer necessary after using it to start the bump from v1 to v2. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7e50780..cbf23a9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ pydantic>=2.2.1,<=2.7.1 pyyaml>=5,<7 packaging>=20.0 -bump-pydantic==0.8.0 \ No newline at end of file From 7a01971f01a389c0c247094f4ee2a393687296e0 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 15:01:51 -0500 Subject: [PATCH 29/49] Fix: Name field is not nullable in every location. Also, throw proper error if version is not a ValidVersion. --- src/fideslang/models.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 2a0ec021..4c3d37f6 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -6,8 +6,7 @@ from __future__ import annotations from typing_extensions import Annotated -from packaging.version import Version -from pydantic_core import core_schema +from packaging.version import InvalidVersion, Version from pydantic import StringConstraints, ValidationInfo, BeforeValidator from datetime import datetime @@ -37,7 +36,7 @@ parse_data_type_string, sort_list_objects_by_name, unique_items_in_list, - valid_data_type, validate_fides_key, + valid_data_type, validate_fides_key, FidesValidationError, ) matching_parent_key_validator = field_validator("parent_key")( @@ -57,7 +56,7 @@ ) # Reusable Fields -name_field = Field(default=None, description="Human-Readable name for this resource.") +name_field = Field(description="Human-Readable name for this resource.") description_field = Field( default=None, description="A detailed description of what this resource is." ) @@ -78,7 +77,7 @@ class FidesModel(BaseModel): description="Defines the Organization that this resource belongs to.", ) tags: Optional[List[str]] = None - name: Optional[str] = name_field + name: Optional[str] = Field(default=None, description="Human-Readable name for this resource.") description: Optional[str] = description_field model_config = ConfigDict(extra="ignore", from_attributes=True) @@ -123,7 +122,13 @@ def validate_version_added( if not version_added: return None - Version(version_added) + try: + Version(version_added) + except InvalidVersion: + raise FidesValidationError( + f"Field 'version_added' does not have a valid version: {version_added}" + ) + return version_added @field_validator("version_deprecated") @@ -137,7 +142,13 @@ def validate_version_deprecated( if not version_deprecated: return None - Version(version_deprecated) + try: + Version(version_deprecated) + except InvalidVersion: + raise FidesValidationError( + f"Field 'version_deprecated' does not have a valid version: {version_deprecated}" + ) + return version_deprecated From 221db2b060c433496f4d73c52d51b64cfc499e34 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 15:09:09 -0500 Subject: [PATCH 30/49] Linting --- src/fideslang/models.py | 10 ++++------ src/fideslang/validation.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 4c3d37f6..06aa8dbd 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -4,26 +4,24 @@ Contains all of the Fides resources modeled as Pydantic models. """ from __future__ import annotations +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional, Union from typing_extensions import Annotated from packaging.version import InvalidVersion, Version -from pydantic import StringConstraints, ValidationInfo, BeforeValidator -from datetime import datetime -from enum import Enum -from typing import Any, Dict, List, Optional, Union from pydantic import ( + BeforeValidator, field_validator, model_validator, ConfigDict, AnyUrl, BaseModel, - StringConstraints, Field, HttpUrl, PositiveInt, - validator, ) from fideslang.validation import ( diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 007e2383..f3152fc0 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -7,7 +7,7 @@ from typing_extensions import Annotated from packaging.version import Version -from pydantic import StringConstraints, ValidationInfo, BeforeValidator +from pydantic import BeforeValidator, ValidationInfo FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" From 747a4602eb0ca0f4779654ec4ded9bf7157d6b1a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 15:09:59 -0500 Subject: [PATCH 31/49] Formatting --- src/fideslang/models.py | 185 +++++++++++++++++------------ src/fideslang/validation.py | 6 +- tests/conftest.py | 1 + tests/fideslang/gvl/test_gvl.py | 14 +-- tests/fideslang/test_parse.py | 3 +- tests/fideslang/test_validation.py | 8 +- 6 files changed, 127 insertions(+), 90 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 06aa8dbd..e0890819 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -4,28 +4,28 @@ Contains all of the Fides resources modeled as Pydantic models. """ from __future__ import annotations + from datetime import datetime from enum import Enum from typing import Any, Dict, List, Optional, Union -from typing_extensions import Annotated from packaging.version import InvalidVersion, Version - - from pydantic import ( - BeforeValidator, - field_validator, - model_validator, - ConfigDict, AnyUrl, BaseModel, + BeforeValidator, + ConfigDict, Field, HttpUrl, PositiveInt, + field_validator, + model_validator, ) +from typing_extensions import Annotated from fideslang.validation import ( FidesKey, + FidesValidationError, deprecated_version_later_than_added, has_versioning_if_default, is_deprecated_if_replaced, @@ -34,18 +34,15 @@ parse_data_type_string, sort_list_objects_by_name, unique_items_in_list, - valid_data_type, validate_fides_key, FidesValidationError, + valid_data_type, + validate_fides_key, ) -matching_parent_key_validator = field_validator("parent_key")( - matching_parent_key +matching_parent_key_validator = field_validator("parent_key")(matching_parent_key) +no_self_reference_validator = field_validator("parent_key")(no_self_reference) +has_versioning_if_default_validator = field_validator("is_default")( + has_versioning_if_default ) -no_self_reference_validator = field_validator("parent_key")( - no_self_reference -) -has_versioning_if_default_validator = field_validator( - "is_default" -)(has_versioning_if_default) deprecated_version_later_than_added_validator = field_validator( "version_deprecated", )(deprecated_version_later_than_added) @@ -75,7 +72,9 @@ class FidesModel(BaseModel): description="Defines the Organization that this resource belongs to.", ) tags: Optional[List[str]] = None - name: Optional[str] = Field(default=None, description="Human-Readable name for this resource.") + name: Optional[str] = Field( + default=None, description="Human-Readable name for this resource." + ) description: Optional[str] = description_field model_config = ConfigDict(extra="ignore", from_attributes=True) @@ -111,9 +110,7 @@ class DefaultModel(BaseModel): @field_validator("version_added") @classmethod - def validate_version_added( - cls, version_added: Optional[str] - ) -> Optional[str]: + def validate_version_added(cls, version_added: Optional[str]) -> Optional[str]: """ Validate that the `version_added` field is a proper Version """ @@ -286,7 +283,8 @@ class DataSubjectRights(BaseModel): description="Defines the strategy used when mapping data rights to a data subject.", ) values: Optional[List[DataSubjectRightsEnum]] = Field( - default=None, description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", + default=None, + description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) @model_validator(mode="before") @@ -307,7 +305,9 @@ def include_exclude_has_values(cls, values: Dict) -> Dict: class DataSubject(FidesModel, DefaultModel): """The DataSubject resource model.""" - rights: Optional[DataSubjectRights] = Field(default=None, description=DataSubjectRights.__doc__) + rights: Optional[DataSubjectRights] = Field( + default=None, description=DataSubjectRights.__doc__ + ) automated_decisions_or_profiling: Optional[bool] = Field( default=None, description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.", @@ -348,7 +348,8 @@ class MyDatasetField(DatasetFieldBase): name: str = name_field description: Optional[str] = description_field data_categories: Optional[List[FidesKey]] = Field( - default=None, description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.", + default=None, + description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.", ) @@ -375,25 +376,28 @@ class FidesMeta(BaseModel): default=None, ) identity: Optional[str] = Field( - default=None, description="The type of the identity data that should be used to query this collection for a DSR." + default=None, + description="The type of the identity data that should be used to query this collection for a DSR.", ) primary_key: Optional[bool] = Field( default=None, - description="Whether the current field can be considered a primary key of the current collection" + description="Whether the current field can be considered a primary key of the current collection", ) data_type: Optional[str] = Field( - default=None, description="Optionally specify the data type. Fides will attempt to cast values to this type when querying." + default=None, + description="Optionally specify the data type. Fides will attempt to cast values to this type when querying.", ) length: Optional[PositiveInt] = Field( - default=None, description="Optionally specify the allowable field length. Fides will not generate values that exceed this size." + default=None, + description="Optionally specify the allowable field length. Fides will not generate values that exceed this size.", ) return_all_elements: Optional[bool] = Field( default=None, - description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False." + description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False.", ) read_only: Optional[bool] = Field( default=None, - description="Optionally specify if a field is read-only, meaning it can't be updated or deleted." + description="Optionally specify if a field is read-only, meaning it can't be updated or deleted.", ) @field_validator("data_type") @@ -428,7 +432,8 @@ class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat): fides_meta: Optional[FidesMeta] = None fields: Optional[List[DatasetField]] = Field( - default=None, description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", + default=None, + description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", ) @field_validator("fides_meta") @@ -518,7 +523,8 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): name: str = name_field description: Optional[str] = description_field data_categories: Optional[List[FidesKey]] = Field( - default=None, description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.", + default=None, + description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.", ) fields: List[DatasetField] = Field( description="An array of objects that describe the collection's fields.", @@ -526,12 +532,8 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): fides_meta: Optional[CollectionMeta] = None - _sort_fields: classmethod = field_validator("fields")( - sort_list_objects_by_name - ) - _unique_items_in_list: classmethod = field_validator("fields")( - unique_items_in_list - ) + _sort_fields: classmethod = field_validator("fields")(sort_list_objects_by_name) + _unique_items_in_list: classmethod = field_validator("fields")(unique_items_in_list) class ContactDetails(BaseModel): @@ -579,7 +581,8 @@ class Dataset(FidesModel, FidesopsMetaBackwardsCompat): meta: Optional[Dict] = meta_field data_categories: Optional[List[FidesKey]] = Field( - default=None, description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.", + default=None, + description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.", ) fides_meta: Optional[DatasetMetadata] = Field( description=DatasetMetadata.__doc__, default=None @@ -673,7 +676,8 @@ class OrganizationMetadata(BaseModel): """ resource_filters: Optional[List[ResourceFilter]] = Field( - default=None, description="A list of filters that can be used when generating or scanning systems." + default=None, + description="A list of filters that can be used when generating or scanning systems.", ) @@ -690,16 +694,20 @@ class Organization(FidesModel): description="An inherited field from the FidesModel that is unused with an Organization.", ) controller: Optional[ContactDetails] = Field( - default=None, description=ContactDetails.__doc__, + default=None, + description=ContactDetails.__doc__, ) data_protection_officer: Optional[ContactDetails] = Field( - default=None, description=ContactDetails.__doc__, + default=None, + description=ContactDetails.__doc__, ) fidesctl_meta: Optional[OrganizationMetadata] = Field( - default=None, description=OrganizationMetadata.__doc__, + default=None, + description=OrganizationMetadata.__doc__, ) representative: Optional[ContactDetails] = Field( - default=None, description=ContactDetails.__doc__, + default=None, + description=ContactDetails.__doc__, ) security_policy: Optional[HttpUrl] = Field( default=None, description="Am optional URL to the organization security policy." @@ -765,9 +773,7 @@ class Policy(FidesModel): description=PolicyRule.__doc__, ) - _sort_rules: classmethod = field_validator("rules")( - sort_list_objects_by_name - ) + _sort_rules: classmethod = field_validator("rules")(sort_list_objects_by_name) class PrivacyDeclaration(BaseModel): @@ -779,7 +785,8 @@ class PrivacyDeclaration(BaseModel): """ name: Optional[str] = Field( - default=None, description="The name of the privacy declaration on the system.", + default=None, + description="The name of the privacy declaration on the system.", ) data_categories: List[FidesKey] = Field( description="An array of data categories describing a system in a privacy declaration.", @@ -792,13 +799,16 @@ class PrivacyDeclaration(BaseModel): description="An array of data subjects describing a system in a privacy declaration.", ) dataset_references: Optional[List[FidesKey]] = Field( - default=None, description="Referenced Dataset fides keys used by the system.", + default=None, + description="Referenced Dataset fides keys used by the system.", ) egress: Optional[List[FidesKey]] = Field( - default=None, description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied." + default=None, + description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied.", ) ingress: Optional[List[FidesKey]] = Field( - default=None, description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied." + default=None, + description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied.", ) features: List[str] = Field( default_factory=list, description="The features of processing personal data." @@ -808,34 +818,40 @@ class PrivacyDeclaration(BaseModel): default=True, ) legal_basis_for_processing: Optional[LegalBasisForProcessingEnum] = Field( - default=None, description="The legal basis under which personal data is processed for this purpose." + default=None, + description="The legal basis under which personal data is processed for this purpose.", ) impact_assessment_location: Optional[str] = Field( - default=None, description="Where the legitimate interest impact assessment is stored" + default=None, + description="Where the legitimate interest impact assessment is stored", ) retention_period: Optional[str] = Field( - default=None, description="An optional string to describe the time period for which data is retained for this purpose." + default=None, + description="An optional string to describe the time period for which data is retained for this purpose.", ) processes_special_category_data: bool = Field( default=False, description="This system processes special category data", ) special_category_legal_basis: Optional[SpecialCategoryLegalBasisEnum] = Field( - default=None, description="The legal basis under which the special category data is processed.", + default=None, + description="The legal basis under which the special category data is processed.", ) data_shared_with_third_parties: bool = Field( default=False, description="This system shares data with third parties for this purpose.", ) third_parties: Optional[str] = Field( - default=None, description="The types of third parties the data is shared with.", + default=None, + description="The types of third parties the data is shared with.", ) shared_categories: List[str] = Field( default_factory=list, description="The categories of personal data that this system shares with third parties.", ) cookies: Optional[List[Cookies]] = Field( - default=None, description="Cookies associated with this data use to deliver services and functionality", + default=None, + description="Cookies associated with this data use to deliver services and functionality", ) model_config = ConfigDict(from_attributes=True) @@ -848,13 +864,16 @@ class SystemMetadata(BaseModel): """ resource_id: Optional[str] = Field( - default=None, description="The external resource id for the system being modeled." + default=None, + description="The external resource id for the system being modeled.", ) endpoint_address: Optional[str] = Field( - default=None, description="The host of the external resource for the system being modeled." + default=None, + description="The host of the external resource for the system being modeled.", ) endpoint_port: Optional[str] = Field( - default=None, description="The port of the external resource for the system being modeled." + default=None, + description="The port of the external resource for the system being modeled.", ) @@ -884,7 +903,8 @@ class DataFlow(BaseModel): description=f"Specifies the resource model class for which the `fides_key` applies. May be any of {', '.join([member.value for member in FlowableResources])}.", ) data_categories: Optional[List[FidesKey]] = Field( - default=None, description="An array of data categories describing the data in transit.", + default=None, + description="An array of data categories describing the data in transit.", ) @model_validator(mode="before") @@ -924,7 +944,8 @@ class System(FidesModel): meta: Optional[Dict] = meta_field fidesctl_meta: Optional[SystemMetadata] = Field( - default=None, description=SystemMetadata.__doc__, + default=None, + description=SystemMetadata.__doc__, ) system_type: str = Field( description="A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc.", @@ -943,13 +964,16 @@ class System(FidesModel): description="An optional value to identify the owning department or group of the system within your organization", ) vendor_id: Optional[str] = Field( - default=None, description="The unique identifier for the vendor that's associated with this system." + default=None, + description="The unique identifier for the vendor that's associated with this system.", ) previous_vendor_id: Optional[str] = Field( - default=None, description="If specified, the unique identifier for the vendor that was previously associated with this system." + default=None, + description="If specified, the unique identifier for the vendor that was previously associated with this system.", ) vendor_deleted_date: Optional[datetime] = Field( - default=None, description="The deleted date of the vendor that's associated with this system." + default=None, + description="The deleted date of the vendor that's associated with this system.", ) dataset_references: List[FidesKey] = Field( default_factory=list, @@ -964,7 +988,8 @@ class System(FidesModel): description="This toggle indicates whether the system is exempt from privacy regulation if they do process personal data.", ) reason_for_exemption: Optional[str] = Field( - default=None, description="The reason that the system is exempt from privacy regulation." + default=None, + description="The reason that the system is exempt from privacy regulation.", ) uses_profiling: bool = Field( default=False, @@ -990,16 +1015,20 @@ class System(FidesModel): default=None, description="Location where the DPAs or DIPAs can be found." ) dpa_progress: Optional[str] = Field( - default=None, description="The optional status of a Data Protection Impact Assessment" + default=None, + description="The optional status of a Data Protection Impact Assessment", ) privacy_policy: Optional[AnyUrl] = Field( - default=None, description="A URL that points to the system's publicly accessible privacy policy." + default=None, + description="A URL that points to the system's publicly accessible privacy policy.", ) legal_name: Optional[str] = Field( - default=None, description="The legal name for the business represented by the system." + default=None, + description="The legal name for the business represented by the system.", ) legal_address: Optional[str] = Field( - default=None, description="The legal address for the business represented by the system." + default=None, + description="The legal address for the business represented by the system.", ) responsibility: List[DataResponsibilityTitle] = Field( default_factory=list, @@ -1009,13 +1038,15 @@ class System(FidesModel): default=None, description="The official privacy contact address or DPO." ) joint_controller_info: Optional[str] = Field( - default=None, description="The party or parties that share the responsibility for processing personal data." + default=None, + description="The party or parties that share the responsibility for processing personal data.", ) data_security_practices: Optional[str] = Field( default=None, description="The data security practices employed by this system." ) cookie_max_age_seconds: Optional[int] = Field( - default=None, description="The maximum storage duration, in seconds, for cookies used by this system." + default=None, + description="The maximum storage duration, in seconds, for cookies used by this system.", ) uses_cookies: bool = Field( default=False, description="Whether this system uses cookie storage." @@ -1029,15 +1060,17 @@ class System(FidesModel): description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.", ) legitimate_interest_disclosure_url: Optional[AnyUrl] = Field( - default=None, description="A URL that points to the system's publicly accessible legitimate interest disclosure." + default=None, + description="A URL that points to the system's publicly accessible legitimate interest disclosure.", ) cookies: Optional[List[Cookies]] = Field( - default=None, description="System-level cookies unassociated with a data use to deliver services and functionality", + default=None, + description="System-level cookies unassociated with a data use to deliver services and functionality", ) - _sort_privacy_declarations: classmethod = field_validator( - "privacy_declarations" - )(sort_list_objects_by_name) + _sort_privacy_declarations: classmethod = field_validator("privacy_declarations")( + sort_list_objects_by_name + ) @model_validator(mode="before") @classmethod diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index f3152fc0..7c04d1ba 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -4,10 +4,10 @@ import re from collections import Counter from typing import Dict, List, Optional, Pattern, Set, Tuple -from typing_extensions import Annotated from packaging.version import Version from pydantic import BeforeValidator, ValidationInfo +from typing_extensions import Annotated FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" @@ -87,7 +87,9 @@ def deprecated_version_later_than_added( version_added: Optional[str] = values.data.get("version_added") # Convert into Versions - transformed_version_added: Version = Version(version_added) if version_added else Version("0") + transformed_version_added: Version = ( + Version(version_added) if version_added else Version("0") + ) transformed_version_deprecated: Version = Version(version_deprecated) if transformed_version_deprecated < transformed_version_added: diff --git a/tests/conftest.py b/tests/conftest.py index 8994e3dd..91e306c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import pytest import yaml + from fideslang import models diff --git a/tests/fideslang/gvl/test_gvl.py b/tests/fideslang/gvl/test_gvl.py index abda6fd2..633d13b9 100644 --- a/tests/fideslang/gvl/test_gvl.py +++ b/tests/fideslang/gvl/test_gvl.py @@ -67,19 +67,15 @@ def test_feature_id_to_feature_name(): assert feature_id_to_feature_name(feature_id=1001) is None - def test_data_category_id_to_data_categories(): - assert data_category_id_to_data_categories(1) == [ - "user.device.ip_address" - ] + assert data_category_id_to_data_categories(1) == ["user.device.ip_address"] # let's test one other data category just to be comprehensive assert data_category_id_to_data_categories(5) == [ - "user.account", - "user.unique_id", - "user.device" - ] - + "user.account", + "user.unique_id", + "user.device", + ] # assert invalid categories raise KeyErrors with pytest.raises(KeyError): diff --git a/tests/fideslang/test_parse.py b/tests/fideslang/test_parse.py index d8e75171..5c90cd74 100644 --- a/tests/fideslang/test_parse.py +++ b/tests/fideslang/test_parse.py @@ -1,7 +1,6 @@ import pytest -from fideslang import models -from fideslang import parse +from fideslang import models, parse @pytest.mark.unit diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index c17d7d76..e888b6e7 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -21,7 +21,12 @@ PrivacyRule, System, ) -from fideslang.validation import FidesKey, FidesValidationError, valid_data_type, validate_fides_key +from fideslang.validation import ( + FidesKey, + FidesValidationError, + valid_data_type, + validate_fides_key, +) DEFAULT_TAXONOMY_CLASSES = [DataCategory, DataUse, DataSubject] @@ -458,6 +463,7 @@ def test_fides_key_validate_bad_key(): with pytest.raises(FidesValidationError): validate_fides_key("hi!") + @pytest.mark.unit def test_fides_key_validate_good_key(): validate_fides_key("hello_test_file.txt") From 4f4a3028f3e3b8d63849c163bc1c2a2e9e73f451 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 16 Jun 2024 15:44:14 -0500 Subject: [PATCH 32/49] Fix pyyaml typo. --- .github/workflows/pr_checks.yml | 2 +- noxfile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index bf7f7365..5e0e0e8b 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -95,7 +95,7 @@ jobs: matrix: python_version: ["3.9", "3.10", "3.11"] pydantic_version: ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] - pyyaml_version: ["5.5.1", "6.0.1"] + pyyaml_version: ["5.4.1", "6.0.1"] runs-on: ubuntu-latest continue-on-error: true steps: diff --git a/noxfile.py b/noxfile.py index 62cfdfbf..3b28ac1f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,7 +6,7 @@ # These should match what is in the `pr_checks.yml` file for CI runs TESTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] TESTED_PYDANTIC_VERSIONS = ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] -TESTED_PYYAML_VERSIONS = ["5.5.1", "6.0.1"] +TESTED_PYYAML_VERSIONS = ["5.4.1", "6.0.1"] def install_requirements(session: nox.Session) -> None: From 922417a14868deb783677ba0f5108fae14f7038f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 17 Jun 2024 09:16:29 -0500 Subject: [PATCH 33/49] Fix validator order discovered when testing that new-style validators still work. - Place self-reference check before matching parent check when validating data categories and data uses because a self-reference would actually fail in the matching parent check, but the error is less clear there. - Also fix typo in existing error message --- src/fideslang/models.py | 5 +++-- src/fideslang/validation.py | 2 +- tests/fideslang/test_validation.py | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index e0890819..717bc741 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -257,8 +257,8 @@ class DataCategory(FidesModel, DefaultModel): parent_key: Optional[FidesKey] = Field(default=None, validate_default=True) - _matching_parent_key: classmethod = matching_parent_key_validator _no_self_reference: classmethod = no_self_reference_validator + _matching_parent_key: classmethod = matching_parent_key_validator class Cookies(BaseModel): @@ -318,8 +318,9 @@ class DataUse(FidesModel, DefaultModel): """The DataUse resource model.""" parent_key: Optional[FidesKey] = Field(default=None, validate_default=True) - _matching_parent_key: classmethod = matching_parent_key_validator + _no_self_reference: classmethod = no_self_reference_validator + _matching_parent_key: classmethod = matching_parent_key_validator # Dataset diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 7c04d1ba..80ea8d21 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -156,7 +156,7 @@ def matching_parent_key(parent_key: FidesKey, values: ValidationInfo) -> FidesKe parent_key_from_fides_key = ".".join(split_fides_key[:-1]) if parent_key_from_fides_key != parent_key: raise FidesValidationError( - "The parent_key ({0}) does match the parent parsed ({1}) from the fides_key ({2})!".format( + "The parent_key ({0}) does not match the parent parsed ({1}) from the fides_key ({2})!".format( parent_key, parent_key_from_fides_key, fides_key ) ) diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index e888b6e7..e497e101 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -261,7 +261,7 @@ def test_top_level_resource(): @pytest.mark.unit def test_fides_key_doesnt_match_stated_parent_key(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: DataCategory( organization_fides_key="1", fides_key="user.custom_test_data", @@ -269,6 +269,7 @@ def test_fides_key_doesnt_match_stated_parent_key(): description="Custom Test Data Category", parent_key="user.account", ) + assert 'The parent_key (user.account) does not match the parent parsed (user) from the fides_key (user.custom_test_data)!' in str(exc.value) assert DataCategory @@ -333,7 +334,7 @@ def test_create_valid_data_category(): @pytest.mark.unit def test_circular_dependency_data_category(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: DataCategory( organization_fides_key="1", fides_key="user", @@ -341,7 +342,7 @@ def test_circular_dependency_data_category(): description="Test Data Category", parent_key="user", ) - assert True + assert "FidesKey can not self-reference!" in str(exc.value) @pytest.mark.unit From 3656a145c9484c98444e9a1e6c4ecc8352028117 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 17 Jun 2024 12:21:31 -0500 Subject: [PATCH 34/49] Update existing tests to make sure they're still testing what we intended to test. - Assert exception thrown in test for duplicate collections errors. Fix integer dataset fides keys because this was failing first and preventing the duplicate entries found error from being thrown - Add more assertions on the error message itself to ensure we're testing the right point of failure --- src/fideslang/validation.py | 2 +- tests/__init__.py | 0 tests/conftest.py | 4 + tests/fideslang/test_models.py | 89 ++++++++++++++-- tests/fideslang/test_validation.py | 160 +++++++++++++++++++---------- 5 files changed, 192 insertions(+), 63 deletions(-) create mode 100644 tests/__init__.py diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 80ea8d21..d1894302 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -67,7 +67,7 @@ def no_self_reference(value: FidesKey, values: ValidationInfo) -> FidesKey: """ fides_key = validate_fides_key(values.data.get("fides_key", "")) if value == fides_key: - raise FidesValidationError("FidesKey can not self-reference!") + raise FidesValidationError("FidesKey cannot self-reference!") return value diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py index 91e306c9..2f177e9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,10 @@ from fideslang import models +def assert_error_message_includes(exception_info, error_excerpt): + assert error_excerpt in str(exception_info.value) + + @pytest.fixture(scope="session") def resources_dict(): """ diff --git a/tests/fideslang/test_models.py b/tests/fideslang/test_models.py index e4d0242b..7cf8ff9e 100644 --- a/tests/fideslang/test_models.py +++ b/tests/fideslang/test_models.py @@ -5,11 +5,13 @@ from fideslang import DataFlow, Dataset, Organization, PrivacyDeclaration, System from fideslang.models import ( ContactDetails, + Cookies, DataResponsibilityTitle, DatasetCollection, DatasetField, DataUse, ) +from tests.conftest import assert_error_message_includes pytestmark = mark.unit @@ -55,16 +57,25 @@ def test_dataflow_valid(self) -> None: ) def test_dataflow_user_fides_key_no_user_type(self) -> None: - with raises(ValueError): + with raises(ValueError) as exc: assert DataFlow(fides_key="user", type="system") + assert_error_message_includes( + exc, "The 'user' fides_key is required for, and requires, the type 'user'" + ) def test_dataflow_user_type_no_user_fides_key(self) -> None: - with raises(ValueError): + with raises(ValueError) as exc: assert DataFlow(fides_key="test_system_1", type="user") + assert_error_message_includes( + exc, "The 'user' fides_key is required for, and requires, the type 'user'" + ) def test_dataflow_invalid_type(self) -> None: - with raises(ValueError): + with raises(ValueError) as exc: assert DataFlow(fides_key="test_system_1", type="invalid") + assert_error_message_includes( + exc, "'type' must be one of dataset, system, user" + ) class TestPrivacyDeclaration: @@ -85,7 +96,7 @@ class TestSystem: # We need to update these tests to assert that the provided args are actually being set # as attributes on the System instance that's instantiated. def test_system_valid(self) -> None: - assert System( + system = System( description="Test Policy", egress=[ DataFlow( @@ -122,9 +133,53 @@ def test_system_valid(self) -> None: system_type="SYSTEM", tags=["some", "tags"], ) + assert system.name == "Test System" + assert system.fides_key == "test_system" + assert system.description == "Test Policy" + assert system.egress == [ + DataFlow( + fides_key="test_system_2", + type="system", + data_categories=[], + ) + ] + assert system.ingress == [ + DataFlow( + fides_key="test_system_3", + type="system", + data_categories=[], + ) + ] + assert system.meta == {"some": "meta stuff"} + assert system.organization_fides_key == "1" + assert system.cookies == [Cookies(name="test_cookie", path=None, domain=None)] + assert system.system_type == "SYSTEM" + assert system.tags == ["some", "tags"] + assert system.privacy_declarations == [ + PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="provide", + data_subjects=[], + dataset_references=None, + egress=["test_system_2"], + ingress=["test_system_3"], + features=[], + flexible_legal_basis_for_processing=True, + legal_basis_for_processing=None, + impact_assessment_location=None, + retention_period=None, + processes_special_category_data=False, + special_category_legal_basis=None, + data_shared_with_third_parties=False, + third_parties=None, + shared_categories=[], + cookies=[Cookies(name="test_cookie", path="/", domain="example.com")], + ) + ] def test_system_valid_nested_meta(self) -> None: - assert System( + system = System( description="Test Policy", egress=[ DataFlow( @@ -168,6 +223,18 @@ def test_system_valid_nested_meta(self) -> None: system_type="SYSTEM", tags=["some", "tags"], ) + assert system.meta == { + "some": "meta stuff", + "some": { + "nested": "meta stuff", + "more nested": "meta stuff", + }, + "some more": { + "doubly": { + "nested": "meta stuff", + } + }, + } def test_system_valid_no_meta(self) -> None: system = System( @@ -225,7 +292,7 @@ def test_system_valid_no_egress_or_ingress(self) -> None: ) def test_system_no_egress(self) -> None: - with raises(ValueError): + with raises(ValueError) as exc: assert System( description="Test Policy", fides_key="test_system", @@ -252,9 +319,13 @@ def test_system_no_egress(self) -> None: system_type="SYSTEM", tags=["some", "tags"], ) + assert_error_message_includes( + exc, + "PrivacyDeclaration 'declaration-name' defines egress with one or more resources and is applied to the System 'test_system', which does not itself define any egress.", + ) def test_system_no_ingress(self) -> None: - with raises(ValueError): + with raises(ValueError) as exc: assert System( description="Test Policy", egress=[ @@ -281,6 +352,10 @@ def test_system_no_ingress(self) -> None: system_type="SYSTEM", tags=["some", "tags"], ) + assert_error_message_includes( + exc, + "PrivacyDeclaration 'declaration-name' defines ingress with one or more resources and is applied to the System 'test_system', which does not itself define any ingress.", + ) def test_system_user_ingress_valid(self) -> None: assert System( diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index e497e101..9572c5db 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -22,11 +22,11 @@ System, ) from fideslang.validation import ( - FidesKey, FidesValidationError, valid_data_type, validate_fides_key, ) +from tests.conftest import assert_error_message_includes DEFAULT_TAXONOMY_CLASSES = [DataCategory, DataUse, DataSubject] @@ -38,7 +38,7 @@ class TestVersioning: @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_default_no_versions_error(self, TaxonomyClass): """There should be version info for default items.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -46,11 +46,14 @@ def test_default_no_versions_error(self, TaxonomyClass): description="Custom Test Data Category", is_default=True, ) + assert_error_message_includes( + exc, "Default items must have version information!" + ) @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_not_default_no_versions_error(self, TaxonomyClass): """There shouldn't be version info on a non-default item.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -58,11 +61,14 @@ def test_not_default_no_versions_error(self, TaxonomyClass): description="Custom Test Data Category", version_added="1.2.3", ) + assert_error_message_includes( + exc, "Non-default items can't have version information!" + ) @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_deprecated_when_added(self, TaxonomyClass): """Item can't be deprecated in a version earlier than it was added.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -72,11 +78,14 @@ def test_deprecated_when_added(self, TaxonomyClass): version_added="1.2", version_deprecated="1.2", ) + assert_error_message_includes( + exc, "Deprecated version number can't be the same as the version added!" + ) @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_deprecated_after_added(self, TaxonomyClass): """Item can't be deprecated in a version earlier than it was added.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -86,6 +95,9 @@ def test_deprecated_after_added(self, TaxonomyClass): version_added="1.2.3", version_deprecated="0.2", ) + assert_error_message_includes( + exc, "Deprecated version number can't be earlier than version added!" + ) @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_built_from_dict_with_empty_versions(self, TaxonomyClass) -> None: @@ -106,7 +118,7 @@ def test_built_from_dict_with_empty_versions(self, TaxonomyClass) -> None: @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_built_with_empty_versions(self, TaxonomyClass) -> None: """Try building directly with explicit None values.""" - TaxonomyClass( + tc = TaxonomyClass( organization_fides_key="1", fides_key="user", name="Custom Test Data", @@ -116,11 +128,13 @@ def test_built_with_empty_versions(self, TaxonomyClass) -> None: replaced_by=None, is_default=False, ) + assert tc.version_added is None + assert not tc.is_default @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_deprecated_not_added(self, TaxonomyClass): """Can't be deprecated without being added in an earlier version.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -129,11 +143,14 @@ def test_deprecated_not_added(self, TaxonomyClass): is_default=True, version_deprecated="0.2", ) + assert_error_message_includes( + exc, "Default items must have version information!" + ) @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_replaced_not_deprecated(self, TaxonomyClass): """If the field is replaced, it must also be deprecated.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: TaxonomyClass( organization_fides_key="1", fides_key="user", @@ -143,11 +160,12 @@ def test_replaced_not_deprecated(self, TaxonomyClass): version_added="1.2.3", replaced_by="some.field", ) + assert_error_message_includes(exc, "Cannot be replaced without deprecation!") @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_replaced_and_deprecated(self, TaxonomyClass): """If the field is replaced, it must also be deprecated.""" - assert TaxonomyClass( + tc = TaxonomyClass( organization_fides_key="1", fides_key="user", name="Custom Test Data", @@ -157,6 +175,9 @@ def test_replaced_and_deprecated(self, TaxonomyClass): version_deprecated="1.3", replaced_by="some.field", ) + assert tc.version_added == "1.2.3" + assert tc.version_deprecated == "1.3" + assert tc.replaced_by == "some.field" @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_version_error(self, TaxonomyClass): @@ -170,11 +191,14 @@ def test_version_error(self, TaxonomyClass): is_default=True, version_added="a.2.3", ) + assert_error_message_includes( + exc, "Field 'version_added' does not have a valid version" + ) @pytest.mark.parametrize("TaxonomyClass", DEFAULT_TAXONOMY_CLASSES) def test_versions_valid(self, TaxonomyClass): """Check that versions are validated.""" - assert TaxonomyClass( + tc = TaxonomyClass( organization_fides_key="1", fides_key="user", name="Custom Test Data", @@ -182,38 +206,40 @@ def test_versions_valid(self, TaxonomyClass): is_default=True, version_added="1.2.3", ) + assert tc.version_added == "1.2.3" @pytest.mark.unit def test_collections_duplicate_fields_error(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: DatasetCollection( name="foo", description="Fides Generated Description for Table: foo", data_categories=[], fields=[ DatasetField( - name=1, + name="1", description="Fides Generated Description for Column: 1", data_categories=[], ), DatasetField( - name=2, + name="2", description="Fides Generated Description for Column: 1", data_categories=[], ), DatasetField( - name=1, + name="1", description="Fides Generated Description for Column: 1", data_categories=[], ), ], ) + assert_error_message_includes(exc, "Duplicate entries found: [1]") @pytest.mark.unit def test_dataset_duplicate_collections_error(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: Dataset( name="ds", fides_key="ds", @@ -226,7 +252,7 @@ def test_dataset_duplicate_collections_error(): data_categories=[], fields=[ DatasetField( - name=1, + name="1", description="Fides Generated Description for Column: 1", data_categories=[], ), @@ -238,7 +264,7 @@ def test_dataset_duplicate_collections_error(): data_categories=[], fields=[ DatasetField( - name=4, + name="4", description="Fides Generated Description for Column: 4", data_categories=[], ), @@ -246,6 +272,7 @@ def test_dataset_duplicate_collections_error(): ), ], ) + assert_error_message_includes(exc, "Duplicate entries found: [foo]") @pytest.mark.unit @@ -269,67 +296,74 @@ def test_fides_key_doesnt_match_stated_parent_key(): description="Custom Test Data Category", parent_key="user.account", ) - assert 'The parent_key (user.account) does not match the parent parsed (user) from the fides_key (user.custom_test_data)!' in str(exc.value) - assert DataCategory + assert_error_message_includes( + exc, + "The parent_key (user.account) does not match the parent parsed (user) from the fides_key (user.custom_test_data)!", + ) @pytest.mark.unit def test_fides_key_matches_stated_parent_key(): - DataCategory( + dc = DataCategory( organization_fides_key="1", fides_key="user.account.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", parent_key="user.account", ) - assert DataCategory + assert dc.fides_key == "user.account.custom_test_data" + assert dc.parent_key == "user.account" @pytest.mark.unit def test_no_parent_key_but_fides_key_contains_parent_key(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: DataCategory( organization_fides_key="1", fides_key="user.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", ) - assert DataCategory + assert_error_message_includes( + exc, "The parent_key (None) does not match the parent parsed" + ) @pytest.mark.unit def test_fides_key_with_carets(): - DataCategory( + dc = DataCategory( organization_fides_key="1", fides_key="", name="Example valid key with brackets", description="This key contains a <> which is valid", ) - assert DataCategory + assert dc.fides_key == "" @pytest.mark.unit def test_invalid_chars_in_fides_key(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: DataCategory( organization_fides_key="1", fides_key="!", name="Example invalid key", description="This key contains a ! so it is invalid", ) - assert DataCategory + assert_error_message_includes( + exc, "FidesKeys must only contain alphanumeric characters" + ) @pytest.mark.unit def test_create_valid_data_category(): - DataCategory( + dc = DataCategory( organization_fides_key="1", fides_key="user.custom_test_data", name="Custom Test Data", description="Custom Test Data Category", parent_key="user", ) - assert DataCategory + assert dc.name == "Custom Test Data" @pytest.mark.unit @@ -342,24 +376,24 @@ def test_circular_dependency_data_category(): description="Test Data Category", parent_key="user", ) - assert "FidesKey can not self-reference!" in str(exc.value) + assert_error_message_includes(exc, "FidesKey cannot self-reference!") @pytest.mark.unit def test_create_valid_data_use(): - DataUse( + du = DataUse( organization_fides_key="1", fides_key="provide.service", name="Provide the Product or Service", parent_key="provide", description="Test Data Use", ) - assert True + assert du.name == "Provide the Product or Service" @pytest.mark.unit def test_circular_dependency_data_use(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: DataUse( organization_fides_key="1", fides_key="provide.service", @@ -367,12 +401,12 @@ def test_circular_dependency_data_use(): description="Test Data Use", parent_key="provide.service", ) - assert True + assert_error_message_includes(exc, "FidesKey cannot self-reference!") @pytest.mark.unit @pytest.mark.parametrize("fides_key", ["foo_bar", "foo-bar", "foo.bar", "foo_bar_8"]) -def test_fides_model_valid(fides_key: str): +def test_fides_model_fides_key_valid(fides_key: str): fides_key = FidesModel(fides_key=fides_key, name="Foo Bar") assert fides_key @@ -381,8 +415,11 @@ def test_fides_model_valid(fides_key: str): @pytest.mark.parametrize("fides_key", ["foo/bar", "foo%bar", "foo^bar"]) def test_fides_model_fides_key_invalid(fides_key): """Check for a bunch of different possible bad characters here.""" - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: FidesModel(fides_key=fides_key) + assert_error_message_includes( + exc, "FidesKeys must only contain alphanumeric characters" + ) @pytest.mark.unit @@ -393,16 +430,20 @@ def test_valid_privacy_rule(): @pytest.mark.unit def test_invalid_fides_key_privacy_rule(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: PrivacyRule(matches="ANY", values=["foo^bar"]) - assert True + assert_error_message_includes( + exc, "FidesKeys must only contain alphanumeric characters" + ) @pytest.mark.unit def test_invalid_matches_privacy_rule(): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: PrivacyRule(matches="AN", values=["foo_bar"]) - assert True + assert_error_message_includes( + exc, "Input should be 'ANY', 'ALL', 'NONE' or 'OTHER' " + ) @pytest.mark.unit @@ -429,7 +470,6 @@ def test_valid_policy(): description="Test Policy", rules=[], ) - assert True @pytest.mark.unit @@ -456,7 +496,6 @@ def test_create_valid_system(): ), ], ) - assert True @pytest.mark.unit @@ -473,27 +512,29 @@ def test_fides_key_validate_good_key(): @pytest.mark.unit class TestFidesDatasetReference: def test_dataset_invalid(self): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: FidesDatasetReference(dataset="bad fides key!", field="test_field") + assert_error_message_includes( + exc, "FidesKeys must only contain alphanumeric characters" + ) def test_invalid_direction(self): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: FidesDatasetReference( dataset="test_dataset", field="test_field", direction="backwards" ) + assert_error_message_includes(exc, "Input should be 'from' or 'to'") def valid_dataset_reference_to(self): ref = FidesDatasetReference( dataset="test_dataset", field="test_field", direction="to" ) - assert ref def valid_dataset_reference_from(self): ref = FidesDatasetReference( dataset="test_dataset", field="test_field", direction="from" ) - assert ref def valid_dataset_reference_no_direction(self): @@ -620,8 +661,8 @@ def test_valid_length(self): class TestValidateDatasetField: - def test_return_all_elements_not_string_field(self): - with pytest.raises(ValidationError): + def test_return_all_elements_not_array_field(self): + with pytest.raises(ValidationError) as exc: DatasetField( name="test_field", fides_meta=FidesMeta( @@ -634,6 +675,10 @@ def test_return_all_elements_not_string_field(self): read_only=None, ), ) + assert_error_message_includes( + exc, + "The 'return_all_elements' attribute can only be specified on array fields.", + ) def test_return_all_elements_on_array_field(self): assert DatasetField( @@ -665,8 +710,8 @@ def test_data_categories_at_object_level(self): ), fields=[DatasetField(name="nested_field")], ) - assert "Object field 'test_field' cannot have specified data_categories" in str( - exc + assert_error_message_includes( + exc, "Object field 'test_field' cannot have specified data_categories" ) def test_object_field_conflicting_types(self): @@ -685,9 +730,8 @@ def test_object_field_conflicting_types(self): ), fields=[DatasetField(name="nested_field")], ) - assert ( - "The data type 'string' on field 'test_field' is not compatible with" - in str(exc) + assert_error_message_includes( + exc, "The data type 'string' on field 'test_field' is not compatible with" ) def test_data_categories_on_nested_fields(self): @@ -707,14 +751,20 @@ def test_data_categories_on_nested_fields(self): class TestCollectionMeta: def test_invalid_collection_key(self): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: CollectionMeta(after=[FidesCollectionKey("test_key")]) + assert_error_message_includes( + exc, "FidesCollection must be specified in the form 'FidesKey.FidesKey'" + ) def test_collection_key_has_too_many_components(self): - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc: CollectionMeta( after=[FidesCollectionKey("test_dataset.test_collection.test_field")] ) + assert_error_message_includes( + exc, "FidesCollection must be specified in the form 'FidesKey.FidesKey'" + ) def test_valid_collection_key(self): CollectionMeta(after=[FidesCollectionKey("test_dataset.test_collection")]) From 5557236b5799b58b7ba3086ef422b3ab6ab8e672 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 17 Jun 2024 13:42:56 -0500 Subject: [PATCH 35/49] Update types. Where I'm using model_validator with after model, adjust so it's an instance method and is receiving an instance and returning the instance --- src/fideslang/models.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 717bc741..f5c7c826 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -7,7 +7,7 @@ from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Dict, List, Optional, Union from packaging.version import InvalidVersion, Version from pydantic import ( @@ -20,6 +20,7 @@ PositiveInt, field_validator, model_validator, + ValidationInfo, ) from typing_extensions import Annotated @@ -454,21 +455,20 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: return meta_values @model_validator(mode="after") - @classmethod - def validate_object_fields( # type: ignore - cls, - values: Dict[str, Any], - ) -> Optional[List["DatasetField"]]: + def validate_object_fields( + self, + _: ValidationInfo, + ) -> DatasetField: """Two validation checks for object fields: - If there are sub-fields specified, type should be either empty or 'object' - Additionally object fields cannot have data_categories. """ - fields = values.fields + fields = self.fields declared_data_type = None - field_name: str = values.name # type: ignore + field_name: str = self.name - if values.fides_meta: - declared_data_type = values.fides_meta.data_type + if self.fides_meta: + declared_data_type = self.fides_meta.data_type if fields and declared_data_type: data_type, _ = parse_data_type_string(declared_data_type) @@ -477,12 +477,11 @@ def validate_object_fields( # type: ignore f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field." ) - if (fields or declared_data_type == "object") and values.data_categories: + if (fields or declared_data_type == "object") and self.data_categories: raise ValueError( f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead" ) - - return values + return self # this is required for the recursive reference in the pydantic model: @@ -533,8 +532,8 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): fides_meta: Optional[CollectionMeta] = None - _sort_fields: classmethod = field_validator("fields")(sort_list_objects_by_name) - _unique_items_in_list: classmethod = field_validator("fields")(unique_items_in_list) + _sort_fields: classmethod = field_validator("fields")(sort_list_objects_by_name) # type: ignore[assignment] + _unique_items_in_list: classmethod = field_validator("fields")(unique_items_in_list) # type: ignore[assignment] class ContactDetails(BaseModel): @@ -592,10 +591,10 @@ class Dataset(FidesModel, FidesopsMetaBackwardsCompat): description="An array of objects that describe the Dataset's collections.", ) - _sort_collections: classmethod = field_validator("collections")( + _sort_collections: classmethod = field_validator("collections")( # type: ignore[assignment] sort_list_objects_by_name ) - _unique_items_in_list: classmethod = field_validator("collections")( + _unique_items_in_list: classmethod = field_validator("collections")( # type: ignore[assignment] unique_items_in_list ) @@ -774,7 +773,7 @@ class Policy(FidesModel): description=PolicyRule.__doc__, ) - _sort_rules: classmethod = field_validator("rules")(sort_list_objects_by_name) + _sort_rules: classmethod = field_validator("rules")(sort_list_objects_by_name) # type: ignore[assignment] class PrivacyDeclaration(BaseModel): @@ -1069,7 +1068,7 @@ class System(FidesModel): description="System-level cookies unassociated with a data use to deliver services and functionality", ) - _sort_privacy_declarations: classmethod = field_validator("privacy_declarations")( + _sort_privacy_declarations: classmethod = field_validator("privacy_declarations")( # type: ignore[assignment] sort_list_objects_by_name ) @@ -1078,7 +1077,7 @@ class System(FidesModel): def privacy_declarations_reference_data_flows( cls, values: Dict, - ) -> PrivacyDeclaration: + ) -> Dict: """ Any `PrivacyDeclaration`s which include `egress` and/or `ingress` fields must only reference the `fides_key`s of defined `DataFlow`s in said field(s). From eb37c7d8d31d04b080b0f7a844fc9aa95ba11365 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 17 Jun 2024 13:51:24 -0500 Subject: [PATCH 36/49] Certain versions have a space in this error message, some don't. Shorten the error message I'm checking to get the gist of the check. --- tests/fideslang/test_validation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 9572c5db..66f1df9c 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -441,9 +441,7 @@ def test_invalid_fides_key_privacy_rule(): def test_invalid_matches_privacy_rule(): with pytest.raises(ValidationError) as exc: PrivacyRule(matches="AN", values=["foo_bar"]) - assert_error_message_includes( - exc, "Input should be 'ANY', 'ALL', 'NONE' or 'OTHER' " - ) + assert_error_message_includes(exc, "Input should be 'ANY'") @pytest.mark.unit From 761fd61e4c8965a86c70df4f0f60dfc7e8b1995b Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 17 Jun 2024 13:55:11 -0500 Subject: [PATCH 37/49] Bump to Pydantic 2.3.0 as lowest version supported --- .github/workflows/pr_checks.yml | 2 +- noxfile.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 5e0e0e8b..54191aeb 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -94,7 +94,7 @@ jobs: strategy: matrix: python_version: ["3.9", "3.10", "3.11"] - pydantic_version: ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] + pydantic_version: ["2.3.0", "2.4.2", "2.5.3", "2.6.4", "2.7.1"] pyyaml_version: ["5.4.1", "6.0.1"] runs-on: ubuntu-latest continue-on-error: true diff --git a/noxfile.py b/noxfile.py index 3b28ac1f..2a4aae3f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,7 @@ # These should match what is in the `pr_checks.yml` file for CI runs TESTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] -TESTED_PYDANTIC_VERSIONS = ["2.2.1", "2.3.0", "2.4.2", "2.7.1"] +TESTED_PYDANTIC_VERSIONS = ["2.3.0", "2.4.2", "2.5.3", "2.6.4", "2.7.1"] TESTED_PYYAML_VERSIONS = ["5.4.1", "6.0.1"] diff --git a/requirements.txt b/requirements.txt index cbf23a9f..50bd6bb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -pydantic>=2.2.1,<=2.7.1 +pydantic>=2.3.0,<=2.7.1 pyyaml>=5,<7 packaging>=20.0 From b2cc053bc6afc6e8f965ac2e10ce1f9e7e40a8c8 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sun, 23 Jun 2024 10:55:58 -0500 Subject: [PATCH 38/49] Switch FidesKey to use an AfterValidator so strings are validated before running regex. --- src/fideslang/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index d1894302..95011efb 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Pattern, Set, Tuple from packaging.version import Version -from pydantic import BeforeValidator, ValidationInfo +from pydantic import AfterValidator, ValidationInfo from typing_extensions import Annotated FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" @@ -27,7 +27,7 @@ def validate_fides_key(value: str) -> str: return value -FidesKey = Annotated[str, BeforeValidator(validate_fides_key)] +FidesKey = Annotated[str, AfterValidator(validate_fides_key)] def sort_list_objects_by_name(values: List) -> List: From b37aa75d69f4765806ad666f162db6850ea8f467 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 24 Jun 2024 08:49:18 -0500 Subject: [PATCH 39/49] Convert remaining model validators in "before" mode to use after mode given recurring issues where sometimes the data that is validated is a dictionary but it can also be the instance itself. --- src/fideslang/models.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index f5c7c826..76fa0e7c 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -13,7 +13,7 @@ from pydantic import ( AnyUrl, BaseModel, - BeforeValidator, + AfterValidator, ConfigDict, Field, HttpUrl, @@ -288,19 +288,18 @@ class DataSubjectRights(BaseModel): description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) - @model_validator(mode="before") - @classmethod - def include_exclude_has_values(cls, values: Dict) -> Dict: + @model_validator(mode="after") + def include_exclude_has_values(self) -> "DataSubjectRights": """ Validate the if include or exclude is chosen, that at least one value is present. """ - strategy, rights = values.get("strategy"), values.get("values") + strategy, rights = self.strategy, self.values if strategy in ("INCLUDE", "EXCLUDE"): assert ( rights is not None ), f"If {strategy} is chosen, rights must also be listed." - return values + return self class DataSubject(FidesModel, DefaultModel): @@ -503,7 +502,7 @@ def validate_fides_collection_key(value: str) -> str: ) -FidesCollectionKey = Annotated[str, BeforeValidator(validate_fides_collection_key)] +FidesCollectionKey = Annotated[str, AfterValidator(validate_fides_collection_key)] class CollectionMeta(BaseModel): @@ -907,20 +906,19 @@ class DataFlow(BaseModel): description="An array of data categories describing the data in transit.", ) - @model_validator(mode="before") - @classmethod - def user_special_case(cls, values: Dict) -> Dict: + @model_validator(mode="after") + def user_special_case(self) -> "DataFlow": """ If either the `fides_key` or the `type` are set to "user", then the other must also be set to "user". """ - if values["fides_key"] == "user" or values["type"] == "user": + if self.fides_key == "user" or self.type == "user": assert ( - values["fides_key"] == "user" and values["type"] == "user" + self.fides_key == "user" and self.type == "user" ), "The 'user' fides_key is required for, and requires, the type 'user'" - return values + return self @field_validator("type") @classmethod @@ -1072,23 +1070,21 @@ class System(FidesModel): sort_list_objects_by_name ) - @model_validator(mode="before") - @classmethod + @model_validator(mode="after") def privacy_declarations_reference_data_flows( - cls, - values: Dict, - ) -> Dict: + self, + ) -> "System": """ Any `PrivacyDeclaration`s which include `egress` and/or `ingress` fields must only reference the `fides_key`s of defined `DataFlow`s in said field(s). """ - privacy_declarations = values.get("privacy_declarations") or [] + privacy_declarations = self.privacy_declarations or [] for privacy_declaration in privacy_declarations: for direction in ["egress", "ingress"]: fides_keys = getattr(privacy_declaration, direction, None) if fides_keys is not None: - data_flows = values.get(direction) - system = values["fides_key"] + data_flows = getattr(self, direction) + system = self.fides_key assert ( data_flows is not None and len(data_flows) > 0 ), f"PrivacyDeclaration '{privacy_declaration.name}' defines {direction} with one or more resources and is applied to the System '{system}', which does not itself define any {direction}." @@ -1098,7 +1094,7 @@ def privacy_declarations_reference_data_flows( data_flow.fides_key for data_flow in data_flows ], f"PrivacyDeclaration '{privacy_declaration.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource." - return values + return self model_config = ConfigDict(use_enum_values=True) From 27a5e15cffb786c56c59afb3c700e67099e7fef9 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 26 Jun 2024 09:03:49 -0500 Subject: [PATCH 40/49] For backwards compat, coerce SystemMetadata endpoint ports from integers to strings if applicable. --- src/fideslang/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 76fa0e7c..146453e6 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -875,6 +875,8 @@ class SystemMetadata(BaseModel): description="The port of the external resource for the system being modeled.", ) + model_config = ConfigDict(coerce_numbers_to_str=True) # For backwards compat of endpoint_port + class FlowableResources(str, Enum): """ From 63b3816acae5f9476d47de2046101f96359ff639 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 5 Jul 2024 10:15:44 -0500 Subject: [PATCH 41/49] Add AnyUrlString to Fideslang which returns a string instead of a Url when it is validated. Use this for System.privacy_policy and System.legitimate_interest_disclosure_url. --- src/fideslang/gvl/models.py | 5 +++-- src/fideslang/models.py | 20 ++++++++++------- src/fideslang/validation.py | 12 +++++++--- tests/fideslang/test_validation.py | 35 ++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/fideslang/gvl/models.py b/src/fideslang/gvl/models.py index 9f0155cf..def49320 100644 --- a/src/fideslang/gvl/models.py +++ b/src/fideslang/gvl/models.py @@ -1,6 +1,6 @@ from typing import List -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class Purpose(BaseModel): @@ -32,7 +32,8 @@ class MappedPurpose(Purpose): class Feature(BaseModel): - "Pydantic model for GVL feature records" + """Pydantic model for GVL feature records""" + id: int = Field(description="Official GVL feature ID or special feature ID") name: str = Field(description="Name of the GVL feature or special feature.") description: str = Field( diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 146453e6..2a76c942 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -7,24 +7,24 @@ from datetime import datetime from enum import Enum -from typing import Dict, List, Optional, Union +from typing import Annotated, Dict, List, Optional, Union from packaging.version import InvalidVersion, Version from pydantic import ( + AfterValidator, AnyUrl, BaseModel, - AfterValidator, ConfigDict, Field, HttpUrl, PositiveInt, + ValidationInfo, field_validator, model_validator, - ValidationInfo, ) -from typing_extensions import Annotated from fideslang.validation import ( + AnyUrlString, FidesKey, FidesValidationError, deprecated_version_later_than_added, @@ -77,7 +77,9 @@ class FidesModel(BaseModel): default=None, description="Human-Readable name for this resource." ) description: Optional[str] = description_field - model_config = ConfigDict(extra="ignore", from_attributes=True) + model_config = ConfigDict( + extra="ignore", from_attributes=True, coerce_numbers_to_str=True + ) class DefaultModel(BaseModel): @@ -875,7 +877,9 @@ class SystemMetadata(BaseModel): description="The port of the external resource for the system being modeled.", ) - model_config = ConfigDict(coerce_numbers_to_str=True) # For backwards compat of endpoint_port + model_config = ConfigDict( + coerce_numbers_to_str=True + ) # For backwards compat of endpoint_port class FlowableResources(str, Enum): @@ -1018,7 +1022,7 @@ class System(FidesModel): default=None, description="The optional status of a Data Protection Impact Assessment", ) - privacy_policy: Optional[AnyUrl] = Field( + privacy_policy: Optional[AnyUrlString] = Field( default=None, description="A URL that points to the system's publicly accessible privacy policy.", ) @@ -1059,7 +1063,7 @@ class System(FidesModel): default=False, description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.", ) - legitimate_interest_disclosure_url: Optional[AnyUrl] = Field( + legitimate_interest_disclosure_url: Optional[AnyUrlString] = Field( default=None, description="A URL that points to the system's publicly accessible legitimate interest disclosure.", ) diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index 95011efb..f215c6c1 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -3,11 +3,10 @@ """ import re from collections import Counter -from typing import Dict, List, Optional, Pattern, Set, Tuple +from typing import Annotated, Dict, List, Optional, Pattern, Set, Tuple from packaging.version import Version -from pydantic import AfterValidator, ValidationInfo -from typing_extensions import Annotated +from pydantic import AfterValidator, AnyUrl, ValidationInfo FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" @@ -204,3 +203,10 @@ def valid_data_type(data_type_str: Optional[str]) -> Optional[str]: raise ValueError(f"The data type {data_type_str} is not supported.") return data_type_str + + +def validate_path_of_url(value: AnyUrl) -> str: + return str(value) + + +AnyUrlString = Annotated[AnyUrl, AfterValidator(validate_path_of_url)] diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 66f1df9c..5b30501e 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -22,9 +22,11 @@ System, ) from fideslang.validation import ( + AnyUrlString, FidesValidationError, valid_data_type, validate_fides_key, + validate_path_of_url, ) from tests.conftest import assert_error_message_includes @@ -766,3 +768,36 @@ def test_collection_key_has_too_many_components(self): def test_valid_collection_key(self): CollectionMeta(after=[FidesCollectionKey("test_dataset.test_collection")]) + + +class TestAnyUrlString: + def test_valid_url(self): + assert AnyUrlString("https://www.example.com") + + def test_invalid_url(self): + with pytest.raises(ValidationError) as exc: + AnyUrlString("invalid_url") + + assert_error_message_includes(exc, "Input should be a valid URL") + + def test_validate_path_of_url(self): + assert ( + validate_path_of_url("https://www.example.com/") + == "https://www.example.com/" + ) + + def test_system_urls(self): + system = System( + description="Test Policy", + fides_key="test_system", + name="Test System", + organization_fides_key="1", + privacy_declarations=[], + system_type="SYSTEM", + privacy_policy="https://www.example.com", + ) + + # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString + assert ( + system.privacy_policy == "https://www.example.com/" + ) # Trailing slash is added by AnyUrl From c692bcf498c4436427a4c223da504cf174332f31 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 11 Jul 2024 15:41:44 -0500 Subject: [PATCH 42/49] Wrap usages of AnyUrlString with SerializeAsAny to suppress Pydantic serializer warnings. We are intentionally serializing these as strings. --- src/fideslang/models.py | 5 +++-- tests/fideslang/test_models.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 2a76c942..b70d0be8 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -18,6 +18,7 @@ Field, HttpUrl, PositiveInt, + SerializeAsAny, ValidationInfo, field_validator, model_validator, @@ -1022,7 +1023,7 @@ class System(FidesModel): default=None, description="The optional status of a Data Protection Impact Assessment", ) - privacy_policy: Optional[AnyUrlString] = Field( + privacy_policy: SerializeAsAny[Optional[AnyUrlString]] = Field( default=None, description="A URL that points to the system's publicly accessible privacy policy.", ) @@ -1063,7 +1064,7 @@ class System(FidesModel): default=False, description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.", ) - legitimate_interest_disclosure_url: Optional[AnyUrlString] = Field( + legitimate_interest_disclosure_url: SerializeAsAny[Optional[AnyUrlString]] = Field( default=None, description="A URL that points to the system's publicly accessible legitimate interest disclosure.", ) diff --git a/tests/fideslang/test_models.py b/tests/fideslang/test_models.py index 7cf8ff9e..ff96123f 100644 --- a/tests/fideslang/test_models.py +++ b/tests/fideslang/test_models.py @@ -385,7 +385,7 @@ def test_system_user_ingress_valid(self) -> None: ) def test_expanded_system(self): - assert System( + system = System( fides_key="test_system", organization_fides_key="1", tags=["some", "tags"], @@ -480,6 +480,7 @@ def test_expanded_system(self): } ], ) + print(f"dumped={system.model_dump()}") def test_flexible_legal_basis_default(self): pd = PrivacyDeclaration( From 797662a001e216a1898728759d7e6c296eeb1285 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 15 Jul 2024 09:39:22 -0500 Subject: [PATCH 43/49] Adjust mkdocks Dockerfile to use 3.9 instead of 3.8. --- mkdocs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/Dockerfile b/mkdocs/Dockerfile index e0e4e86d..5162caa9 100644 --- a/mkdocs/Dockerfile +++ b/mkdocs/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bullseye +FROM python:3.9-slim-bullseye # Install auxiliary software RUN apt-get update From 2184527dbdf46c408d8d6c045456507b38ca3f90 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 15 Jul 2024 15:20:03 -0500 Subject: [PATCH 44/49] Update behavior so AnyUrlSting removes trailing slash by default, as well as AnyHttpUrlString. --- src/fideslang/gvl/models.py | 2 +- src/fideslang/models.py | 1 - src/fideslang/validation.py | 13 +++++++++++-- tests/fideslang/test_validation.py | 23 ++++++++++++++++++++--- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/fideslang/gvl/models.py b/src/fideslang/gvl/models.py index def49320..1f599e4c 100644 --- a/src/fideslang/gvl/models.py +++ b/src/fideslang/gvl/models.py @@ -1,6 +1,6 @@ from typing import List -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field class Purpose(BaseModel): diff --git a/src/fideslang/models.py b/src/fideslang/models.py index b70d0be8..4c53debc 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -12,7 +12,6 @@ from packaging.version import InvalidVersion, Version from pydantic import ( AfterValidator, - AnyUrl, BaseModel, ConfigDict, Field, diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index f215c6c1..c1d00bd5 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -6,7 +6,7 @@ from typing import Annotated, Dict, List, Optional, Pattern, Set, Tuple from packaging.version import Version -from pydantic import AfterValidator, AnyUrl, ValidationInfo +from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, ValidationInfo FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" @@ -206,7 +206,16 @@ def valid_data_type(data_type_str: Optional[str]) -> Optional[str]: def validate_path_of_url(value: AnyUrl) -> str: - return str(value) + """Converts an AnyUrl to a string and removes trailing slash""" + return str(value).rstrip("/") AnyUrlString = Annotated[AnyUrl, AfterValidator(validate_path_of_url)] + + +def validate_path_of_http_url(value: AnyHttpUrl) -> str: + """Converts an AnyHttpUrl to a string and removes trailing slash""" + return str(value).rstrip("/") + + +AnyHttpUrlString = Annotated[AnyHttpUrl, AfterValidator(validate_path_of_http_url)] diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 5b30501e..4e575363 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -22,10 +22,12 @@ System, ) from fideslang.validation import ( + AnyHttpUrlString, AnyUrlString, FidesValidationError, valid_data_type, validate_fides_key, + validate_path_of_http_url, validate_path_of_url, ) from tests.conftest import assert_error_message_includes @@ -783,7 +785,7 @@ def test_invalid_url(self): def test_validate_path_of_url(self): assert ( validate_path_of_url("https://www.example.com/") - == "https://www.example.com/" + == "https://www.example.com" ) def test_system_urls(self): @@ -798,6 +800,21 @@ def test_system_urls(self): ) # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString + assert system.privacy_policy == "https://www.example.com" + + +class TestAnyHttpUrlString: + def test_valid_url(self): + assert AnyHttpUrlString("https://www.example.com") + + def test_invalid_url(self): + with pytest.raises(ValidationError) as exc: + AnyHttpUrlString("invalid_url") + + assert_error_message_includes(exc, "Input should be a valid URL") + + def test_validate_path_of_url(self): assert ( - system.privacy_policy == "https://www.example.com/" - ) # Trailing slash is added by AnyUrl + validate_path_of_http_url("https://www.example.com/") + == "https://www.example.com" + ) From 891ef242b6857b27d59854fdf414d36e6e714d3c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Jul 2024 12:54:29 -0500 Subject: [PATCH 45/49] Get clear on when trailing slash is actually added - revert behavior to always strip a trailing slash. --- src/fideslang/validation.py | 8 ++--- tests/fideslang/test_validation.py | 53 ++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index c1d00bd5..9c1639c0 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -206,16 +206,16 @@ def valid_data_type(data_type_str: Optional[str]) -> Optional[str]: def validate_path_of_url(value: AnyUrl) -> str: - """Converts an AnyUrl to a string and removes trailing slash""" - return str(value).rstrip("/") + """Converts an AnyUrl to a string""" + return str(value) AnyUrlString = Annotated[AnyUrl, AfterValidator(validate_path_of_url)] def validate_path_of_http_url(value: AnyHttpUrl) -> str: - """Converts an AnyHttpUrl to a string and removes trailing slash""" - return str(value).rstrip("/") + """Converts an AnyHttpUrl to a string""" + return str(value) AnyHttpUrlString = Annotated[AnyHttpUrl, AfterValidator(validate_path_of_http_url)] diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 4e575363..e4d2276a 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -1,5 +1,5 @@ import pytest -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from fideslang.models import ( CollectionMeta, @@ -774,7 +774,7 @@ def test_valid_collection_key(self): class TestAnyUrlString: def test_valid_url(self): - assert AnyUrlString("https://www.example.com") + assert AnyUrlString("https://www.example.com/") def test_invalid_url(self): with pytest.raises(ValidationError) as exc: @@ -782,11 +782,27 @@ def test_invalid_url(self): assert_error_message_includes(exc, "Input should be a valid URL") - def test_validate_path_of_url(self): + def test_validate_url(self): assert ( - validate_path_of_url("https://www.example.com/") - == "https://www.example.com" - ) + TypeAdapter(AnyUrlString).validate_python("ftp://user:password@host") + == "ftp://user:password@host/" + ), "Trailing slash added" + assert ( + TypeAdapter(AnyUrlString).validate_python("ftp:user:password@host/") + == "ftp://user:password@host/" + ), "Format corrected" + assert ( + TypeAdapter(AnyUrlString).validate_python("ftp://user:password@host:3341/path") + == "ftp://user:password@host:3341/path" + ), "No change" + assert ( + TypeAdapter(AnyUrlString).validate_python("https://www.example.com/hello") + == "https://www.example.com/hello" + ), "No change" + assert ( + TypeAdapter(AnyUrlString).validate_python("https://www.example.com/hello/") + == "https://www.example.com/hello/" + ), "No change" def test_system_urls(self): system = System( @@ -800,7 +816,7 @@ def test_system_urls(self): ) # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString - assert system.privacy_policy == "https://www.example.com" + assert system.privacy_policy == "https://www.example.com/" class TestAnyHttpUrlString: @@ -815,6 +831,23 @@ def test_invalid_url(self): def test_validate_path_of_url(self): assert ( - validate_path_of_http_url("https://www.example.com/") - == "https://www.example.com" - ) + TypeAdapter(AnyHttpUrlString).validate_python("https://www.example.com") + == "https://www.example.com/" + ), "Trailing slash added" + assert ( + TypeAdapter(AnyHttpUrlString).validate_python("https://www.example.com/") + == "https://www.example.com/" + ), "No change" + assert ( + TypeAdapter(AnyHttpUrlString).validate_python("https://www.example.com/hello") + == "https://www.example.com/hello" + ), "No change" + assert ( + TypeAdapter(AnyHttpUrlString).validate_python("https://www.example.com/hello/") + == "https://www.example.com/hello/" + ), "No change" + + with pytest.raises(ValidationError) as exc: + TypeAdapter(AnyHttpUrlString).validate_python("ftp://user:password@host") + + assert_error_message_includes(exc, "URL scheme should be 'http' or 'https'") \ No newline at end of file From 412424abd685ab1a893e3cbc55b12f510fbf9b77 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Jul 2024 13:05:32 -0500 Subject: [PATCH 46/49] Further add test coverage. --- tests/fideslang/test_validation.py | 31 +++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index e4d2276a..257d2804 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -792,7 +792,9 @@ def test_validate_url(self): == "ftp://user:password@host/" ), "Format corrected" assert ( - TypeAdapter(AnyUrlString).validate_python("ftp://user:password@host:3341/path") + TypeAdapter(AnyUrlString).validate_python( + "ftp://user:password@host:3341/path" + ) == "ftp://user:password@host:3341/path" ), "No change" assert ( @@ -815,9 +817,24 @@ def test_system_urls(self): privacy_policy="https://www.example.com", ) - # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString + # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString. + # It also adds a trailing slash to example.com assert system.privacy_policy == "https://www.example.com/" + system = System( + description="Test Policy", + fides_key="test_system", + name="Test System", + organization_fides_key="1", + privacy_declarations=[], + system_type="SYSTEM", + privacy_policy="https://justtag.com/PRIVACY_POLICY.pdf", + ) + + # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString. + # No trailing slash is added + assert system.privacy_policy == "https://justtag.com/PRIVACY_POLICY.pdf" + class TestAnyHttpUrlString: def test_valid_url(self): @@ -839,15 +856,19 @@ def test_validate_path_of_url(self): == "https://www.example.com/" ), "No change" assert ( - TypeAdapter(AnyHttpUrlString).validate_python("https://www.example.com/hello") + TypeAdapter(AnyHttpUrlString).validate_python( + "https://www.example.com/hello" + ) == "https://www.example.com/hello" ), "No change" assert ( - TypeAdapter(AnyHttpUrlString).validate_python("https://www.example.com/hello/") + TypeAdapter(AnyHttpUrlString).validate_python( + "https://www.example.com/hello/" + ) == "https://www.example.com/hello/" ), "No change" with pytest.raises(ValidationError) as exc: TypeAdapter(AnyHttpUrlString).validate_python("ftp://user:password@host") - assert_error_message_includes(exc, "URL scheme should be 'http' or 'https'") \ No newline at end of file + assert_error_message_includes(exc, "URL scheme should be 'http' or 'https'") From b27bf039a5adaba29e91dc332c5138741a1ac60c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Jul 2024 13:16:34 -0500 Subject: [PATCH 47/49] Fix pylint. --- src/fideslang/models.py | 9 ++++++--- tests/fideslang/test_validation.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 4c53debc..92626a88 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -1084,8 +1084,10 @@ def privacy_declarations_reference_data_flows( Any `PrivacyDeclaration`s which include `egress` and/or `ingress` fields must only reference the `fides_key`s of defined `DataFlow`s in said field(s). """ - privacy_declarations = self.privacy_declarations or [] - for privacy_declaration in privacy_declarations: + privacy_declarations: List[PrivacyDeclaration] = self.privacy_declarations or [] + for ( + privacy_declaration + ) in privacy_declarations: # pylint:disable=not-an-iterable for direction in ["egress", "ingress"]: fides_keys = getattr(privacy_declaration, direction, None) if fides_keys is not None: @@ -1097,7 +1099,8 @@ def privacy_declarations_reference_data_flows( for fides_key in fides_keys: assert fides_key in [ - data_flow.fides_key for data_flow in data_flows + data_flow.fides_key + for data_flow in data_flows # pylint:disable=not-an-iterable ], f"PrivacyDeclaration '{privacy_declaration.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource." return self diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 257d2804..698d983a 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -27,8 +27,6 @@ FidesValidationError, valid_data_type, validate_fides_key, - validate_path_of_http_url, - validate_path_of_url, ) from tests.conftest import assert_error_message_includes @@ -828,12 +826,20 @@ def test_system_urls(self): organization_fides_key="1", privacy_declarations=[], system_type="SYSTEM", - privacy_policy="https://justtag.com/PRIVACY_POLICY.pdf", + privacy_policy="https://policy.samsungrs.com/consent/eu/nsc/privacy_policy_de.html", + legitimate_interest_disclosure_url="https://policy.samsungrs.com/consent/eu/nsc/privacy_policy_de.html#gdpr-article", ) # This is a string and not a Url type, because privacy_policy is using custom type AnyUrlString. # No trailing slash is added - assert system.privacy_policy == "https://justtag.com/PRIVACY_POLICY.pdf" + assert ( + system.privacy_policy + == "https://policy.samsungrs.com/consent/eu/nsc/privacy_policy_de.html" + ) + assert ( + system.legitimate_interest_disclosure_url + == "https://policy.samsungrs.com/consent/eu/nsc/privacy_policy_de.html#gdpr-article" + ) class TestAnyHttpUrlString: From 4c2b6e587e8b6c9eaa38e72f60a08fbd49922365 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 19 Jul 2024 15:17:09 -0500 Subject: [PATCH 48/49] Upgrade mypy --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0d0d24b3..ef60250a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ black==23.3.0 -mypy==1.4.0 +mypy==1.10.0 nox>=2023 packaging>=22.0 pre-commit==3.7.1 From 97bd9e106e5f40191ae40c090853cbff431c86d3 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 20 Aug 2024 09:10:41 -0500 Subject: [PATCH 49/49] Update changelog. --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46bc4594..274a5285 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fideslang/compare/3.0.1...main) +### Changed + +- Upgrades Pydantic for V2 support and removes support for Pydantic V1 [#11](https://github.com/ethyca/fideslang/pull/11) +- Removes Python 3.8 from supported versions [#11](https://github.com/ethyca/fideslang/pull/11) +- ## [3.0.1](https://github.com/ethyca/fideslang/compare/3.0.0...3.0.1) ### Added