Cherry-picking consent model changes

Author: galvana

CHANGELOG.md

@@ -38,6 +38,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o
 - Updated bulk ignore assets toast message [#6043](https://github.com/ethyca/fides/pull/6043)
 - Updated UI behavior for editing languages in the Experience config for consistency and clarity [#6055](https://github.com/ethyca/fides/pull/6055)
 - Moved detection & discovery features out of beta [#6059](https://github.com/ethyca/fides/pull/6059)
+- Updated consent automation models to support echo detection in Fidesplus [#6054](https://github.com/ethyca/fides/pull/6054)
 
 ### Developer Experience
 - Reduced animations on Cypress tests in Privacy Center for quicker results [#5976](https://github.com/ethyca/fides/pull/5976)

src/fides/api/schemas/consentable_item.py

@@ -3,7 +3,6 @@
 from pydantic import BaseModel, Field
 
 from fides.api.models.consent_automation import ConsentableItem as ConsentableItemModel
-from fides.api.models.privacy_notice import UserConsentPreference
 from fides.api.schemas.base_class import FidesSchema
 
 
@@ -85,10 +84,8 @@ class ConsentWebhookResult(BaseModel):
 
     identity_map: Dict[
         Literal["email", "phone_number", "fides_user_device", "external_id"], str
-    ] = {}
-    notice_map: Dict[str, UserConsentPreference] = {}
-
-    @property
-    def success(self) -> bool:
-        """Returns true if both the identity map and notice map are not empty."""
-        return bool(self.identity_map) and bool(self.notice_map)
+    ] = Field(default_factory=dict, description="The identity of the user.")
+    notice_id_map: Dict[str, str] = Field(
+        default_factory=dict,
+        description="A map of privacy notice IDs to user consent preferences.",
+    )

src/fides/api/service/connectors/saas_connector.py

@@ -4,6 +4,7 @@
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import pydash
+from fideslang.validation import FidesKey
 from loguru import logger
 from requests import Response
 from sqlalchemy.orm import Session
@@ -18,7 +19,6 @@
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
-from fides.api.models.privacy_notice import UserConsentPreference
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
 from fides.api.schemas.consentable_item import (
     ConsentableItem,
@@ -687,15 +687,16 @@ def run_consent_request(
 
         if notice_based_override_function:
             # follow the notice-based SaaS consent flow
-            notice_id_to_preference_map, filtered_preferences = (
-                build_user_consent_and_filtered_preferences_for_service(
-                    self.configuration.system,
-                    privacy_request,
-                    session,
-                    True,
-                )
+            (
+                notice_preference_map,
+                filtered_preferences,
+            ) = build_user_consent_and_filtered_preferences_for_service(
+                self.configuration.system,
+                privacy_request,
+                session,
+                True,
             )
-            if not notice_id_to_preference_map:
+            if not notice_preference_map:
                 logger.info(
                     "Skipping consent requests on node {}: No actionable consent preferences to propagate",
                     node.address.value,
@@ -723,12 +724,13 @@ def run_consent_request(
                 )
             consent_propagation_status = self._invoke_consent_request_override(
                 notice_based_override_function,
+                self.configuration.key,
                 self.create_client(),
                 policy,
                 privacy_request,
                 self.secrets,
                 identity_data,
-                notice_id_to_preference_map,  # type: ignore[arg-type]
+                notice_preference_map,  # type: ignore[arg-type]
                 notice_based_consentable_item_hierarchy,
             )
             if consent_propagation_status == ConsentPropagationStatus.no_update_needed:
@@ -800,6 +802,7 @@ def run_consent_request(
                     )
                     consent_propagation_status = self._invoke_consent_request_override(
                         override_function,
+                        self.configuration.key,
                         self.create_client(),
                         policy,
                         privacy_request,
@@ -997,12 +1000,13 @@ def _invoke_masking_request_override(
     @staticmethod
     def _invoke_consent_request_override(
         override_function: RequestOverrideFunction,
+        connection_key: FidesKey,
         client: AuthenticatedClient,
         policy: Policy,
         privacy_request: PrivacyRequest,
         secrets: Any,
         identity_data: Optional[Dict[str, Any]] = None,
-        notice_id_to_preference_map: Optional[Dict[str, UserConsentPreference]] = None,
+        notice_preference_map: Optional[Dict[str, Dict[str, Any]]] = None,
         consentable_items_hierarchy: Optional[List[ConsentableItem]] = None,
     ) -> ConsentPropagationStatus:
         """
@@ -1011,13 +1015,14 @@ def _invoke_consent_request_override(
         """
         try:
             logger.info("Invoking consent request override function...")
-            if notice_id_to_preference_map:
+            if notice_preference_map:
                 # At this point, we've already validated the override function signature to take these params
                 return override_function(
+                    connection_key,
                     client,
                     secrets,
                     identity_data,
-                    notice_id_to_preference_map,
+                    notice_preference_map,
                     consentable_items_hierarchy,
                 )  # type: ignore
             return override_function(

src/fides/api/service/saas_request/saas_request_override_factory.py

@@ -248,7 +248,7 @@ def validate_update_consent_function(f: Callable) -> None:
         )
     if len(sig.parameters) < 5:
         raise InvalidSaaSRequestOverrideException(
-            "Provided SaaS update consent function must declare at least 4 parameters"
+            "Provided SaaS update consent function must declare at least 5 parameters"
         )
 
 
@@ -258,9 +258,9 @@ def validate_process_consent_webhook_function(f: Callable) -> None:
         raise InvalidSaaSRequestOverrideException(
             "Provided SaaS process consent webhook function must return a ConsentWebhookResult"
         )
-    if len(sig.parameters) < 5:
+    if len(sig.parameters) < 4:
         raise InvalidSaaSRequestOverrideException(
-            "Provided SaaS process consent webhook function must declare at least 5 parameters"
+            "Provided SaaS process consent webhook function must declare at least 4 parameters"
         )
 
 

src/fides/api/task/graph_task.py

@@ -125,16 +125,15 @@ def result(*args: Any, **kwargs: Any) -> Any:
                     ActionDisabled,
                     NotSupportedForCollection,
                 ) as exc:
-                    traceback.print_exc()
                     logger.warning(
-                        "Skipping collection {} for privacy_request: {}",
+                        "{} - Skipping collection {} for privacy_request: {}",
+                        exc.__class__.__name__,
                         self.execution_node.address,
                         self.resources.request.id,
                     )
                     self.log_skipped(action_type, exc)
                     return default_return
                 except SkippingConsentPropagation as exc:
-                    traceback.print_exc()
                     logger.warning(
                         "Skipping consent propagation on collection {} for privacy_request: {}",
                         self.execution_node.address,

src/fides/api/util/consent_util.py

@@ -64,7 +64,7 @@ def build_user_consent_and_filtered_preferences_for_service(
     session: Session,
     is_notice_based: bool = False,
 ) -> Tuple[
-    Optional[Union[bool, Dict[str, UserConsentPreference]]],
+    Optional[Union[bool, Dict[str, Dict[str, Any]]]],
     List[PrivacyPreferenceHistory],
 ]:
     """
@@ -109,7 +109,7 @@ def build_user_consent_and_filtered_preferences_for_service(
 
     # 1. NOTICE-BASED WORKFLOW
     if is_notice_based:
-        notice_id_to_preference_map: Dict[str, UserConsentPreference] = {}
+        notice_preference_map: Dict[str, Dict[str, Any]] = {}
         # retrieve notices from the DB if they are associated with a relevant preference
         notices_with_preference: Query = session.query(PrivacyNotice).filter(
             PrivacyNotice.notice_key.in_(
@@ -125,10 +125,16 @@ def build_user_consent_and_filtered_preferences_for_service(
             ]
             # we only expect max 1 preference in this list
             if preference and preference[0]:
-                notice_id_to_preference_map[notice.id] = preference[0].preference  # type: ignore[assignment]
+                notice_preference_map[notice.id] = {
+                    "preference": preference[0].preference,
+                    "notice_key": notice.notice_key,
+                }
                 filtered_preferences.append(preference[0])
 
-        return notice_id_to_preference_map, filtered_preferences
+        return (
+            notice_preference_map,
+            filtered_preferences,
+        )
 
     # 2. GLOBAL (OPT-IN/OUT) WORKFLOW
     preference_to_propagate: UserConsentPreference = (

tests/ops/service/connectors/test_saas_connector.py

@@ -6,6 +6,7 @@
 from uuid import uuid4
 
 import pytest
+from fideslang.models import FidesKey
 from requests import Response
 from sqlalchemy.orm import Session
 from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUND
@@ -50,10 +51,11 @@ def uuid():
 
 
 def valid_consent_update_override(
+    connection_key: FidesKey,
     client: AuthenticatedClient,
     secrets: Dict[str, Any],
     input_data: Dict[str, List[Any]],
-    notice_id_to_preference_map: Dict[str, UserConsentPreference],
+    notice_preference_map: Dict[str, Dict[str, Any]],
     consentable_items_hierarchy: List[ConsentableItem],
 ) -> ConsentPropagationStatus:
     """

tests/ops/service/saas_request/test_saas_request_override_factory.py

@@ -2,6 +2,7 @@
 from uuid import uuid4
 
 import pytest
+from fideslang.validation import FidesKey
 
 from fides.api.common_exceptions import (
     InvalidSaaSRequestOverrideException,
@@ -91,6 +92,7 @@ def valid_consent_override(
 
 
 def valid_consent_update_override(
+    connection_key: FidesKey,
     client: AuthenticatedClient,
     secrets: Dict[str, Any],
     input_data: Dict[str, List[Any]],
@@ -107,7 +109,6 @@ def valid_process_consent_webhook_override(
     client: AuthenticatedClient,
     secrets: Dict[str, Any],
     payload: Any,
-    notice_id_to_preference_map: Dict[str, UserConsentPreference],
     consentable_items: List[ConsentableItem],
 ) -> ConsentWebhookResult:
     """

tests/ops/util/test_consent_util.py

@@ -163,17 +163,23 @@ def test_notice_based_consent_multiple_preferences(
         pref_4.privacy_request_id = privacy_request_with_consent_policy.id
         pref_4.save(db)
 
-        notice_id_to_preference_map, filtered_preferences = (
+        notice_preference_map, filtered_preferences = (
             build_user_consent_and_filtered_preferences_for_service(
                 system,
                 privacy_request_with_consent_policy,
                 db,
                 True,  # signal notice-based consent
             )
         )
-        assert notice_id_to_preference_map == {
-            privacy_notice.id: UserConsentPreference.opt_in,
-            privacy_notice_2.id: UserConsentPreference.opt_out,
+        assert notice_preference_map == {
+            privacy_notice.id: {
+                "preference": UserConsentPreference.opt_in,
+                "notice_key": privacy_notice.notice_key,
+            },
+            privacy_notice_2.id: {
+                "preference": UserConsentPreference.opt_out,
+                "notice_key": privacy_notice_2.notice_key,
+            },
         }
         assert filtered_preferences == [pref_1, pref_2]