diff --git a/changelog/8122-threading-header-support.yaml b/changelog/8122-threading-header-support.yaml new file mode 100644 index 00000000000..f2ccdac6db0 --- /dev/null +++ b/changelog/8122-threading-header-support.yaml @@ -0,0 +1,4 @@ +type: Added +description: Added threading header support (Reply-To, Message-ID, In-Reply-To, References) and plaintext alternative to all email providers +pr: 8122 +labels: [] diff --git a/src/fides/api/schemas/messaging/messaging.py b/src/fides/api/schemas/messaging/messaging.py index b79ddfa47ce..d590e2758ef 100644 --- a/src/fides/api/schemas/messaging/messaging.py +++ b/src/fides/api/schemas/messaging/messaging.py @@ -309,8 +309,14 @@ class EmailForActionType(BaseModel): """ subject: str - body: str + body: str # HTML body template_variables: Optional[Dict[str, Any]] = {} + # Threading / envelope fields (all optional, backward compatible) + reply_to: str | None = None + message_id: str | None = None # RFC 5322 Message-ID + in_reply_to: str | None = None + references: str | None = None + body_text: str | None = None # plaintext alternative for multipart/alternative class MessagingServiceDetails(Enum): diff --git a/src/fides/api/service/messaging/messaging_providers/aws_ses_service.py b/src/fides/api/service/messaging/messaging_providers/aws_ses_service.py index b83bb8a5e94..52ce19c2a98 100644 --- a/src/fides/api/service/messaging/messaging_providers/aws_ses_service.py +++ b/src/fides/api/service/messaging/messaging_providers/aws_ses_service.py @@ -29,12 +29,12 @@ def get_identity_verification_attributes( self, Identities: list[str] ) -> dict[str, dict[str, dict[str, str]]]: ... - def send_email( + def send_raw_email( self, Source: str, - Destination: dict[str, list[str]], - Message: dict[str, Any], - ) -> None: ... + Destinations: list[str], + RawMessage: dict[str, bytes], + ) -> dict[str, Any]: ... def _sanitize_aws_error(exc: Exception) -> str: @@ -133,10 +133,10 @@ def validate_on_save(self) -> None: raise MessageDispatchException(f"{identity} is not verified in SES.") def send_email(self, to: str, message: EmailForActionType) -> None: - """Send an email using AWS SES simple API. + """Send an email using AWS SES raw API for custom header support. - Does NOT call validate_email_and_domain_status() — that is done at - config save/test time. SES rejects sends from unverified identities. + Builds a MIME message using ``email.message.EmailMessage`` (modern + Python 3.6+ API). """ ses_client = self.get_ses_client() @@ -149,14 +149,14 @@ def send_email(self, to: str, message: EmailForActionType) -> None: from_address = f"noreply@{self.details.domain}" try: - ses_client.send_email( + msg = self.build_mime(from_address, to.strip(), message) + ses_client.send_raw_email( Source=from_address, - Destination={"ToAddresses": [to.strip()]}, - Message={ - "Subject": {"Data": message.subject}, - "Body": {"Html": {"Data": message.body}}, - }, + Destinations=[to.strip()], + RawMessage={"Data": msg.as_bytes()}, ) + except MessageDispatchException: + raise except Exception as exc: logger.error("Email failed to send: {}", str(exc)) raise MessageDispatchException( diff --git a/src/fides/api/service/messaging/messaging_providers/base.py b/src/fides/api/service/messaging/messaging_providers/base.py index 35f8d6a7b9f..0fbfc44adbd 100644 --- a/src/fides/api/service/messaging/messaging_providers/base.py +++ b/src/fides/api/service/messaging/messaging_providers/base.py @@ -1,4 +1,6 @@ from abc import ABC, abstractmethod +from email import policy as email_policy +from email.message import EmailMessage from typing import ClassVar from loguru import logger @@ -78,9 +80,56 @@ def _get_optional_secret(self, key: MessagingServiceSecrets) -> str | None: class BaseEmailProviderService(BaseMessageProviderService): """Base class for email provider services.""" + HEADER_REPLY_TO = "Reply-To" + HEADER_MESSAGE_ID = "Message-ID" + HEADER_IN_REPLY_TO = "In-Reply-To" + HEADER_REFERENCES = "References" + @abstractmethod def send_email(self, to: str, message: EmailForActionType) -> None: ... + @staticmethod + def get_threading_headers( + message: EmailForActionType, header_prefix: str = "" + ) -> dict[str, str]: + """Return non-None threading headers from the message.""" + candidates = { + BaseEmailProviderService.HEADER_REPLY_TO: message.reply_to, + BaseEmailProviderService.HEADER_MESSAGE_ID: message.message_id, + BaseEmailProviderService.HEADER_IN_REPLY_TO: message.in_reply_to, + BaseEmailProviderService.HEADER_REFERENCES: message.references, + } + return {f"{header_prefix}{k}": v for k, v in candidates.items() if v} + + @staticmethod + def build_mime( + from_address: str, to: str, message: EmailForActionType + ) -> EmailMessage: + """Build a MIME EmailMessage with optional threading headers. + + Reusable by any provider that sends raw MIME (e.g., SES, SMTP). + """ + msg = EmailMessage(policy=email_policy.SMTP) + msg["From"] = from_address + msg["To"] = to + msg["Subject"] = message.subject + + for header, value in BaseEmailProviderService.get_threading_headers( + message + ).items(): + msg[header] = value + + if message.body_text: + # The RFC 2046 multipart/alternative spec says parts should be + # ordered from simplest to richest. The email client picks the + # richest part it can render successfully. + msg.set_content(message.body_text) + msg.add_alternative(message.body, subtype="html") + else: + msg.set_content(message.body, subtype="html") + + return msg + class BaseSMSProviderService(BaseMessageProviderService): """Base class for SMS provider services.""" diff --git a/src/fides/api/service/messaging/messaging_providers/mailchimp_transactional_service.py b/src/fides/api/service/messaging/messaging_providers/mailchimp_transactional_service.py index 09b26537d30..62cf3bd3b7b 100644 --- a/src/fides/api/service/messaging/messaging_providers/mailchimp_transactional_service.py +++ b/src/fides/api/service/messaging/messaging_providers/mailchimp_transactional_service.py @@ -28,15 +28,29 @@ def __init__(self, messaging_config: MessagingConfig): ) def send_email(self, to: str, message: EmailForActionType) -> None: + msg_payload: dict = { + "from_email": self.from_email, + "subject": message.subject, + "html": message.body, + "to": [{"email": to.strip(), "type": "to"}], + } + + # Reply-to uses Mandrill's native field + if message.reply_to: + msg_payload["reply_to"] = message.reply_to + + # Threading headers (exclude Reply-To — handled natively above) + threading_headers = self.get_threading_headers(message) + threading_headers.pop(BaseEmailProviderService.HEADER_REPLY_TO, None) + if threading_headers: + msg_payload["headers"] = threading_headers + if message.body_text: + msg_payload["text"] = message.body_text + data = json.dumps( { "key": self.api_key, - "message": { - "from_email": self.from_email, - "subject": message.subject, - "html": message.body, - "to": [{"email": to.strip(), "type": "to"}], - }, + "message": msg_payload, } ) diff --git a/src/fides/api/service/messaging/messaging_providers/mailgun_service.py b/src/fides/api/service/messaging/messaging_providers/mailgun_service.py index eb2604aad3b..25e948e5791 100644 --- a/src/fides/api/service/messaging/messaging_providers/mailgun_service.py +++ b/src/fides/api/service/messaging/messaging_providers/mailgun_service.py @@ -57,6 +57,11 @@ def send_email(self, to: str, message: EmailForActionType) -> None: else: data["html"] = message.body + # Threading / envelope headers + data.update(self.get_threading_headers(message, header_prefix="h:")) + if message.body_text: + data["text"] = message.body_text + response = requests.post( f"{self.base_url}/{self.api_version}/{self.domain}/messages", auth=("api", self.api_key), diff --git a/src/fides/api/service/messaging/messaging_providers/twilio_email_service.py b/src/fides/api/service/messaging/messaging_providers/twilio_email_service.py index df8ed29b88c..9e93014f170 100644 --- a/src/fides/api/service/messaging/messaging_providers/twilio_email_service.py +++ b/src/fides/api/service/messaging/messaging_providers/twilio_email_service.py @@ -2,7 +2,16 @@ import sendgrid from loguru import logger -from sendgrid.helpers.mail import Content, Email, Mail, Personalization, TemplateId, To +from sendgrid.helpers.mail import ( + Content, + Email, + Header, + Mail, + Personalization, + ReplyTo, + TemplateId, + To, +) from fides.api.common_exceptions import MessageDispatchException from fides.api.models.messaging import MessagingConfig @@ -41,9 +50,21 @@ def send_email(self, to: str, message: EmailForActionType) -> None: from_email = Email(self.from_email) to_email = To(to.strip()) mail = self._compose_mail( - from_email, to_email, message.subject, message.body, template_id + from_email, + to_email, + message.subject, + message.body, + template_id, + body_text=message.body_text, ) + # Threading / envelope headers + for key, value in self.get_threading_headers(message).items(): + if key == BaseEmailProviderService.HEADER_REPLY_TO: + mail.reply_to = ReplyTo(value) + else: + mail.header = Header(key, value) + response = sg.client.mail.send.post(request_body=mail.get()) if response.status_code >= 400: logger.error( @@ -77,8 +98,13 @@ def _compose_mail( subject: str, message_body: str, template_id: str | None = None, + body_text: str | None = None, ) -> Mail: - """Composes a SendGrid Mail object, using a template if one exists.""" + """Composes a SendGrid Mail object, using a template if one exists. + + When body_text is provided, builds multipart/alternative with text/plain + before text/html (RFC 2046: last part is most preferred). + """ if template_id: mail = Mail(from_email=from_email, subject=subject) mail.template_id = TemplateId(template_id) @@ -86,6 +112,12 @@ def _compose_mail( personalization.dynamic_template_data = {"fides_email_body": message_body} personalization.add_email(to_email) mail.add_personalization(personalization) + elif body_text: + mail = Mail(from_email=from_email, subject=subject) + mail.add_personalization(Personalization()) + mail.personalizations[0].add_email(to_email) + mail.content = Content("text/plain", body_text) + mail.content = Content("text/html", message_body) else: content = Content("text/html", message_body) mail = Mail(from_email, to_email, subject, content) diff --git a/tests/service/messaging/test_provider_headers.py b/tests/service/messaging/test_provider_headers.py new file mode 100644 index 00000000000..61eb506dcc9 --- /dev/null +++ b/tests/service/messaging/test_provider_headers.py @@ -0,0 +1,333 @@ +"""Per-provider threading header mapping tests. + +Tests that each email provider correctly maps EmailForActionType threading +fields to provider-specific API payloads, and omits them when None. +""" + +import json +from email import policy +from email.parser import BytesParser +from unittest import mock +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from fides.api.common_exceptions import MessageDispatchException +from fides.api.schemas.messaging.messaging import EmailForActionType +from fides.api.service.messaging.messaging_providers.aws_ses_service import ( + AwsSesService, +) +from fides.api.service.messaging.messaging_providers.mailchimp_transactional_service import ( + MailchimpTransactionalService, +) +from fides.api.service.messaging.messaging_providers.mailgun_service import ( + MailgunService, +) +from fides.api.service.messaging.messaging_providers.twilio_email_service import ( + TwilioEmailService, +) + + +def _make_messaging_config(service_type, details=None, secrets=None): + """Create a minimal mock MessagingConfig for provider tests.""" + config = MagicMock() + config.service_type = service_type + config.details = details or {} + config.secrets = secrets or {} + return config + + +def _email_with_headers(**overrides): + """Create an EmailForActionType with all threading fields set.""" + defaults = { + "subject": "Test Subject", + "body": "

Test body

", + "reply_to": "reply+token123@replies.example.com", + "message_id": "", + "in_reply_to": "", + "references": "", + "body_text": "Test body plaintext", + } + defaults.update(overrides) + return EmailForActionType(**defaults) + + +def _email_without_headers(): + """Create an EmailForActionType with no threading fields.""" + return EmailForActionType(subject="Test Subject", body="

Test body

") + + +class TestMailgunHeaders: + @pytest.fixture() + def mailgun_service(self): + config = _make_messaging_config( + "mailgun", + details={ + "domain": "example.com", + "is_eu_domain": False, + "api_version": "v3", + }, + secrets={"mailgun_api_key": "test-key"}, + ) + return MailgunService(config) + + @patch("fides.api.service.messaging.messaging_providers.mailgun_service.requests") + def test_headers_included_when_set(self, mock_requests, mailgun_service): + mock_requests.get.return_value = Mock(status_code=404) + mock_requests.post.return_value = Mock(ok=True) + + mailgun_service.send_email("to@test.com", _email_with_headers()) + + call_kwargs = mock_requests.post.call_args + data = call_kwargs.kwargs.get("data") or call_kwargs[1].get("data") + + assert data["h:Reply-To"] == "reply+token123@replies.example.com" + assert data["h:Message-ID"] == "" + assert data["h:In-Reply-To"] == "" + assert data["h:References"] == "" + assert data["text"] == "Test body plaintext" + + @patch("fides.api.service.messaging.messaging_providers.mailgun_service.requests") + def test_headers_omitted_when_none(self, mock_requests, mailgun_service): + mock_requests.get.return_value = Mock(status_code=404) + mock_requests.post.return_value = Mock(ok=True) + + mailgun_service.send_email("to@test.com", _email_without_headers()) + + call_kwargs = mock_requests.post.call_args + data = call_kwargs.kwargs.get("data") or call_kwargs[1].get("data") + + assert "h:Reply-To" not in data + assert "h:Message-ID" not in data + assert "h:In-Reply-To" not in data + assert "h:References" not in data + assert "text" not in data + + +class TestTwilioEmailHeaders: + @pytest.fixture() + def twilio_email_service(self): + config = _make_messaging_config( + "twilio_email", + details={"twilio_email_from": "from@test.com"}, + secrets={"twilio_api_key": "test-key"}, + ) + return TwilioEmailService(config) + + @patch( + "fides.api.service.messaging.messaging_providers.twilio_email_service.sendgrid" + ) + def test_headers_included_when_set(self, mock_sendgrid, twilio_email_service): + mock_sg = MagicMock() + mock_sendgrid.SendGridAPIClient.return_value = mock_sg + mock_sg.client.templates.get.return_value = Mock( + body=json.dumps({"result": []}).encode() + ) + mock_sg.client.mail.send.post.return_value = Mock(status_code=200) + + twilio_email_service.send_email("to@test.com", _email_with_headers()) + + call_kwargs = mock_sg.client.mail.send.post.call_args + request_body = call_kwargs.kwargs.get("request_body") or call_kwargs[1].get( + "request_body" + ) + + # SendGrid Mail.get() returns a dict with headers and content + assert request_body is not None + # Check reply_to is set + assert "reply_to" in request_body + assert request_body["reply_to"]["email"] == "reply+token123@replies.example.com" + # Check headers + headers = request_body.get("headers", {}) + assert headers.get("Message-ID") == "" + assert headers.get("In-Reply-To") == "" + assert headers.get("References") == "" + # Check plaintext content + content_types = [c["type"] for c in request_body.get("content", [])] + assert "text/plain" in content_types + + @patch( + "fides.api.service.messaging.messaging_providers.twilio_email_service.sendgrid" + ) + def test_headers_omitted_when_none(self, mock_sendgrid, twilio_email_service): + mock_sg = MagicMock() + mock_sendgrid.SendGridAPIClient.return_value = mock_sg + mock_sg.client.templates.get.return_value = Mock( + body=json.dumps({"result": []}).encode() + ) + mock_sg.client.mail.send.post.return_value = Mock(status_code=200) + + twilio_email_service.send_email("to@test.com", _email_without_headers()) + + call_kwargs = mock_sg.client.mail.send.post.call_args + request_body = call_kwargs.kwargs.get("request_body") or call_kwargs[1].get( + "request_body" + ) + + assert "reply_to" not in request_body + assert "headers" not in request_body or request_body["headers"] == {} + content_types = [c["type"] for c in request_body.get("content", [])] + assert "text/plain" not in content_types + + +class TestMailchimpTransactionalHeaders: + @pytest.fixture() + def mailchimp_service(self): + config = _make_messaging_config( + "mailchimp_transactional", + details={"email_from": "from@test.com"}, + secrets={"mailchimp_transactional_api_key": "test-key"}, + ) + return MailchimpTransactionalService(config) + + @patch( + "fides.api.service.messaging.messaging_providers.mailchimp_transactional_service.requests" + ) + def test_headers_included_when_set(self, mock_requests, mailchimp_service): + mock_requests.post.return_value = Mock( + ok=True, json=lambda: [{"status": "sent"}] + ) + + mailchimp_service.send_email("to@test.com", _email_with_headers()) + + call_kwargs = mock_requests.post.call_args + payload = json.loads( + call_kwargs.kwargs.get("data") or call_kwargs[1].get("data") + ) + msg = payload["message"] + + assert msg["reply_to"] == "reply+token123@replies.example.com" + assert "Reply-To" not in msg.get("headers", {}) + assert msg["headers"]["Message-ID"] == "" + assert msg["headers"]["In-Reply-To"] == "" + assert msg["headers"]["References"] == "" + assert msg["text"] == "Test body plaintext" + + @patch( + "fides.api.service.messaging.messaging_providers.mailchimp_transactional_service.requests" + ) + def test_headers_omitted_when_none(self, mock_requests, mailchimp_service): + mock_requests.post.return_value = Mock( + ok=True, json=lambda: [{"status": "sent"}] + ) + + mailchimp_service.send_email("to@test.com", _email_without_headers()) + + call_kwargs = mock_requests.post.call_args + payload = json.loads( + call_kwargs.kwargs.get("data") or call_kwargs[1].get("data") + ) + msg = payload["message"] + + assert "headers" not in msg or msg["headers"] == {} + assert "text" not in msg + + +class TestAwsSesHeaders: + """Tests for SES send_raw_email MIME construction with threading headers.""" + + @pytest.fixture() + def ses_service(self): + config = _make_messaging_config( + "aws_ses", + details={ + "aws_region": "us-east-1", + "email_from": "test@example.com", + "domain": "example.com", + }, + secrets={ + "aws_access_key_id": "fake", + "aws_secret_access_key": "fake", + "auth_method": "secret_keys", + }, + ) + service = AwsSesService(config) + service._ses_client = MagicMock() + return service + + def test_headers_included_in_raw_mime(self, ses_service): + ses_service.send_email("to@test.com", _email_with_headers()) + + call_args = ses_service._ses_client.send_raw_email.call_args + raw_data = ( + call_args.kwargs.get("RawMessage", {}).get("Data") + or call_args[1]["RawMessage"]["Data"] + ) + + # Parse the MIME message + msg = BytesParser(policy=policy.default).parsebytes( + raw_data if isinstance(raw_data, bytes) else raw_data.encode() + ) + + assert msg["Reply-To"] == "reply+token123@replies.example.com" + assert msg["Message-ID"] == "" + assert msg["In-Reply-To"] == "" + assert msg["References"] == "" + + # Should be multipart/alternative with text/plain and text/html + assert msg.get_content_type() == "multipart/alternative" + parts = list(msg.iter_parts()) + content_types = [p.get_content_type() for p in parts] + assert "text/plain" in content_types + assert "text/html" in content_types + + def test_headers_omitted_when_none(self, ses_service): + ses_service.send_email("to@test.com", _email_without_headers()) + + call_args = ses_service._ses_client.send_raw_email.call_args + raw_data = ( + call_args.kwargs.get("RawMessage", {}).get("Data") + or call_args[1]["RawMessage"]["Data"] + ) + + msg = BytesParser(policy=policy.default).parsebytes( + raw_data if isinstance(raw_data, bytes) else raw_data.encode() + ) + + assert msg["Reply-To"] is None + assert msg["Message-ID"] is None or msg["Message-ID"] == "" + assert msg["In-Reply-To"] is None + assert msg["References"] is None + # No body_text, so should be text/html only (not multipart) + assert msg.get_content_type() == "text/html" + + def test_non_ascii_subject_encoding(self, ses_service): + """Non-ASCII subjects must be RFC 2047 encoded in raw MIME.""" + email = _email_with_headers(subject="Ré: données personnelles") + ses_service.send_email("to@test.com", email) + + call_args = ses_service._ses_client.send_raw_email.call_args + raw_data = ( + call_args.kwargs.get("RawMessage", {}).get("Data") + or call_args[1]["RawMessage"]["Data"] + ) + + msg = BytesParser(policy=policy.default).parsebytes( + raw_data if isinstance(raw_data, bytes) else raw_data.encode() + ) + # Decoded subject should match + assert msg["Subject"] == "Ré: données personnelles" + + def test_non_ascii_body_encoding(self, ses_service): + """Non-ASCII body content must use appropriate transfer encoding.""" + email = _email_with_headers( + body="

Données personnelles: café résumé

", + body_text="Données personnelles: café résumé", + ) + ses_service.send_email("to@test.com", email) + + call_args = ses_service._ses_client.send_raw_email.call_args + raw_data = ( + call_args.kwargs.get("RawMessage", {}).get("Data") + or call_args[1]["RawMessage"]["Data"] + ) + + msg = BytesParser(policy=policy.default).parsebytes( + raw_data if isinstance(raw_data, bytes) else raw_data.encode() + ) + + # Both parts should decode correctly + for part in msg.iter_parts(): + content = part.get_content() + assert "café" in content + assert "résumé" in content diff --git a/tests/service/messaging/test_provider_map.py b/tests/service/messaging/test_provider_map.py new file mode 100644 index 00000000000..40d22106ad2 --- /dev/null +++ b/tests/service/messaging/test_provider_map.py @@ -0,0 +1,14 @@ +"""_PROVIDER_MAP completeness invariant test. + +Ensures every MessagingServiceType value has a corresponding provider class +in the dispatch provider map. If a new service type is added without a +provider, this test catches it. +""" + +from fides.api.schemas.messaging.messaging import MessagingServiceType +from fides.api.service.messaging.message_dispatch_service import _PROVIDER_MAP + + +class TestProviderMapCompleteness: + def test_all_service_types_have_providers(self): + assert set(_PROVIDER_MAP.keys()) == set(MessagingServiceType)