From e55084673702ddc4cafdf5463f416d66ea229920 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Fri, 14 Feb 2025 11:55:30 -0600 Subject: [PATCH] feat(backend): add ability to send emails to notification service (#9469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need a way to send emails for the email service to function. We will depend on Postmark to do that. This PR adds a simple email-sending service with the required settings to make it work. It also builds on the previous agent run by sending the emails that are in the immediate queue. Keep in mind that the email template leaves a bit to be desired. ### Changes 🏗️ - Add `email.py` with the minimum required to send an email (plus type handling) - Add settings configs for the token and the send address to the `settings.py` and `.env.example` - Add a db call to get user email by ID since the `metadata` field of `prisma.models.User` isn't serializable over our message bus tool `Pyro` that the `DatabaseManager` uses - Add a horrible `AgentRun` email template using `jinja2` - Add `postmarker` to `pyproject.toml` ### Checklist 📋 #### For code changes: - [x] I have clearly listed my changes in the PR description - [x] I have made a test plan - [x] I have tested my changes according to the test plan: - [x] Build and run an agent and make sure it emails me (must be signed into same domain as receiving address for now) #### For configuration changes: - [x] `.env.example` is updated or already compatible with my changes - [ ] `docker-compose.yml` is updated or already compatible with my changes - [x] I have added a check that disables email if config is not set correctly - [x] I have included a list of my configuration changes in the PR description (under **Changes**) --------- Co-authored-by: Reinier van der Leer --- autogpt_platform/backend/.env.example | 4 + .../backend/backend/data/notifications.py | 19 +- autogpt_platform/backend/backend/data/user.py | 8 + .../backend/backend/executor/database.py | 2 + .../backend/backend/executor/manager.py | 4 +- .../backend/backend/notifications/email.py | 99 +++++ .../backend/notifications/notifications.py | 21 +- .../templates/agent_run.html.jinja2 | 75 ++++ .../notifications/templates/base.html.jinja2 | 352 ++++++++++++++++++ .../backend/backend/util/settings.py | 9 + autogpt_platform/backend/backend/util/text.py | 58 ++- autogpt_platform/backend/poetry.lock | 47 ++- autogpt_platform/backend/pyproject.toml | 2 + 13 files changed, 689 insertions(+), 11 deletions(-) create mode 100644 autogpt_platform/backend/backend/notifications/email.py create mode 100644 autogpt_platform/backend/backend/notifications/templates/agent_run.html.jinja2 create mode 100644 autogpt_platform/backend/backend/notifications/templates/base.html.jinja2 diff --git a/autogpt_platform/backend/.env.example b/autogpt_platform/backend/.env.example index ceb3570343..11383b9589 100644 --- a/autogpt_platform/backend/.env.example +++ b/autogpt_platform/backend/.env.example @@ -25,6 +25,10 @@ BEHAVE_AS=local PYRO_HOST=localhost SENTRY_DSN= +# Email For Postmark so we can send emails +POSTMARK_SERVER_API_TOKEN= +POSTMARK_SENDER_EMAIL=invalid@invalid.com + ## User auth with Supabase is required for any of the 3rd party integrations with auth to work. ENABLE_AUTH=true SUPABASE_URL=http://localhost:8000 diff --git a/autogpt_platform/backend/backend/data/notifications.py b/autogpt_platform/backend/backend/data/notifications.py index a4549de632..1dbfb8bc4d 100644 --- a/autogpt_platform/backend/backend/data/notifications.py +++ b/autogpt_platform/backend/backend/data/notifications.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timedelta from enum import Enum -from typing import Annotated, Generic, Optional, TypeVar, Union +from typing import Annotated, Any, Generic, Optional, TypeVar, Union from prisma import Json from prisma.enums import NotificationType @@ -35,10 +35,10 @@ class BaseNotificationData(BaseModel): class AgentRunData(BaseNotificationData): agent_name: str credits_used: float - # remaining_balance: float execution_time: float - graph_id: str node_count: int = Field(..., description="Number of nodes executed") + graph_id: str + outputs: dict[str, Any] = Field(..., description="Outputs of the agent") class ZeroBalanceData(BaseNotificationData): @@ -203,6 +203,19 @@ class NotificationTypeOverride: NotificationType.MONTHLY_SUMMARY: "monthly_summary.html", }[self.notification_type] + @property + def subject(self) -> str: + return { + NotificationType.AGENT_RUN: "Agent Run Report", + NotificationType.ZERO_BALANCE: "You're out of credits!", + NotificationType.LOW_BALANCE: "Low Balance Warning!", + NotificationType.BLOCK_EXECUTION_FAILED: "Uh oh! Block Execution Failed", + NotificationType.CONTINUOUS_AGENT_ERROR: "Shoot! Continuous Agent Error", + NotificationType.DAILY_SUMMARY: "Here's your daily summary!", + NotificationType.WEEKLY_SUMMARY: "Look at all the cool stuff you did last week!", + NotificationType.MONTHLY_SUMMARY: "We did a lot this month!", + }[self.notification_type] + class NotificationPreference(BaseModel): user_id: str diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py index f7545b6e5a..f15816d650 100644 --- a/autogpt_platform/backend/backend/data/user.py +++ b/autogpt_platform/backend/backend/data/user.py @@ -49,6 +49,14 @@ async def get_user_by_id(user_id: str) -> User: return User.model_validate(user) +async def get_user_email_by_id(user_id: str) -> str: + try: + user = await prisma.user.find_unique_or_raise(where={"id": user_id}) + return user.email + except Exception as e: + raise DatabaseError(f"Failed to get user email for user {user_id}: {e}") from e + + async def create_default_user() -> Optional[User]: user = await prisma.user.find_unique(where={"id": DEFAULT_USER_ID}) if not user: diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.py index 75730b3726..02e1535f5e 100644 --- a/autogpt_platform/backend/backend/executor/database.py +++ b/autogpt_platform/backend/backend/executor/database.py @@ -28,6 +28,7 @@ from backend.data.user import ( get_active_user_ids_in_timerange, get_active_users_ids, get_user_by_id, + get_user_email_by_id, get_user_integrations, get_user_metadata, get_user_notification_preference, @@ -105,6 +106,7 @@ class DatabaseManager(AppService): get_active_user_ids_in_timerange ) get_user_by_id = exposed_run_and_wait(get_user_by_id) + get_user_email_by_id = exposed_run_and_wait(get_user_email_by_id) get_user_notification_preference = exposed_run_and_wait( get_user_notification_preference ) diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py index 513e384b08..65c84ee1bc 100644 --- a/autogpt_platform/backend/backend/executor/manager.py +++ b/autogpt_platform/backend/backend/executor/manager.py @@ -206,13 +206,14 @@ def execute_node( # This is fine because for now, there is no block that is charged by time. cost = db_client.spend_credits(data, input_size + output_size, 0) + outputs: dict[str, Any] = {} for output_name, output_data in node_block.execute( input_data, **extra_exec_kwargs ): output_size += len(json.dumps(output_data)) log_metadata.info("Node produced output", **{output_name: output_data}) db_client.upsert_execution_output(node_exec_id, output_name, output_data) - + outputs[output_name] = output_data for execution in _enqueue_next_nodes( db_client=db_client, node=node, @@ -230,6 +231,7 @@ def execute_node( user_id=user_id, type=NotificationType.AGENT_RUN, data=AgentRunData( + outputs=outputs, agent_name=node_block.name, credits_used=cost, execution_time=0, diff --git a/autogpt_platform/backend/backend/notifications/email.py b/autogpt_platform/backend/backend/notifications/email.py new file mode 100644 index 0000000000..a57f5b3a41 --- /dev/null +++ b/autogpt_platform/backend/backend/notifications/email.py @@ -0,0 +1,99 @@ +import logging +import pathlib + +from postmarker.core import PostmarkClient +from postmarker.models.emails import EmailManager +from prisma.enums import NotificationType +from pydantic import BaseModel + +from backend.data.notifications import ( + NotificationEventModel, + NotificationTypeOverride, + T_co, +) +from backend.util.settings import Settings +from backend.util.text import TextFormatter + +logger = logging.getLogger(__name__) +settings = Settings() + + +# The following is a workaround to get the type checker to recognize the EmailManager type +# This is a temporary solution and should be removed once the Postmark library is updated +# to support type annotations. +class TypedPostmarkClient(PostmarkClient): + emails: EmailManager + + +class Template(BaseModel): + subject_template: str + body_template: str + base_template: str + + +class EmailSender: + def __init__(self): + if settings.secrets.postmark_server_api_token: + self.postmark = TypedPostmarkClient( + server_token=settings.secrets.postmark_server_api_token + ) + else: + logger.warning( + "Postmark server API token not found, email sending disabled" + ) + self.formatter = TextFormatter() + + def send_templated( + self, + notification: NotificationType, + user_email: str, + data: NotificationEventModel[T_co] | list[NotificationEventModel[T_co]], + ): + """Send an email to a user using a template pulled from the notification type""" + if not self.postmark: + logger.warning("Postmark client not initialized, email not sent") + return + template = self._get_template(notification) + + try: + subject, full_message = self.formatter.format_email( + base_template=template.base_template, + subject_template=template.subject_template, + content_template=template.body_template, + data=data, + unsubscribe_link="https://autogpt.com/unsubscribe", + ) + + except Exception as e: + logger.error(f"Error formatting full message: {e}") + raise e + + self._send_email(user_email, subject, full_message) + + def _get_template(self, notification: NotificationType): + # convert the notification type to a notification type override + notification_type_override = NotificationTypeOverride(notification) + # find the template in templates/name.html (the .template returns with the .html) + template_path = f"templates/{notification_type_override.template}.jinja2" + logger.debug( + f"Template full path: {pathlib.Path(__file__).parent / template_path}" + ) + base_template_path = "templates/base.html.jinja2" + with open(pathlib.Path(__file__).parent / base_template_path, "r") as file: + base_template = file.read() + with open(pathlib.Path(__file__).parent / template_path, "r") as file: + template = file.read() + return Template( + subject_template=notification_type_override.subject, + body_template=template, + base_template=base_template, + ) + + def _send_email(self, user_email: str, subject: str, body: str): + logger.debug(f"Sending email to {user_email} with subject {subject}") + self.postmark.emails.send( + From=settings.config.postmark_sender_email, + To=user_email, + Subject=subject, + HtmlBody=body, + ) diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.py index d4e591ac8b..b198b4c3db 100644 --- a/autogpt_platform/backend/backend/notifications/notifications.py +++ b/autogpt_platform/backend/backend/notifications/notifications.py @@ -14,6 +14,7 @@ from backend.data.notifications import ( ) from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig from backend.executor.database import DatabaseManager +from backend.notifications.email import EmailSender from backend.util.service import AppService, expose, get_service_client from backend.util.settings import Settings @@ -100,6 +101,7 @@ class NotificationManager(AppService): self.use_db = True self.rabbitmq_config = create_notification_config() self.running = True + self.email_sender = EmailSender() @classmethod def get_port(cls) -> int: @@ -150,14 +152,27 @@ class NotificationManager(AppService): return NotificationResult(success=False, message=str(e)) async def _process_immediate(self, message: str) -> bool: - """Process a single notification immediately""" + """Process a single notification immediately, returning whether to put into the failed queue""" try: event = NotificationEventDTO.model_validate_json(message) parsed_event = NotificationEventModel[ get_data_type(event.type) ].model_validate_json(message) - # Implementation of actual notification sending would go here - # self.email_sender.send_templated(event.type, event.user_id, parsed_event) + user_email = get_db_client().get_user_email_by_id(event.user_id) + should_send = ( + get_db_client() + .get_user_notification_preference(event.user_id) + .preferences[event.type] + ) + if not user_email: + logger.error(f"User email not found for user {event.user_id}") + return False + if not should_send: + logger.debug( + f"User {event.user_id} does not want to receive {event.type} notifications" + ) + return True + self.email_sender.send_templated(event.type, user_email, parsed_event) logger.info(f"Processing notification: {parsed_event}") return True except Exception as e: diff --git a/autogpt_platform/backend/backend/notifications/templates/agent_run.html.jinja2 b/autogpt_platform/backend/backend/notifications/templates/agent_run.html.jinja2 new file mode 100644 index 0000000000..f4416fa9ff --- /dev/null +++ b/autogpt_platform/backend/backend/notifications/templates/agent_run.html.jinja2 @@ -0,0 +1,75 @@ +{# Agent Run #} +{# Template variables: +data.name: the name of the agent +data.credits_used: the number of credits used by the agent +data.node_count: the number of nodes the agent ran on +data.execution_time: the time it took to run the agent +data.graph_id: the id of the graph the agent ran on +data.outputs: the dict[str, Any] of outputs of the agent +#} +

+ Hi, +

+

+ We've run your agent {{ data.name }} and it took {{ data.execution_time }} seconds to complete. +

+

+ It ran on {{ data.node_count }} nodes and used {{ data.credits_used }} credits. +

+ +

+ Your feedback has been instrumental in shaping AutoGPT, and we couldn't have + done it without you. We look forward to continuing this journey together as we + bring AI-powered automation to the world. +

+

+ Thank you again for your time and support. +

\ No newline at end of file diff --git a/autogpt_platform/backend/backend/notifications/templates/base.html.jinja2 b/autogpt_platform/backend/backend/notifications/templates/base.html.jinja2 new file mode 100644 index 0000000000..3ed4035f83 --- /dev/null +++ b/autogpt_platform/backend/backend/notifications/templates/base.html.jinja2 @@ -0,0 +1,352 @@ +{# Base Template #} +{# Template variables: + data.message: the message to display in the email + data.title: the title of the email + data.unsubscribe_link: the link to unsubscribe from the email +#} + + + + + + + + + + + + + + + {{data.title}} + + + +
+ + + + + +
+ + + + + +
+ + + + + + + + +
+
+ + + + +
+ +
+
+ + + + + + +
+ {{data.message|safe}} +
+ + + + + + +
+ + + + + + +
+ + +

+ John Ababseh
Product Manager
+ john.ababseh@agpt.co +

+
+
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + +
+ + + + + + + + +
+ + x + + + + discord + + + + website + +
+
+
+
+ AutoGPT +
+
+

+ 3rd Floor 1 Ashley Road, Cheshire, United Kingdom, WA14 2DT, Altrincham
United Kingdom +

+
+

+ You received this email because you signed up on our website.

+
+

+ Unsubscribe +

+
+
+
+
+
+ + + \ No newline at end of file diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py index 7eead6f735..5e50a26d63 100644 --- a/autogpt_platform/backend/backend/util/settings.py +++ b/autogpt_platform/backend/backend/util/settings.py @@ -186,6 +186,11 @@ class Config(UpdateTrackingModel["Config"], BaseSettings): description="The vhost for the RabbitMQ server", ) + postmark_sender_email: str = Field( + default="invalid@invalid.com", + description="The email address to use for sending emails", + ) + @field_validator("platform_base_url", "frontend_base_url") @classmethod def validate_platform_base_url(cls, v: str, info: ValidationInfo) -> str: @@ -282,6 +287,10 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings): default="", description="RabbitMQ default password" ) + postmark_server_api_token: str = Field( + default="", description="Postmark server API token used for sending emails" + ) + # OAuth server credentials for integrations # --8<-- [start:OAuthServerCredentialsExample] github_client_id: str = Field(default="", description="GitHub OAuth client ID") diff --git a/autogpt_platform/backend/backend/util/text.py b/autogpt_platform/backend/backend/util/text.py index c867b7a40f..04ac5a0942 100644 --- a/autogpt_platform/backend/backend/util/text.py +++ b/autogpt_platform/backend/backend/util/text.py @@ -1,17 +1,69 @@ +import logging + +import bleach from jinja2 import BaseLoader from jinja2.sandbox import SandboxedEnvironment +from markupsafe import Markup + +logger = logging.getLogger(__name__) class TextFormatter: def __init__(self): - # Create a sandboxed environment self.env = SandboxedEnvironment(loader=BaseLoader(), autoescape=True) - - # Clear any registered filters, tests, and globals to minimize attack surface self.env.filters.clear() self.env.tests.clear() self.env.globals.clear() + self.allowed_tags = ["p", "b", "i", "u", "ul", "li", "br", "strong", "em"] + self.allowed_attributes = {"*": ["style", "class"]} + def format_string(self, template_str: str, values=None, **kwargs) -> str: + """Regular template rendering with escaping""" template = self.env.from_string(template_str) return template.render(values or {}, **kwargs) + + def format_email( + self, + subject_template: str, + base_template: str, + content_template: str, + data=None, + **kwargs, + ) -> tuple[str, str]: + """ + Special handling for email templates where content needs to be rendered as HTML + """ + # First render the content template + content = self.format_string(content_template, data, **kwargs) + + # Clean the HTML but don't escape it + clean_content = bleach.clean( + content, + tags=self.allowed_tags, + attributes=self.allowed_attributes, + strip=True, + ) + + # Mark the cleaned HTML as safe using Markup + safe_content = Markup(clean_content) + + rendered_subject_template = self.format_string(subject_template, data, **kwargs) + + # Create new env just for HTML template + html_env = SandboxedEnvironment(loader=BaseLoader(), autoescape=True) + html_env.filters["safe"] = lambda x: ( + x if isinstance(x, Markup) else Markup(str(x)) + ) + + # Render base template with the safe content + template = html_env.from_string(base_template) + rendered_base_template = template.render( + data={ + "message": safe_content, + "title": rendered_subject_template, + "unsubscribe_link": kwargs.get("unsubscribe_link", ""), + } + ) + + return rendered_subject_template, rendered_base_template diff --git a/autogpt_platform/backend/poetry.lock b/autogpt_platform/backend/poetry.lock index cf1e3c977b..f1ddaac2fa 100644 --- a/autogpt_platform/backend/poetry.lock +++ b/autogpt_platform/backend/poetry.lock @@ -365,6 +365,24 @@ d = ["aiohttp (>=3.10)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "bleach" +version = "6.2.0" +description = "An easy safelist-based HTML-sanitizing tool." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "bleach-6.2.0-py3-none-any.whl", hash = "sha256:117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e"}, + {file = "bleach-6.2.0.tar.gz", hash = "sha256:123e894118b8a599fd80d3ec1a6d4cc7ce4e5882b1317a7e1ba69b56e95f991f"}, +] + +[package.dependencies] +webencodings = "*" + +[package.extras] +css = ["tinycss2 (>=1.1.0,<1.5)"] + [[package]] name = "cachetools" version = "5.5.1" @@ -2870,6 +2888,21 @@ langchain = ["langchain (>=0.2.0)"] sentry = ["django", "sentry-sdk"] test = ["coverage", "django", "flake8", "freezegun (==0.3.15)", "langchain-community (>=0.2.0)", "langchain-openai (>=0.2.0)", "mock (>=2.0.0)", "pylint", "pytest", "pytest-asyncio", "pytest-timeout"] +[[package]] +name = "postmarker" +version = "1.0" +description = "Python client library for Postmark API" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "postmarker-1.0-py3-none-any.whl", hash = "sha256:0fa49f236c7193650896cbf31bbfac34043e352574c6c7e3e2ad2b954704f064"}, + {file = "postmarker-1.0.tar.gz", hash = "sha256:e735303fdf8ede667a1c6e64a95a96e97f0dabbeca726d0ae1f066bdd799fe34"}, +] + +[package.dependencies] +requests = ">=2.20.0" + [[package]] name = "praw" version = "7.8.1" @@ -4784,6 +4817,18 @@ files = [ [package.dependencies] anyio = ">=3.0.0" +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] + [[package]] name = "websocket-client" version = "1.8.0" @@ -5122,4 +5167,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "4052d96f95ad3dbf8bef4d651168f6df1ef21c506f152ddca119ad8f23caf159" +content-hash = "fd396e770353328ac3fe71aaba5b73b5a147dd445e4866e5328d6173a13bf7a3" diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.toml index e9dfb039d4..7c5a49e37c 100644 --- a/autogpt_platform/backend/pyproject.toml +++ b/autogpt_platform/backend/pyproject.toml @@ -13,6 +13,7 @@ aio-pika = "^9.5.4" anthropic = "^0.45.2" apscheduler = "^3.11.0" autogpt-libs = { path = "../autogpt_libs", develop = true } +bleach = "^6.2.0" click = "^8.1.7" cryptography = "^43.0" discord-py = "^2.4.0" @@ -36,6 +37,7 @@ ollama = "^0.4.1" openai = "^1.61.1" pika = "^1.3.2" pinecone = "^5.3.1" +postmarker = "^1.0" praw = "~7.8.1" prisma = "^0.15.0" psutil = "^6.1.0"