feat(server): Add credentials API endpoints (#8024)

- Add two endpoints to OAuth `integrations.py`:
  - `GET /integrations/{provider}/credentials` - list all credentials for a provider, without secrets (metadata only)
   - `GET /integrations/{provider}/credentials/{cred_id}` - retrieve a set of credentials (including secrets)

- Add `username` property to `Credentials` types
   - Add logic to populate `username` in OAuth handlers

- Expand `CredentialsMetaResponse` and remove `credentials_` prefix from properties

- Fix `autogpt_libs` dependency caching issue

- Remove accidentally duplicated OAuth handler files in `autogpt_server/integrations`

rnd/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, SecretStr, field_serializer
 class _BaseCredentials(BaseModel):
     id: str = Field(default_factory=lambda: str(uuid4()))
     provider: str
-    title: str
+    title: Optional[str]
 
     @field_serializer("*")
     def dump_secret_strings(value: Any, _info):
@@ -18,6 +18,8 @@ class _BaseCredentials(BaseModel):
 
 class OAuth2Credentials(_BaseCredentials):
     type: Literal["oauth2"] = "oauth2"
+    username: Optional[str]
+    """Username of the third-party service user that these credentials belong to"""
     access_token: SecretStr
     access_token_expires_at: Optional[int]
     """Unix timestamp (seconds) indicating when the access token expires (if at all)"""

rnd/autogpt_server/autogpt_server/integrations/github.py

@@ -1,99 +0,0 @@
-import time
-from typing import Optional
-from urllib.parse import urlencode
-
-import requests
-from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials
-
-from autogpt_server.integrations.oauth import BaseOAuthHandler
-
-
-class GitHubOAuthHandler(BaseOAuthHandler):
-    """
-    Based on the documentation at:
-    - [Authorizing OAuth apps - GitHub Docs](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps)
-    - [Refreshing user access tokens - GitHub Docs](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/refreshing-user-access-tokens)
-
-    Notes:
-    - By default, token expiration is disabled on GitHub Apps. This means the access
-      token doesn't expire and no refresh token is returned by the authorization flow.
-    - When token expiration gets enabled, any existing tokens will remain non-expiring.
-    - When token expiration gets disabled, token refreshes will return a non-expiring
-      access token *with no refresh token*.
-    """  # noqa
-
-    PROVIDER_NAME = "github"
-
-    def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
-        self.client_id = client_id
-        self.client_secret = client_secret
-        self.redirect_uri = redirect_uri
-        self.auth_base_url = "https://github.com/login/oauth/authorize"
-        self.token_url = "https://github.com/login/oauth/access_token"
-
-    def get_login_url(self, scopes: list[str], state: str) -> str:
-        params = {
-            "client_id": self.client_id,
-            "redirect_uri": self.redirect_uri,
-            "scope": " ".join(scopes),
-            "state": state,
-        }
-        return f"{self.auth_base_url}?{urlencode(params)}"
-
-    def exchange_code_for_tokens(self, code: str) -> OAuth2Credentials:
-        return self._request_tokens({"code": code, "redirect_uri": self.redirect_uri})
-
-    def _refresh_tokens(self, credentials: OAuth2Credentials) -> OAuth2Credentials:
-        if not credentials.refresh_token:
-            return credentials
-
-        return self._request_tokens(
-            {
-                "refresh_token": credentials.refresh_token.get_secret_value(),
-                "grant_type": "refresh_token",
-            }
-        )
-
-    def _request_tokens(
-        self,
-        params: dict[str, str],
-        current_credentials: Optional[OAuth2Credentials] = None,
-    ) -> OAuth2Credentials:
-        request_body = {
-            "client_id": self.client_id,
-            "client_secret": self.client_secret,
-            **params,
-        }
-        headers = {"Accept": "application/json"}
-        response = requests.post(self.token_url, data=request_body, headers=headers)
-        response.raise_for_status()
-        token_data: dict = response.json()
-
-        now = int(time.time())
-        new_credentials = OAuth2Credentials(
-            provider=self.PROVIDER_NAME,
-            title=current_credentials.title if current_credentials else "GitHub",
-            access_token=token_data["access_token"],
-            # Token refresh responses have an empty `scope` property (see docs),
-            # so we have to get the scope from the existing credentials object.
-            scopes=(
-                token_data.get("scope", "").split(",")
-                or (current_credentials.scopes if current_credentials else [])
-            ),
-            # Refresh token and expiration intervals are only given if token expiration
-            # is enabled in the GitHub App's settings.
-            refresh_token=token_data.get("refresh_token"),
-            access_token_expires_at=(
-                now + expires_in
-                if (expires_in := token_data.get("expires_in", None))
-                else None
-            ),
-            refresh_token_expires_at=(
-                now + expires_in
-                if (expires_in := token_data.get("refresh_token_expires_in", None))
-                else None
-            ),
-        )
-        if current_credentials:
-            new_credentials.id = current_credentials.id
-        return new_credentials

rnd/autogpt_server/autogpt_server/integrations/google.py

@@ -1,96 +0,0 @@
-from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials
-from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
-from google_auth_oauthlib.flow import Flow
-from pydantic import SecretStr
-
-from .oauth import BaseOAuthHandler
-
-
-class GoogleOAuthHandler(BaseOAuthHandler):
-    """
-    Based on the documentation at https://developers.google.com/identity/protocols/oauth2/web-server
-    """  # noqa
-
-    PROVIDER_NAME = "google"
-
-    def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
-        self.client_id = client_id
-        self.client_secret = client_secret
-        self.redirect_uri = redirect_uri
-        self.token_uri = "https://oauth2.googleapis.com/token"
-
-    def get_login_url(self, scopes: list[str], state: str) -> str:
-        flow = self._setup_oauth_flow(scopes)
-        flow.redirect_uri = self.redirect_uri
-        authorization_url, _ = flow.authorization_url(
-            access_type="offline",
-            include_granted_scopes="true",
-            state=state,
-            prompt="consent",
-        )
-        return authorization_url
-
-    def exchange_code_for_tokens(self, code: str) -> OAuth2Credentials:
-        flow = self._setup_oauth_flow(None)
-        flow.redirect_uri = self.redirect_uri
-        flow.fetch_token(code=code)
-
-        google_creds = flow.credentials
-        # Google's OAuth library is poorly typed so we need some of these:
-        assert google_creds.token
-        assert google_creds.refresh_token
-        assert google_creds.expiry
-        assert google_creds.scopes
-        return OAuth2Credentials(
-            provider=self.PROVIDER_NAME,
-            title="Google",
-            access_token=SecretStr(google_creds.token),
-            refresh_token=SecretStr(google_creds.refresh_token),
-            access_token_expires_at=int(google_creds.expiry.timestamp()),
-            refresh_token_expires_at=None,
-            scopes=google_creds.scopes,
-        )
-
-    def _refresh_tokens(self, credentials: OAuth2Credentials) -> OAuth2Credentials:
-        # Google credentials should ALWAYS have a refresh token
-        assert credentials.refresh_token
-
-        google_creds = Credentials(
-            token=credentials.access_token.get_secret_value(),
-            refresh_token=credentials.refresh_token.get_secret_value(),
-            token_uri=self.token_uri,
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            scopes=credentials.scopes,
-        )
-        # Google's OAuth library is poorly typed so we need some of these:
-        assert google_creds.refresh_token
-        assert google_creds.scopes
-
-        google_creds.refresh(Request())
-        assert google_creds.expiry
-
-        return OAuth2Credentials(
-            id=credentials.id,
-            provider=self.PROVIDER_NAME,
-            title=credentials.title,
-            access_token=SecretStr(google_creds.token),
-            refresh_token=SecretStr(google_creds.refresh_token),
-            access_token_expires_at=int(google_creds.expiry.timestamp()),
-            refresh_token_expires_at=None,
-            scopes=google_creds.scopes,
-        )
-
-    def _setup_oauth_flow(self, scopes: list[str] | None) -> Flow:
-        return Flow.from_client_config(
-            {
-                "web": {
-                    "client_id": self.client_id,
-                    "client_secret": self.client_secret,
-                    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                    "token_uri": self.token_uri,
-                }
-            },
-            scopes=scopes,
-        )

rnd/autogpt_server/autogpt_server/integrations/notion.py

@@ -1,76 +0,0 @@
-from base64 import b64encode
-from urllib.parse import urlencode
-
-import requests
-from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials
-
-from autogpt_server.integrations.oauth import BaseOAuthHandler
-
-
-class NotionOAuthHandler(BaseOAuthHandler):
-    """
-    Based on the documentation at https://developers.notion.com/docs/authorization
-
-    Notes:
-    - Notion uses non-expiring access tokens and therefore doesn't have a refresh flow
-    - Notion doesn't use scopes
-    """
-
-    PROVIDER_NAME = "notion"
-
-    def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
-        self.client_id = client_id
-        self.client_secret = client_secret
-        self.redirect_uri = redirect_uri
-        self.auth_base_url = "https://api.notion.com/v1/oauth/authorize"
-        self.token_url = "https://api.notion.com/v1/oauth/token"
-
-    def get_login_url(self, scopes: list[str], state: str) -> str:
-        params = {
-            "client_id": self.client_id,
-            "redirect_uri": self.redirect_uri,
-            "response_type": "code",
-            "owner": "user",
-            "state": state,
-        }
-        return f"{self.auth_base_url}?{urlencode(params)}"
-
-    def exchange_code_for_tokens(self, code: str) -> OAuth2Credentials:
-        request_body = {
-            "grant_type": "authorization_code",
-            "code": code,
-            "redirect_uri": self.redirect_uri,
-        }
-        auth_str = b64encode(f"{self.client_id}:{self.client_secret}".encode()).decode()
-        headers = {
-            "Authorization": f"Basic {auth_str}",
-            "Accept": "application/json",
-        }
-        response = requests.post(self.token_url, json=request_body, headers=headers)
-        response.raise_for_status()
-        token_data = response.json()
-
-        return OAuth2Credentials(
-            provider=self.PROVIDER_NAME,
-            title=token_data.get("workspace_name", "Notion"),
-            access_token=token_data["access_token"],
-            refresh_token=None,
-            access_token_expires_at=None,  # Notion tokens don't expire
-            refresh_token_expires_at=None,
-            scopes=[],
-            metadata={
-                "owner": token_data["owner"],
-                "bot_id": token_data["bot_id"],
-                "workspace_id": token_data["workspace_id"],
-                "workspace_name": token_data.get("workspace_name"),
-                "workspace_icon": token_data.get("workspace_icon"),
-            },
-        )
-
-    def _refresh_tokens(self, credentials: OAuth2Credentials) -> OAuth2Credentials:
-        # Notion doesn't support token refresh
-        return credentials
-
-    def needs_refresh(self, credentials: OAuth2Credentials) -> bool:
-        # Notion access tokens don't expire
-        return False

rnd/autogpt_server/autogpt_server/integrations/oauth.py

@@ -1,48 +0,0 @@
-import time
-from abc import ABC, abstractmethod
-from typing import ClassVar
-
-from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials
-
-
-class BaseOAuthHandler(ABC):
-    PROVIDER_NAME: ClassVar[str]
-
-    @abstractmethod
-    def __init__(self, client_id: str, client_secret: str, redirect_uri: str): ...
-
-    @abstractmethod
-    def get_login_url(self, scopes: list[str], state: str) -> str:
-        """Constructs a login URL that the user can be redirected to"""
-        ...
-
-    @abstractmethod
-    def exchange_code_for_tokens(self, code: str) -> OAuth2Credentials:
-        """Exchanges the acquired authorization code from login for a set of tokens"""
-        ...
-
-    @abstractmethod
-    def _refresh_tokens(self, credentials: OAuth2Credentials) -> OAuth2Credentials:
-        """Implements the token refresh mechanism"""
-        ...
-
-    def refresh_tokens(self, credentials: OAuth2Credentials) -> OAuth2Credentials:
-        if credentials.provider != self.PROVIDER_NAME:
-            raise ValueError(
-                f"{self.__class__.__name__} can not refresh tokens "
-                f"for other provider '{credentials.provider}'"
-            )
-        return self._refresh_tokens(credentials)
-
-    def get_access_token(self, credentials: OAuth2Credentials) -> str:
-        """Returns a valid access token, refreshing it first if needed"""
-        if self.needs_refresh(credentials):
-            credentials = self.refresh_tokens(credentials)
-        return credentials.access_token.get_secret_value()
-
-    def needs_refresh(self, credentials: OAuth2Credentials) -> bool:
-        """Indicates whether the given tokens need to be refreshed"""
-        return (
-            credentials.access_token_expires_at is not None
-            and credentials.access_token_expires_at < int(time.time()) + 300
-        )

rnd/autogpt_server/autogpt_server/integrations/oauth/github.py

@@ -23,6 +23,7 @@ class GitHubOAuthHandler(BaseOAuthHandler):
     """  # noqa
 
     PROVIDER_NAME = "github"
+    EMAIL_ENDPOINT = "https://api.github.com/user/emails"
 
     def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
         self.client_id = client_id
@@ -69,10 +70,13 @@ class GitHubOAuthHandler(BaseOAuthHandler):
         response.raise_for_status()
         token_data: dict = response.json()
 
+        username = self._request_username(token_data["access_token"])
+
         now = int(time.time())
         new_credentials = OAuth2Credentials(
             provider=self.PROVIDER_NAME,
-            title=current_credentials.title if current_credentials else "GitHub",
+            title=current_credentials.title if current_credentials else None,
+            username=username,
             access_token=token_data["access_token"],
             # Token refresh responses have an empty `scope` property (see docs),
             # so we have to get the scope from the existing credentials object.
@@ -97,3 +101,19 @@ class GitHubOAuthHandler(BaseOAuthHandler):
         if current_credentials:
             new_credentials.id = current_credentials.id
         return new_credentials
+
+    def _request_username(self, access_token: str) -> str | None:
+        url = "https://api.github.com/user"
+        headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {access_token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+
+        response = requests.get(url, headers=headers)
+
+        if not response.ok:
+            return None
+
+        # Get the login (username)
+        return response.json().get("login")

rnd/autogpt_server/autogpt_server/integrations/oauth/google.py

@@ -1,5 +1,8 @@
 from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials
-from google.auth.transport.requests import Request
+from google.auth.external_account_authorized_user import (
+    Credentials as ExternalAccountCredentials,
+)
+from google.auth.transport.requests import AuthorizedSession, Request
 from google.oauth2.credentials import Credentials
 from google_auth_oauthlib.flow import Flow
 from pydantic import SecretStr
@@ -13,6 +16,7 @@ class GoogleOAuthHandler(BaseOAuthHandler):
     """  # noqa
 
     PROVIDER_NAME = "google"
+    EMAIL_ENDPOINT = "https://www.googleapis.com/oauth2/v2/userinfo"
 
     def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
         self.client_id = client_id
@@ -37,6 +41,8 @@ class GoogleOAuthHandler(BaseOAuthHandler):
         flow.fetch_token(code=code)
 
         google_creds = flow.credentials
+        username = self._request_email(google_creds)
+
         # Google's OAuth library is poorly typed so we need some of these:
         assert google_creds.token
         assert google_creds.refresh_token
@@ -44,7 +50,8 @@ class GoogleOAuthHandler(BaseOAuthHandler):
         assert google_creds.scopes
         return OAuth2Credentials(
             provider=self.PROVIDER_NAME,
-            title="Google",
+            title=None,
+            username=username,
             access_token=SecretStr(google_creds.token),
             refresh_token=SecretStr(google_creds.refresh_token),
             access_token_expires_at=int(google_creds.expiry.timestamp()),
@@ -52,6 +59,15 @@ class GoogleOAuthHandler(BaseOAuthHandler):
             scopes=google_creds.scopes,
         )
 
+    def _request_email(
+        self, creds: Credentials | ExternalAccountCredentials
+    ) -> str | None:
+        session = AuthorizedSession(creds)
+        response = session.get(self.EMAIL_ENDPOINT)
+        if not response.ok:
+            return None
+        return response.json()["email"]
+
     def _refresh_tokens(self, credentials: OAuth2Credentials) -> OAuth2Credentials:
         # Google credentials should ALWAYS have a refresh token
         assert credentials.refresh_token
@@ -72,9 +88,10 @@ class GoogleOAuthHandler(BaseOAuthHandler):
         assert google_creds.expiry
 
         return OAuth2Credentials(
-            id=credentials.id,
             provider=self.PROVIDER_NAME,
+            id=credentials.id,
             title=credentials.title,
+            username=credentials.username,
             access_token=SecretStr(google_creds.token),
             refresh_token=SecretStr(google_creds.refresh_token),
             access_token_expires_at=int(google_creds.expiry.timestamp()),

rnd/autogpt_server/autogpt_server/integrations/oauth/notion.py

@@ -49,10 +49,18 @@ class NotionOAuthHandler(BaseOAuthHandler):
         response = requests.post(self.token_url, json=request_body, headers=headers)
         response.raise_for_status()
         token_data = response.json()
+        # Email is only available for non-bot users
+        email = (
+            token_data["owner"]["person"]["email"]
+            if "person" in token_data["owner"]
+            and "email" in token_data["owner"]["person"]
+            else None
+        )
 
         return OAuth2Credentials(
             provider=self.PROVIDER_NAME,
-            title=token_data.get("workspace_name", "Notion"),
+            title=token_data.get("workspace_name"),
+            username=email,
             access_token=token_data["access_token"],
             refresh_token=None,
             access_token_expires_at=None,  # Notion tokens don't expire

rnd/autogpt_server/autogpt_server/server/integrations.py

@@ -4,6 +4,10 @@ from typing import Annotated, Literal
 from autogpt_libs.supabase_integration_credentials_store import (
     SupabaseIntegrationCredentialsStore,
 )
+from autogpt_libs.supabase_integration_credentials_store.types import (
+    Credentials,
+    OAuth2Credentials,
+)
 from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request
 from pydantic import BaseModel
 from supabase import Client
@@ -48,8 +52,11 @@ async def login(
 
 
 class CredentialsMetaResponse(BaseModel):
-    credentials_id: str
-    credentials_type: Literal["oauth2", "api_key"]
+    id: str
+    type: Literal["oauth2", "api_key"]
+    title: str | None
+    scopes: list[str] | None
+    username: str | None
 
 
 @integrations_api_router.post("/{provider}/callback")
@@ -73,13 +80,53 @@ async def callback(
         logger.warning(f"Code->Token exchange failed for provider {provider}: {e}")
         raise HTTPException(status_code=400, detail=str(e))
 
+    # TODO: Allow specifying `title` to set on `credentials`
     store.add_creds(user_id, credentials)
     return CredentialsMetaResponse(
-        credentials_id=credentials.id,
-        credentials_type=credentials.type,
+        id=credentials.id,
+        type=credentials.type,
+        title=credentials.title,
+        scopes=credentials.scopes,
+        username=credentials.username,
     )
 
 
+@integrations_api_router.get("/{provider}/credentials")
+async def list_credentials(
+    provider: Annotated[str, Path(title="The provider to list credentials for")],
+    user_id: Annotated[str, Depends(get_user_id)],
+    store: Annotated[SupabaseIntegrationCredentialsStore, Depends(get_store)],
+) -> list[CredentialsMetaResponse]:
+    credentials = store.get_creds_by_provider(user_id, provider)
+    return [
+        CredentialsMetaResponse(
+            id=cred.id,
+            type=cred.type,
+            title=cred.title,
+            scopes=cred.scopes if isinstance(cred, OAuth2Credentials) else None,
+            username=cred.username if isinstance(cred, OAuth2Credentials) else None,
+        )
+        for cred in credentials
+    ]
+
+
+@integrations_api_router.get("/{provider}/credentials/{cred_id}")
+async def get_credential(
+    provider: Annotated[str, Path(title="The provider to retrieve credentials for")],
+    cred_id: Annotated[str, Path(title="The ID of the credentials to retrieve")],
+    user_id: Annotated[str, Depends(get_user_id)],
+    store: Annotated[SupabaseIntegrationCredentialsStore, Depends(get_store)],
+) -> Credentials:
+    credential = store.get_creds_by_id(user_id, cred_id)
+    if not credential:
+        raise HTTPException(status_code=404, detail="Credentials not found")
+    if credential.provider != provider:
+        raise HTTPException(
+            status_code=404, detail="Credentials do not match the specified provider"
+        )
+    return credential
+
+
 # -------- UTILITIES --------- #
 
 

rnd/autogpt_server/poetry.lock

@@ -289,7 +289,7 @@ description = "Shared libraries across NextGen AutoGPT"
 optional = false
 python-versions = ">=3.10,<4.0"
 files = []
-develop = false
+develop = true
 
 [package.dependencies]
 colorama = "^0.4.6"
@@ -2022,6 +2022,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs
 optional = false
 python-versions = ">=3.8"
 files = [
+    {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},
     {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},
 ]
 
@@ -2032,6 +2033,7 @@ description = "A collection of ASN.1-based protocols modules"
 optional = false
 python-versions = ">=3.8"
 files = [
+    {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},
     {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},
 ]
 
@@ -3621,4 +3623,4 @@ type = ["pytest-mypy"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "fbc928c40dc95041f7750ab34677fa3eebacd06a84944de900dedd639f847a9c"
+content-hash = "311c527a1d1947af049dac27c7a2b2f49d7fa4cdede52ef436422a528b0ad866"

rnd/autogpt_server/pyproject.toml

@@ -13,7 +13,7 @@ python = "^3.10"
 aio-pika = "^9.4.3"
 anthropic = "^0.25.1"
 apscheduler = "^3.10.4"
-autogpt-libs = { path = "../autogpt_libs" }
+autogpt-libs = { path = "../autogpt_libs", develop = true }
 click = "^8.1.7"
 croniter = "^2.0.5"
 discord-py = "^2.4.0"