github/AutoGPT
1
1
Fork 0
mirror of https://github.com/Significant-Gravitas/AutoGPT.git synced 2026-04-08 03:00:28 -04:00
Code Issues Packages Projects Releases Wiki Activity

switch to gcs

Browse Source
This commit is contained in:
SwiftyOS
2024-11-26 11:04:08 +01:00
parent 518f196e6b
commit 9616baf695
4 changed files with 134 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
autogpt_platform/backend/backend/server/v2/store/media.py
View File

@@ -3,10 +3,13 @@ import os
import uuid
import fastapi
import supabase
from google.cloud import storage
import dotenv
import backend.server.v2.store.exceptions
dotenv.load_dotenv()
logger = logging.getLogger(__name__)
ALLOWED_IMAGE_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
@@ -15,14 +18,12 @@ MAX_FILE_SIZE = 50 * 1024 * 1024 # 50MB
async def upload_media(user_id: str, file: fastapi.UploadFile) -> str:
try:
supabase_url = os.environ["SUPABASE_URL"]
supabase_key = os.environ["SUPABASE_KEY"]
except KeyError as e:
logger.error(f"Missing required environment variable: {str(e)}")
# Check required environment variables first before doing any file processing
if not os.environ.get("MEDIA_GCS_BUCKET_NAME"):
logger.error("Missing required GCS environment variables")
raise backend.server.v2.store.exceptions.StorageConfigError(
"Missing storage configuration"
) from e
)
try:
# Validate file type
@@ -66,27 +67,27 @@ async def upload_media(user_id: str, file: fastapi.UploadFile) -> str:
# Construct storage path
media_type = "images" if content_type in ALLOWED_IMAGE_TYPES else "videos"
storage_path = f"store/{user_id}/{media_type}/{unique_filename}"
storage_path = f"users/{user_id}/{media_type}/{unique_filename}"
try:
supabase_client = supabase.create_client(supabase_url, supabase_key)
storage_client = storage.Client()
bucket = storage_client.bucket(os.environ["MEDIA_GCS_BUCKET_NAME"])
blob = bucket.blob(storage_path)
blob.content_type = content_type
file_bytes = await file.read()
supabase_client.storage.from_("media").upload(
path=storage_path,
file=file_bytes,
file_options={"content-type": content_type},
blob.upload_from_string(
file_bytes,
content_type=content_type
)
public_url = supabase_client.storage.from_("media").get_public_url(
storage_path
)
public_url = blob.public_url
logger.info(f"Successfully uploaded file to: {storage_path}")
return public_url
except Exception as e:
logger.error(f"Supabase storage error: {str(e)}")
logger.error(f"GCS storage error: {str(e)}")
raise backend.server.v2.store.exceptions.StorageUploadError(
"Failed to upload file to storage"
) from e

53
autogpt_platform/backend/backend/server/v2/store/media_test.py
View File

@@ -11,29 +11,25 @@ import backend.server.v2.store.media
@pytest.fixture
def mock_env_vars(monkeypatch):
monkeypatch.setenv("SUPABASE_URL", "http://test-url")
monkeypatch.setenv("SUPABASE_KEY", "test-key")
monkeypatch.setenv("GCS_BUCKET_NAME", "test-bucket")
@pytest.fixture
def mock_supabase(mocker):
def mock_storage_client(mocker):
mock_client = unittest.mock.MagicMock()
mock_storage = unittest.mock.MagicMock()
mock_bucket = unittest.mock.MagicMock()
mock_blob = unittest.mock.MagicMock()
mock_storage.from_.return_value = mock_bucket
mock_bucket.upload.return_value = {"Key": "test-key"}
mock_bucket.get_public_url.return_value = "http://test-url/media/test.jpg"
mock_client.bucket.return_value = mock_bucket
mock_bucket.blob.return_value = mock_blob
mock_blob.public_url = "http://test-url/media/test.jpg"
mock_client.storage = mock_storage
mock_client.storage.from_ = unittest.mock.MagicMock(return_value=mock_bucket)
mocker.patch("supabase.create_client", return_value=mock_client)
mocker.patch("google.cloud.storage.Client", return_value=mock_client)
return mock_client
async def test_upload_media_success(mock_env_vars, mock_supabase):
async def test_upload_media_success(mock_env_vars, mock_storage_client):
test_file = fastapi.UploadFile(
filename="test.jpeg",
file=io.BytesIO(b"test data"),
@@ -43,12 +39,12 @@ async def test_upload_media_success(mock_env_vars, mock_supabase):
result = await backend.server.v2.store.media.upload_media("test-user", test_file)
assert result == "http://test-url/media/test.jpg"
mock_bucket = mock_supabase.storage.from_.return_value
mock_bucket.upload.assert_called_once()
mock_bucket.get_public_url.assert_called_once()
mock_bucket = mock_storage_client.bucket.return_value
mock_blob = mock_bucket.blob.return_value
mock_blob.upload_from_string.assert_called_once()
async def test_upload_media_invalid_type(mock_env_vars, mock_supabase):
async def test_upload_media_invalid_type(mock_env_vars, mock_storage_client):
test_file = fastapi.UploadFile(
filename="test.txt",
file=io.BytesIO(b"test data"),
@@ -58,8 +54,9 @@ async def test_upload_media_invalid_type(mock_env_vars, mock_supabase):
with pytest.raises(backend.server.v2.store.exceptions.InvalidFileTypeError):
await backend.server.v2.store.media.upload_media("test-user", test_file)
mock_bucket = mock_supabase.storage.from_.return_value
mock_bucket.upload.assert_not_called()
mock_bucket = mock_storage_client.bucket.return_value
mock_blob = mock_bucket.blob.return_value
mock_blob.upload_from_string.assert_not_called()
async def test_upload_media_missing_credentials():
@@ -73,7 +70,7 @@ async def test_upload_media_missing_credentials():
await backend.server.v2.store.media.upload_media("test-user", test_file)
async def test_upload_media_video_type(mock_env_vars, mock_supabase):
async def test_upload_media_video_type(mock_env_vars, mock_storage_client):
test_file = fastapi.UploadFile(
filename="test.mp4",
file=io.BytesIO(b"test video data"),
@@ -83,6 +80,18 @@ async def test_upload_media_video_type(mock_env_vars, mock_supabase):
result = await backend.server.v2.store.media.upload_media("test-user", test_file)
assert result == "http://test-url/media/test.jpg"
mock_bucket = mock_supabase.storage.from_.return_value
mock_bucket.upload.assert_called_once()
mock_bucket.get_public_url.assert_called_once()
mock_bucket = mock_storage_client.bucket.return_value
mock_blob = mock_bucket.blob.return_value
mock_blob.upload_from_string.assert_called_once()
async def test_upload_media_file_too_large(mock_env_vars, mock_storage_client):
large_data = b"x" * (50 * 1024 * 1024 + 1) # 50MB + 1 byte
test_file = fastapi.UploadFile(
filename="test.jpeg",
file=io.BytesIO(large_data),
headers=starlette.datastructures.Headers({"content-type": "image/jpeg"}),
)
with pytest.raises(backend.server.v2.store.exceptions.FileSizeTooLargeError):
await backend.server.v2.store.media.upload_media("test-user", test_file)

88
autogpt_platform/backend/poetry.lock generated
View File

@@ -921,8 +921,8 @@ grpcio-status = [
{version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},
]
proto-plus = [
{version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""},
{version = ">=1.22.3,<2.0.0dev", markers = "python_version < \"3.13\""},
{version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""},
]
protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0"
requests = ">=2.18.0,<3.0.0.dev0"
@@ -1022,8 +1022,8 @@ files = [
google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev"
proto-plus = [
{version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""},
{version = ">=1.22.3,<2.0.0dev", markers = "python_version < \"3.13\""},
{version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""},
]
protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev"
@@ -1085,6 +1085,86 @@ proto-plus = [
]
protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev"
[[package]]
name = "google-cloud-storage"
version = "2.18.2"
description = "Google Cloud Storage API client library"
optional = false
python-versions = ">=3.7"
files = [
{file = "google_cloud_storage-2.18.2-py2.py3-none-any.whl", hash = "sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166"},
{file = "google_cloud_storage-2.18.2.tar.gz", hash = "sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99"},
]
[package.dependencies]
google-api-core = ">=2.15.0,<3.0.0dev"
google-auth = ">=2.26.1,<3.0dev"
google-cloud-core = ">=2.3.0,<3.0dev"
google-crc32c = ">=1.0,<2.0dev"
google-resumable-media = ">=2.7.2"
requests = ">=2.18.0,<3.0.0dev"
[package.extras]
protobuf = ["protobuf (<6.0.0dev)"]
tracing = ["opentelemetry-api (>=1.1.0)"]
[[package]]
name = "google-crc32c"
version = "1.6.0"
description = "A python wrapper of the C library 'Google CRC32C'"
optional = false
python-versions = ">=3.9"
files = [
{file = "google_crc32c-1.6.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5bcc90b34df28a4b38653c36bb5ada35671ad105c99cfe915fb5bed7ad6924aa"},
{file = "google_crc32c-1.6.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:d9e9913f7bd69e093b81da4535ce27af842e7bf371cde42d1ae9e9bd382dc0e9"},
{file = "google_crc32c-1.6.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a184243544811e4a50d345838a883733461e67578959ac59964e43cca2c791e7"},
{file = "google_crc32c-1.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:236c87a46cdf06384f614e9092b82c05f81bd34b80248021f729396a78e55d7e"},
{file = "google_crc32c-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebab974b1687509e5c973b5c4b8b146683e101e102e17a86bd196ecaa4d099fc"},
{file = "google_crc32c-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:50cf2a96da226dcbff8671233ecf37bf6e95de98b2a2ebadbfdf455e6d05df42"},
{file = "google_crc32c-1.6.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f7a1fc29803712f80879b0806cb83ab24ce62fc8daf0569f2204a0cfd7f68ed4"},
{file = "google_crc32c-1.6.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:40b05ab32a5067525670880eb5d169529089a26fe35dce8891127aeddc1950e8"},
{file = "google_crc32c-1.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9e4b426c3702f3cd23b933436487eb34e01e00327fac20c9aebb68ccf34117d"},
{file = "google_crc32c-1.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51c4f54dd8c6dfeb58d1df5e4f7f97df8abf17a36626a217f169893d1d7f3e9f"},
{file = "google_crc32c-1.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb8b3c75bd157010459b15222c3fd30577042a7060e29d42dabce449c087f2b3"},
{file = "google_crc32c-1.6.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d"},
{file = "google_crc32c-1.6.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b"},
{file = "google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00"},
{file = "google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3"},
{file = "google_crc32c-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760"},
{file = "google_crc32c-1.6.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:e2806553238cd076f0a55bddab37a532b53580e699ed8e5606d0de1f856b5205"},
{file = "google_crc32c-1.6.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:bb0966e1c50d0ef5bc743312cc730b533491d60585a9a08f897274e57c3f70e0"},
{file = "google_crc32c-1.6.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:386122eeaaa76951a8196310432c5b0ef3b53590ef4c317ec7588ec554fec5d2"},
{file = "google_crc32c-1.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2952396dc604544ea7476b33fe87faedc24d666fb0c2d5ac971a2b9576ab871"},
{file = "google_crc32c-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35834855408429cecf495cac67ccbab802de269e948e27478b1e47dfb6465e57"},
{file = "google_crc32c-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:d8797406499f28b5ef791f339594b0b5fdedf54e203b5066675c406ba69d705c"},
{file = "google_crc32c-1.6.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48abd62ca76a2cbe034542ed1b6aee851b6f28aaca4e6551b5599b6f3ef175cc"},
{file = "google_crc32c-1.6.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d"},
{file = "google_crc32c-1.6.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24"},
{file = "google_crc32c-1.6.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91ca8145b060679ec9176e6de4f89b07363d6805bd4760631ef254905503598d"},
{file = "google_crc32c-1.6.0.tar.gz", hash = "sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc"},
]
[package.extras]
testing = ["pytest"]
[[package]]
name = "google-resumable-media"
version = "2.7.2"
description = "Utilities for Google Media Downloads and Resumable Uploads"
optional = false
python-versions = ">=3.7"
files = [
{file = "google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa"},
{file = "google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0"},
]
[package.dependencies]
google-crc32c = ">=1.0,<2.0dev"
[package.extras]
aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"]
requests = ["requests (>=2.18.0,<3.0.0dev)"]
[[package]]
name = "googleapis-common-protos"
version = "1.66.0"
@@ -2049,8 +2129,8 @@ python-dateutil = ">=2.5.3"
tqdm = ">=4.64.1"
typing-extensions = ">=3.7.4"
urllib3 = [
{version = ">=1.26.5", markers = "python_version >= \"3.12\" and python_version < \"4.0\""},
{version = ">=1.26.0", markers = "python_version >= \"3.8\" and python_version < \"3.12\""},
{version = ">=1.26.5", markers = "python_version >= \"3.12\" and python_version < \"4.0\""},
]
[package.extras]
@@ -4125,4 +4205,4 @@ type = ["pytest-mypy"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
content-hash = "0b2bdc34bde2b2c8b81ad382cf8ef7a7b858153b3db76ffb0e7f5ce5e8bcf787"
content-hash = "2a040ac5bd0770db5a5863fe259c82f2207f847b8f722fe81f9b82e61979b5b7"

1
autogpt_platform/backend/pyproject.toml
View File

@@ -50,6 +50,7 @@ cryptography = "^43.0.3"
python-multipart = "^0.0.17"
sqlalchemy = "^2.0.36"
psycopg2-binary = "^2.9.10"
google-cloud-storage = "^2.18.2"
[tool.poetry.group.dev.dependencies]
poethepoet = "^0.30.0"
httpx = "^0.27.0"