Compare commits

...

6 Commits

Author SHA1 Message Date
Robert Brennan 7c84c361e6 empty commit 2025-01-02 14:40:13 -05:00
Chuck eb7d984e6d Merge branch 'main' into aws 2025-01-02 13:54:16 -05:00
Chuck b47ff0d527 merge main branch into aws 2025-01-02 13:48:15 -05:00
Chuck 2d53738ba8 Fix string formatting 2024-12-28 14:07:54 -05:00
Chuck 4b65feef9f Change flle_store_path to file_store_location 2024-12-28 13:56:11 -05:00
Chuck f5a434e413 Fix s3 bucket reference 2024-12-27 11:11:27 -05:00
22 changed files with 162 additions and 48 deletions
+1 -1
View File
@@ -37,7 +37,7 @@ workspace_base = "./workspace"
#trajectories_path="./trajectories"
# File store path
#file_store_path = "/tmp/file_store"
#file_store_location = "/tmp/file_store"
# File store type
#file_store = "memory"
+2 -2
View File
@@ -43,8 +43,8 @@ ENV WORKSPACE_BASE=/opt/workspace_base
ENV OPENHANDS_BUILD_VERSION=$OPENHANDS_BUILD_VERSION
ENV SANDBOX_USER_ID=0
ENV FILE_STORE=local
ENV FILE_STORE_PATH=/.openhands-state
RUN mkdir -p $FILE_STORE_PATH
ENV FILE_STORE_LOCATION=/.openhands-state
RUN mkdir -p $FILE_STORE_LOCATION
RUN mkdir -p $WORKSPACE_BASE
RUN apt-get update -y \
@@ -100,7 +100,7 @@ Les options de configuration de base sont définies dans la section `[core]` du
- Description : Chemin pour stocker les trajectoires (peut être un dossier ou un fichier). Si c'est un dossier, les trajectoires seront enregistrées dans un fichier nommé avec l'ID de session et l'extension .json, dans ce dossier.
**Stockage de fichiers**
- `file_store_path`
- `file_store_location`
- Type : `str`
- Valeur par défaut : `"/tmp/file_store"`
- Description : Chemin de stockage des fichiers
@@ -13,6 +13,6 @@ Dans le fichier `config.toml`, spécifiez ce qui suit :
[core]
...
file_store="local"
file_store_path="/absolute/path/to/openhands/cache/directory"
file_store_location="/absolute/path/to/openhands/cache/directory"
jwt_secret="secretpass"
```
@@ -97,7 +97,7 @@
- 描述: 存储轨迹的路径(可以是文件夹或文件)。如果是文件夹,轨迹将保存在该文件夹中以会话 ID 命名的 .json 文件中。
**文件存储**
- `file_store_path`
- `file_store_location`
- 类型: `str`
- 默认值: `"/tmp/file_store"`
- 描述: 文件存储路径
@@ -12,6 +12,6 @@
[core]
...
file_store="local"
file_store_path="/absolute/path/to/openhands/cache/directory"
file_store_location="/absolute/path/to/openhands/cache/directory"
jwt_secret="secretpass"
```
+1 -1
View File
@@ -98,7 +98,7 @@ The core configuration options are defined in the `[core]` section of the `confi
- Description: Path to store trajectories (can be a folder or a file). If it's a folder, the trajectories will be saved in a file named with the session id name and .json extension, in that folder.
**File Store**
- `file_store_path`
- `file_store_location`
- Type: `str`
- Default: `"/tmp/file_store"`
- Description: File store path
@@ -11,6 +11,6 @@ In the `config.toml` file, specify the following:
[core]
...
file_store="local"
file_store_path="/absolute/path/to/openhands/cache/directory"
file_store_location="/absolute/path/to/openhands/cache/directory"
jwt_secret="secretpass"
```
+1 -1
View File
@@ -122,7 +122,7 @@ async def main(loop):
config=agent_config,
)
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
event_stream = EventStream(sid, file_store)
runtime_cls = get_runtime_cls(config.runtime)
+2 -2
View File
@@ -26,7 +26,7 @@ class AppConfig:
sandbox: Sandbox configuration settings.
runtime: Runtime environment identifier.
file_store: Type of file store to use.
file_store_path: Path to the file store.
file_store_location: Path to the file store.
trajectories_path: Folder path to store trajectories.
workspace_base: Base path for the workspace. Defaults to `./workspace` as absolute path.
workspace_mount_path: Path to mount the workspace. Defaults to `workspace_base`.
@@ -51,7 +51,7 @@ class AppConfig:
security: SecurityConfig = field(default_factory=SecurityConfig)
runtime: str = 'docker'
file_store: str = 'memory'
file_store_path: str = '/tmp/file_store'
file_store_location: str = '/tmp/file_store'
trajectories_path: str | None = None
workspace_base: str | None = None
workspace_mount_path: str | None = None
+1 -1
View File
@@ -239,7 +239,7 @@ def finalize_config(cfg: AppConfig):
if not cfg.jwt_secret:
cfg.jwt_secret = get_or_create_jwt_secret(
get_file_store(cfg.file_store, cfg.file_store_path)
get_file_store(cfg.file_store, cfg.file_store_location)
)
+1 -1
View File
@@ -70,7 +70,7 @@ def create_runtime(
session_id = sid or generate_sid(config)
# set up the event stream
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
event_stream = EventStream(session_id, file_store)
# agent class
+1 -1
View File
@@ -12,7 +12,7 @@ load_dotenv()
config = load_app_config()
openhands_config = load_openhands_config()
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
client_manager = None
redis_host = os.environ.get('REDIS_HOST')
+11 -6
View File
@@ -2,16 +2,21 @@ from openhands.storage.files import FileStore
from openhands.storage.google_cloud import GoogleCloudFileStore
from openhands.storage.local import LocalFileStore
from openhands.storage.memory import InMemoryFileStore
from openhands.storage.minio import MinioFileStore
from openhands.storage.s3 import S3FileStore
def get_file_store(file_store: str, file_store_path: str | None = None) -> FileStore:
def get_file_store(
file_store: str, file_store_location: str | None = None
) -> FileStore:
if file_store == 'local':
if file_store_path is None:
raise ValueError('file_store_path is required for local file store')
return LocalFileStore(file_store_path)
if file_store_location is None:
raise ValueError('file_store_location is required for local file store')
return LocalFileStore(file_store_location)
elif file_store == 'minio':
return MinioFileStore()
elif file_store == 's3':
return S3FileStore()
return S3FileStore(file_store_location)
elif file_store == 'google_cloud':
return GoogleCloudFileStore(file_store_path)
return GoogleCloudFileStore(file_store_location)
return InMemoryFileStore()
@@ -39,5 +39,5 @@ class FileConversationStore(ConversationStore):
@classmethod
async def get_instance(cls, config: AppConfig, token: str | None):
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
return FileConversationStore(file_store)
+46
View File
@@ -0,0 +1,46 @@
import io
import os
from minio import Minio
from openhands.storage.files import FileStore
class MinioFileStore(FileStore):
def __init__(self) -> None:
access_key = os.getenv('AWS_ACCESS_KEY_ID')
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
endpoint = os.getenv('AWS_S3_ENDPOINT', 's3.amazonaws.com')
secure = os.getenv('AWS_S3_SECURE', 'true').lower() == 'true'
self.bucket = os.getenv('AWS_S3_BUCKET')
self.client = Minio(endpoint, access_key, secret_key, secure=secure)
def write(self, path: str, contents: str) -> None:
as_bytes = contents.encode('utf-8')
stream = io.BytesIO(as_bytes)
try:
self.client.put_object(self.bucket, path, stream, len(as_bytes))
except Exception as e:
raise FileNotFoundError(f'Failed to write to S3 at path {path}: {e}')
def read(self, path: str) -> str:
try:
return self.client.get_object(self.bucket, path).data.decode('utf-8')
except Exception as e:
raise FileNotFoundError(f'Failed to read from S3 at path {path}: {e}')
def list(self, path: str) -> list[str]:
if path and path != '/' and not path.endswith('/'):
path += '/'
try:
return [
obj.object_name for obj in self.client.list_objects(self.bucket, path)
]
except Exception as e:
raise FileNotFoundError(f'Failed to list S3 objects at path {path}: {e}')
def delete(self, path: str) -> None:
try:
self.client.remove_object(self.bucket, path)
except Exception as e:
raise FileNotFoundError(f'Failed to delete S3 object at path {path}: {e}')
+83 -22
View File
@@ -1,46 +1,107 @@
import io
import os
from typing import Optional
from minio import Minio
import boto3
import botocore
from openhands.storage.files import FileStore
class S3FileStore(FileStore):
def __init__(self) -> None:
access_key = os.getenv('AWS_ACCESS_KEY_ID')
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
endpoint = os.getenv('AWS_S3_ENDPOINT', 's3.amazonaws.com')
secure = os.getenv('AWS_S3_SECURE', 'true').lower() == 'true'
self.bucket = os.getenv('AWS_S3_BUCKET')
self.client = Minio(endpoint, access_key, secret_key, secure=secure)
def __init__(self, bucket_name: Optional[str] = None) -> None:
if bucket_name is None:
bucket_name = os.environ['FILE_STORE_BUCKET']
self.bucket = bucket_name
self.client = boto3.client('s3')
def write(self, path: str, contents: str) -> None:
as_bytes = contents.encode('utf-8')
stream = io.BytesIO(as_bytes)
try:
self.client.put_object(self.bucket, path, stream, len(as_bytes))
except Exception as e:
raise FileNotFoundError(f'Failed to write to S3 at path {path}: {e}')
self.client.put_object(Bucket=self.bucket, Key=path, Body=contents)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
raise FileNotFoundError(
f"Error: Access denied to bucket '{self.bucket}'."
)
elif e.response['Error']['Code'] == 'NoSuchBucket':
raise FileNotFoundError(
f"Error: The bucket '{self.bucket}' does not exist."
)
raise FileNotFoundError(
f"Error: Failed to write to bucket '{self.bucket}' at path {path}: {e}"
)
def read(self, path: str) -> str:
try:
return self.client.get_object(self.bucket, path).data.decode('utf-8')
response = self.client.get_object(Bucket=self.bucket, Key=path)
return response['Body'].read().decode('utf-8')
except botocore.exceptions.ClientError as e:
# Catch all S3-related errors
if e.response['Error']['Code'] == 'NoSuchBucket':
raise FileNotFoundError(
f"Error: The bucket '{self.bucket}' does not exist."
)
elif e.response['Error']['Code'] == 'NoSuchKey':
raise FileNotFoundError(
f"Error: The object key '{path}' does not exist in bucket '{self.bucket}'."
)
else:
raise FileNotFoundError(
f"Error: Failed to read from bucket '{self.bucket}' at path {path}: {e}"
)
except Exception as e:
raise FileNotFoundError(f'Failed to read from S3 at path {path}: {e}')
raise FileNotFoundError(
f"Error: Failed to read from bucket '{self.bucket}' at path {path}: {e}"
)
def list(self, path: str) -> list[str]:
if path and path != '/' and not path.endswith('/'):
path += '/'
try:
return [
obj.object_name for obj in self.client.list_objects(self.bucket, path)
]
response = self.client.list_objects_v2(Bucket=self.bucket, Prefix=path)
# Check if 'Contents' exists in the response
if 'Contents' in response:
objects = [obj['Key'] for obj in response['Contents']]
return objects
else:
return list()
except botocore.exceptions.ClientError as e:
# Catch all S3-related errors
if e.response['Error']['Code'] == 'NoSuchBucket':
raise FileNotFoundError(
f"Error: The bucket '{self.bucket}' does not exist."
)
elif e.response['Error']['Code'] == 'AccessDenied':
raise FileNotFoundError(
f"Error: Access denied to bucket '{self.bucket}'."
)
else:
raise FileNotFoundError(f"Error: {e.response['Error']['Message']}")
except Exception as e:
raise FileNotFoundError(f'Failed to list S3 objects at path {path}: {e}')
raise FileNotFoundError(
f"Error: Failed to read from bucket '{self.bucket}' at path {path}: {e}"
)
def delete(self, path: str) -> None:
try:
self.client.remove_object(self.bucket, path)
self.client.delete_object(Bucket=self.bucket, Key=path)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucket':
raise FileNotFoundError(
f"Error: The bucket '{self.bucket}' does not exist."
)
elif e.response['Error']['Code'] == 'AccessDenied':
raise FileNotFoundError(
f"Error: Access denied to bucket '{self.bucket}'."
)
elif e.response['Error']['Code'] == 'NoSuchKey':
raise FileNotFoundError(
f"Error: The object key '{path}' does not exist in bucket '{self.bucket}'."
)
else:
raise FileNotFoundError(
f"Error: Failed to delete key '{path}' from bucket '{self.bucket}': {e}"
)
except Exception as e:
raise FileNotFoundError(f'Failed to delete S3 object at path {path}: {e}')
raise FileNotFoundError(
f"Error: Failed to delete key '{path}' from bucket '{self.bucket}: {e}"
)
@@ -31,5 +31,5 @@ class FileSettingsStore(SettingsStore):
@classmethod
async def get_instance(cls, config: AppConfig, token: str | None):
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
return FileSettingsStore(file_store)
+2
View File
@@ -100,6 +100,7 @@ reportlab = "*"
[tool.coverage.run]
concurrency = ["gevent"]
[tool.poetry.group.runtime.dependencies]
jupyterlab = "*"
notebook = "*"
@@ -129,6 +130,7 @@ ignore = ["D1"]
[tool.ruff.lint.pydocstyle]
convention = "google"
[tool.poetry.group.evaluation.dependencies]
streamlit = "*"
whatthepatch = "*"
+1 -1
View File
@@ -255,7 +255,7 @@ def _load_runtime(
config.sandbox.base_container_image = base_container_image
config.sandbox.runtime_container_image = None
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
event_stream = EventStream(sid, file_store)
runtime = runtime_cls(
+1 -1
View File
@@ -132,7 +132,7 @@ async def test_react_to_exception(mock_agent, mock_event_stream, mock_status_cal
@pytest.mark.asyncio
async def test_run_controller_with_fatal_error(mock_agent, mock_event_stream):
config = AppConfig()
file_store = get_file_store(config.file_store, config.file_store_path)
file_store = get_file_store(config.file_store, config.file_store_location)
event_stream = EventStream(sid='test', file_store=file_store)
agent = MagicMock(spec=Agent)
+1 -1
View File
@@ -66,7 +66,7 @@ async def test_store_and_load_data(file_settings_store):
@pytest.mark.asyncio
async def test_get_instance():
config = AppConfig(file_store='local', file_store_path='/test/path')
config = AppConfig(file_store='local', file_store_location='/test/path')
with patch(
'openhands.storage.settings.file_settings_store.get_file_store'