mirror of
https://github.com/All-Hands-AI/OpenHands.git
synced 2026-04-29 03:00:45 -04:00
Compare commits
94 Commits
openhands/
...
1.0.3-cli
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b673915837 | ||
|
|
97403dfbdb | ||
|
|
2fc31e96d0 | ||
|
|
6558b4f97d | ||
|
|
12d6da8130 | ||
|
|
38f2728cfa | ||
|
|
fab48fe864 | ||
|
|
a196881ab0 | ||
|
|
ca2c9546ad | ||
|
|
704fc6dd69 | ||
|
|
6630d5dc4e | ||
|
|
0e7fefca7e | ||
|
|
4020448d64 | ||
|
|
2fdd4d084a | ||
|
|
aba5d54a86 | ||
|
|
6710a39621 | ||
|
|
fccc6f3196 | ||
|
|
7447cfdb3d | ||
|
|
297af05d53 | ||
|
|
b8f387df94 | ||
|
|
fc67f39b74 | ||
|
|
bc8922d3f9 | ||
|
|
37d58bba4d | ||
|
|
037a2dca8f | ||
|
|
b5920eece6 | ||
|
|
a81bef8cdf | ||
|
|
450aa3b527 | ||
|
|
4decd8b3e9 | ||
|
|
818f743dc7 | ||
|
|
f402371b27 | ||
|
|
92b1fca719 | ||
|
|
8de13457c3 | ||
|
|
8f94b68ea1 | ||
|
|
eb616dfae4 | ||
|
|
26c636d63e | ||
|
|
3ec8d70d04 | ||
|
|
694ac74bb9 | ||
|
|
7ee20067a8 | ||
|
|
054c5b666f | ||
|
|
0ff7329424 | ||
|
|
86c590cdc3 | ||
|
|
319677e629 | ||
|
|
f8b566b858 | ||
|
|
f9694858fb | ||
|
|
7880c39ede | ||
|
|
b5e00f577c | ||
|
|
2631294e79 | ||
|
|
47776ae2ad | ||
|
|
0ad411e162 | ||
|
|
7bc56e0d74 | ||
|
|
e450a3a603 | ||
|
|
17e32af6fe | ||
|
|
4b303ec9b4 | ||
|
|
eb954164a5 | ||
|
|
0c1c2163b1 | ||
|
|
dd2a62c992 | ||
|
|
f3d9faef34 | ||
|
|
134c122026 | ||
|
|
523b40dbfc | ||
|
|
6a5b915088 | ||
|
|
a5c5133961 | ||
|
|
eea1e7f4e1 | ||
|
|
e2d990f3a0 | ||
|
|
f258eafa37 | ||
|
|
19634f364e | ||
|
|
aa6446038c | ||
|
|
dbddc1868e | ||
|
|
cd967ef4bc | ||
|
|
e34c13ea3c | ||
|
|
1f35a73cc4 | ||
|
|
267528fa82 | ||
|
|
49f360d021 | ||
|
|
9520da668c | ||
|
|
9d19292619 | ||
|
|
fc9a87550d | ||
|
|
490d3dba10 | ||
|
|
5ed1dde2e9 | ||
|
|
a68576b876 | ||
|
|
722124ae83 | ||
|
|
44578664ed | ||
|
|
9efe6eb776 | ||
|
|
6d137e883f | ||
|
|
2889f736d9 | ||
|
|
531683abae | ||
|
|
fab64a51b7 | ||
|
|
cc18a18874 | ||
|
|
7525a95af0 | ||
|
|
640f50d525 | ||
|
|
6f2f85073d | ||
|
|
9f3b2425ec | ||
|
|
1ebc3ab04e | ||
|
|
9bd0566e4e | ||
|
|
d82972e126 | ||
|
|
e1b94732a8 |
33
.github/pull_request_template.md
vendored
33
.github/pull_request_template.md
vendored
@@ -1,12 +1,31 @@
|
||||
- [ ] This change is worth documenting at https://docs.all-hands.dev/
|
||||
- [ ] Include this change in the Release Notes. If checked, you **must** provide an **end-user friendly** description for your change below
|
||||
## Summary of PR
|
||||
|
||||
**End-user friendly description of the problem this fixes or functionality this introduces.**
|
||||
<!-- Summarize what the PR does, explaining any non-trivial design decisions. -->
|
||||
|
||||
## Change Type
|
||||
|
||||
---
|
||||
**Summarize what the PR does, explaining any non-trivial design decisions.**
|
||||
<!-- Choose the types that apply to your PR and remove the rest. -->
|
||||
|
||||
- [ ] Bug fix
|
||||
- [ ] New feature
|
||||
- [ ] Breaking change
|
||||
- [ ] Refactor
|
||||
- [ ] Other (dependency update, docs, typo fixes, etc.)
|
||||
|
||||
---
|
||||
**Link of any specific issues this addresses:**
|
||||
## Checklist
|
||||
|
||||
- [ ] I have read and reviewed the code and I understand what the code is doing.
|
||||
- [ ] I have tested the code to the best of my ability and ensured it works as expected.
|
||||
|
||||
## Fixes
|
||||
|
||||
<!-- If this resolves an issue, link it here so it will close automatically upon merge. -->
|
||||
|
||||
Resolves #(issue)
|
||||
|
||||
## Release Notes
|
||||
|
||||
<!-- Check the box if this change is worth adding to the release notes. If checked, you must provide an
|
||||
end-user friendly description for your change below the checkbox. -->
|
||||
|
||||
- [ ] Include this change in the Release Notes.
|
||||
|
||||
6
.github/scripts/update_pr_description.sh
vendored
6
.github/scripts/update_pr_description.sh
vendored
@@ -13,12 +13,12 @@ DOCKER_RUN_COMMAND="docker run -it --rm \
|
||||
-p 3000:3000 \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
-e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:${SHORT_SHA}-nikolaik \
|
||||
-e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.openhands.dev/openhands/runtime:${SHORT_SHA}-nikolaik \
|
||||
--name openhands-app-${SHORT_SHA} \
|
||||
docker.all-hands.dev/all-hands-ai/openhands:${SHORT_SHA}"
|
||||
docker.openhands.dev/openhands/openhands:${SHORT_SHA}"
|
||||
|
||||
# Define the uvx command
|
||||
UVX_RUN_COMMAND="uvx --python 3.12 --from git+https://github.com/All-Hands-AI/OpenHands@${BRANCH_NAME}#subdirectory=openhands-cli openhands"
|
||||
UVX_RUN_COMMAND="uvx --python 3.12 --from git+https://github.com/OpenHands/OpenHands@${BRANCH_NAME}#subdirectory=openhands-cli openhands"
|
||||
|
||||
# Get the current PR body
|
||||
PR_BODY=$(gh pr view "$PR_NUMBER" --json body --jq .body)
|
||||
|
||||
2
.github/workflows/dispatch-to-docs.yml
vendored
2
.github/workflows/dispatch-to-docs.yml
vendored
@@ -12,7 +12,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
repo: ["All-Hands-AI/docs"]
|
||||
repo: ["OpenHands/docs"]
|
||||
steps:
|
||||
- name: Push to docs repo
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
|
||||
2
.github/workflows/enterprise-preview.yml
vendored
2
.github/workflows/enterprise-preview.yml
vendored
@@ -26,4 +26,4 @@ jobs:
|
||||
-H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-d "{\"ref\": \"main\", \"inputs\": {\"openhandsPrNumber\": \"${{ github.event.pull_request.number }}\", \"deployEnvironment\": \"feature\", \"enterpriseImageTag\": \"pr-${{ github.event.pull_request.number }}\" }}" \
|
||||
https://api.github.com/repos/All-Hands-AI/deploy/actions/workflows/deploy.yaml/dispatches
|
||||
https://api.github.com/repos/OpenHands/deploy/actions/workflows/deploy.yaml/dispatches
|
||||
|
||||
6
.github/workflows/ghcr-build.yml
vendored
6
.github/workflows/ghcr-build.yml
vendored
@@ -37,7 +37,6 @@ jobs:
|
||||
shell: bash
|
||||
id: define-base-images
|
||||
run: |
|
||||
# Only build nikolaik on PRs, otherwise build both nikolaik and ubuntu.
|
||||
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
|
||||
json=$(jq -n -c '[
|
||||
{ image: "nikolaik/python-nodejs:python3.12-nodejs22", tag: "nikolaik" },
|
||||
@@ -46,7 +45,6 @@ jobs:
|
||||
else
|
||||
json=$(jq -n -c '[
|
||||
{ image: "nikolaik/python-nodejs:python3.12-nodejs22", tag: "nikolaik" },
|
||||
{ image: "ghcr.io/all-hands-ai/python-nodejs:python3.13-nodejs22-trixie", tag: "trixie" },
|
||||
{ image: "ubuntu:24.04", tag: "ubuntu" }
|
||||
]')
|
||||
fi
|
||||
@@ -200,7 +198,7 @@ jobs:
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/all-hands-ai/enterprise-server
|
||||
images: ghcr.io/openhands/enterprise-server
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
@@ -252,7 +250,7 @@ jobs:
|
||||
-H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-d "{\"ref\": \"main\", \"inputs\": {\"openhandsPrNumber\": \"${{ github.event.pull_request.number }}\", \"deployEnvironment\": \"feature\", \"enterpriseImageTag\": \"pr-${{ github.event.pull_request.number }}\" }}" \
|
||||
https://api.github.com/repos/All-Hands-AI/deploy/actions/workflows/deploy.yaml/dispatches
|
||||
https://api.github.com/repos/OpenHands/deploy/actions/workflows/deploy.yaml/dispatches
|
||||
|
||||
# Run unit tests with the Docker runtime Docker images as root
|
||||
test_runtime_root:
|
||||
|
||||
4
.github/workflows/openhands-resolver.yml
vendored
4
.github/workflows/openhands-resolver.yml
vendored
@@ -201,7 +201,7 @@ jobs:
|
||||
issue_number: ${{ env.ISSUE_NUMBER }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: `[OpenHands](https://github.com/All-Hands-AI/OpenHands) started fixing the ${issueType}! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).`
|
||||
body: `[OpenHands](https://github.com/OpenHands/OpenHands) started fixing the ${issueType}! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).`
|
||||
});
|
||||
|
||||
- name: Install OpenHands
|
||||
@@ -233,7 +233,7 @@ jobs:
|
||||
if (isExperimentalLabel || isIssueCommentExperimental || isReviewCommentExperimental) {
|
||||
console.log("Installing experimental OpenHands...");
|
||||
|
||||
await exec.exec("pip install git+https://github.com/all-hands-ai/openhands.git");
|
||||
await exec.exec("pip install git+https://github.com/openhands/openhands.git");
|
||||
} else {
|
||||
console.log("Installing from requirements.txt...");
|
||||
|
||||
|
||||
2
.github/workflows/run-eval.yml
vendored
2
.github/workflows/run-eval.yml
vendored
@@ -101,7 +101,7 @@ jobs:
|
||||
-H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${{ steps.eval_params.outputs.repo_url }}\", \"github-branch\": \"${{ steps.eval_params.outputs.eval_branch }}\", \"pr-number\": \"${PR_NUMBER}\", \"eval-instances\": \"${{ steps.eval_params.outputs.eval_instances }}\"}}" \
|
||||
https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
|
||||
https://api.github.com/repos/OpenHands/evaluation/actions/workflows/create-branch.yml/dispatches
|
||||
|
||||
# Send Slack message
|
||||
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
||||
|
||||
@@ -83,6 +83,116 @@ VSCode Extension:
|
||||
- Use `vscode.window.createOutputChannel()` for debug logging instead of `showErrorMessage()` popups
|
||||
- Pre-commit process runs both frontend and backend checks when committing extension changes
|
||||
|
||||
## Enterprise Directory
|
||||
|
||||
The `enterprise/` directory contains additional functionality that extends the open-source OpenHands codebase. This includes:
|
||||
- Authentication and user management (Keycloak integration)
|
||||
- Database migrations (Alembic)
|
||||
- Integration services (GitHub, GitLab, Jira, Linear, Slack)
|
||||
- Billing and subscription management (Stripe)
|
||||
- Telemetry and analytics (PostHog, custom metrics framework)
|
||||
|
||||
### Enterprise Development Setup
|
||||
|
||||
**Prerequisites:**
|
||||
- Python 3.12
|
||||
- Poetry (for dependency management)
|
||||
- Node.js 22.x (for frontend)
|
||||
- Docker (optional)
|
||||
|
||||
**Setup Steps:**
|
||||
1. First, build the main OpenHands project: `make build`
|
||||
2. Then install enterprise dependencies: `cd enterprise && poetry install --with dev,test` (This can take a very long time. Be patient.)
|
||||
3. Set up enterprise pre-commit hooks: `poetry run pre-commit install --config ./dev_config/python/.pre-commit-config.yaml`
|
||||
|
||||
**Running Enterprise Tests:**
|
||||
```bash
|
||||
# Enterprise unit tests (full suite)
|
||||
PYTHONPATH=".:$PYTHONPATH" poetry run --project=enterprise pytest --forked -n auto -s -p no:ddtrace -p no:ddtrace.pytest_bdd -p no:ddtrace.pytest_benchmark ./enterprise/tests/unit --cov=enterprise --cov-branch
|
||||
|
||||
# Test specific modules (faster for development)
|
||||
cd enterprise
|
||||
PYTHONPATH=".:$PYTHONPATH" poetry run pytest tests/unit/telemetry/ --confcutdir=tests/unit/telemetry
|
||||
|
||||
# Enterprise linting (IMPORTANT: use --show-diff-on-failure to match GitHub CI)
|
||||
poetry run pre-commit run --all-files --show-diff-on-failure --config ./dev_config/python/.pre-commit-config.yaml
|
||||
```
|
||||
|
||||
**Running Enterprise Server:**
|
||||
```bash
|
||||
cd enterprise
|
||||
make start-backend # Development mode with hot reload
|
||||
# or
|
||||
make run # Full application (backend + frontend)
|
||||
```
|
||||
|
||||
**Key Configuration Files:**
|
||||
- `enterprise/pyproject.toml` - Enterprise-specific dependencies
|
||||
- `enterprise/Makefile` - Enterprise build and run commands
|
||||
- `enterprise/dev_config/python/` - Linting and type checking configuration
|
||||
- `enterprise/migrations/` - Database migration files
|
||||
|
||||
**Database Migrations:**
|
||||
Enterprise uses Alembic for database migrations. When making schema changes:
|
||||
1. Create migration files in `enterprise/migrations/versions/`
|
||||
2. Test migrations thoroughly
|
||||
3. The CI will check for migration conflicts on PRs
|
||||
|
||||
**Integration Development:**
|
||||
The enterprise codebase includes integrations for:
|
||||
- **GitHub** - PR management, webhooks, app installations
|
||||
- **GitLab** - Similar to GitHub but for GitLab instances
|
||||
- **Jira** - Issue tracking and project management
|
||||
- **Linear** - Modern issue tracking
|
||||
- **Slack** - Team communication and notifications
|
||||
|
||||
Each integration follows a consistent pattern with service classes, storage models, and API endpoints.
|
||||
|
||||
**Important Notes:**
|
||||
- Enterprise code is licensed under Polyform Free Trial License (30-day limit)
|
||||
- The enterprise server extends the OSS server through dynamic imports
|
||||
- Database changes require careful migration planning in `enterprise/migrations/`
|
||||
- Always test changes in both OSS and enterprise contexts
|
||||
- Use the enterprise-specific Makefile commands for development
|
||||
|
||||
**Enterprise Testing Best Practices:**
|
||||
|
||||
**Database Testing:**
|
||||
- Use SQLite in-memory databases (`sqlite:///:memory:`) for unit tests instead of real PostgreSQL
|
||||
- Create module-specific `conftest.py` files with database fixtures
|
||||
- Mock external database connections in unit tests to avoid dependency on running services
|
||||
- Use real database connections only for integration tests
|
||||
|
||||
**Import Patterns:**
|
||||
- Use relative imports without `enterprise.` prefix in enterprise code
|
||||
- Example: `from storage.database import session_maker` not `from enterprise.storage.database import session_maker`
|
||||
- This ensures code works in both OSS and enterprise contexts
|
||||
|
||||
**Test Structure:**
|
||||
- Place tests in `enterprise/tests/unit/` following the same structure as the source code
|
||||
- Use `--confcutdir=tests/unit/[module]` when testing specific modules
|
||||
- Create comprehensive fixtures for complex objects (databases, external services)
|
||||
- Write platform-agnostic tests (avoid hardcoded OS-specific assertions)
|
||||
|
||||
**Mocking Strategy:**
|
||||
- Use `AsyncMock` for async operations and `MagicMock` for complex objects
|
||||
- Mock all external dependencies (databases, APIs, file systems) in unit tests
|
||||
- Use `patch` with correct import paths (e.g., `telemetry.registry.logger` not `enterprise.telemetry.registry.logger`)
|
||||
- Test both success and failure scenarios with proper error handling
|
||||
|
||||
**Coverage Goals:**
|
||||
- Aim for 90%+ test coverage on new enterprise modules
|
||||
- Focus on critical business logic and error handling paths
|
||||
- Use `--cov-report=term-missing` to identify uncovered lines
|
||||
|
||||
**Troubleshooting:**
|
||||
- If tests fail, ensure all dependencies are installed: `poetry install --with dev,test`
|
||||
- For database issues, check migration status and run migrations if needed
|
||||
- For frontend issues, ensure the main OpenHands frontend is built: `make build`
|
||||
- Check logs in the `logs/` directory for runtime issues
|
||||
- If tests fail with import errors, verify `PYTHONPATH=".:$PYTHONPATH"` is set
|
||||
- **If GitHub CI fails but local linting passes**: Always use `--show-diff-on-failure` flag to match CI behavior exactly
|
||||
|
||||
## Template for Github Pull Request
|
||||
|
||||
If you are starting a pull request (PR), please follow the template in `.github/pull_request_template.md`.
|
||||
|
||||
16
.vscode/settings.json
vendored
16
.vscode/settings.json
vendored
@@ -3,4 +3,20 @@
|
||||
"files.eol": "\n",
|
||||
"files.trimTrailingWhitespace": true,
|
||||
"files.insertFinalNewline": true,
|
||||
|
||||
"python.defaultInterpreterPath": "./.venv/bin/python",
|
||||
"python.terminal.activateEnvironment": true,
|
||||
"python.analysis.autoImportCompletions": true,
|
||||
"python.analysis.autoSearchPaths": true,
|
||||
"python.analysis.extraPaths": [
|
||||
"./.venv/lib/python3.12/site-packages"
|
||||
],
|
||||
"python.analysis.packageIndexDepths": [
|
||||
{
|
||||
"name": "openhands",
|
||||
"depth": 10,
|
||||
"includeAllSymbols": true
|
||||
}
|
||||
],
|
||||
"python.analysis.stubPath": "./.venv/lib/python3.12/site-packages",
|
||||
}
|
||||
|
||||
@@ -124,7 +124,7 @@ These Slack etiquette guidelines are designed to foster an inclusive, respectful
|
||||
- Post questions or discussions in the most relevant channel (e.g., for [slack - #general](https://openhands-ai.slack.com/archives/C06P5NCGSFP) for general topics, [slack - #questions](https://openhands-ai.slack.com/archives/C06U8UTKSAD) for queries/questions.
|
||||
- When asking for help or raising issues, include necessary details like links, screenshots, or clear explanations to provide context.
|
||||
- Keep discussions in public channels whenever possible to allow others to benefit from the conversation, unless the matter is sensitive or private.
|
||||
- Always adhere to [our standards](https://github.com/All-Hands-AI/OpenHands/blob/main/CODE_OF_CONDUCT.md#our-standards) to ensure a welcoming and collaborative environment.
|
||||
- Always adhere to [our standards](https://github.com/OpenHands/OpenHands/blob/main/CODE_OF_CONDUCT.md#our-standards) to ensure a welcoming and collaborative environment.
|
||||
- If you choose to mute a channel, consider setting up alerts for topics that still interest you to stay engaged. For Slack, Go to Settings → Notifications → My Keywords to add specific keywords that will notify you when mentioned. For example, if you're here for discussions about LLMs, mute the channel if it’s too busy, but set notifications to alert you only when “LLMs” appears in messages.
|
||||
|
||||
## Attribution
|
||||
|
||||
@@ -8,7 +8,7 @@ If this resonates with you, we'd love to have you join us in our quest!
|
||||
|
||||
## 🤝 How to Join
|
||||
|
||||
Check out our [How to Join the Community section.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
|
||||
Check out our [How to Join the Community section.](https://github.com/OpenHands/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
|
||||
|
||||
## 💪 Becoming a Contributor
|
||||
|
||||
|
||||
@@ -13,15 +13,15 @@ To understand the codebase, please refer to the README in each module:
|
||||
|
||||
## Setting up Your Development Environment
|
||||
|
||||
We have a separate doc [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) that tells you how to set up a development workflow.
|
||||
We have a separate doc [Development.md](https://github.com/OpenHands/OpenHands/blob/main/Development.md) that tells you how to set up a development workflow.
|
||||
|
||||
## How Can I Contribute?
|
||||
|
||||
There are many ways that you can contribute:
|
||||
|
||||
1. **Download and use** OpenHands, and send [issues](https://github.com/All-Hands-AI/OpenHands/issues) when you encounter something that isn't working or a feature that you'd like to see.
|
||||
1. **Download and use** OpenHands, and send [issues](https://github.com/OpenHands/OpenHands/issues) when you encounter something that isn't working or a feature that you'd like to see.
|
||||
2. **Send feedback** after each session by [clicking the thumbs-up thumbs-down buttons](https://docs.all-hands.dev/usage/feedback), so we can see where things are working and failing, and also build an open dataset for training code agents.
|
||||
3. **Improve the Codebase** by sending [PRs](#sending-pull-requests-to-openhands) (see details below). In particular, we have some [good first issues](https://github.com/All-Hands-AI/OpenHands/labels/good%20first%20issue) that may be ones to start on.
|
||||
3. **Improve the Codebase** by sending [PRs](#sending-pull-requests-to-openhands) (see details below). In particular, we have some [good first issues](https://github.com/OpenHands/OpenHands/labels/good%20first%20issue) that may be ones to start on.
|
||||
|
||||
## What Can I Build?
|
||||
Here are a few ways you can help improve the codebase.
|
||||
@@ -35,7 +35,7 @@ of the application, please open an issue first, or better, join the #eng-ui-ux c
|
||||
to gather consensus from our design team first.
|
||||
|
||||
#### Improving the agent
|
||||
Our main agent is the CodeAct agent. You can [see its prompts here](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/agenthub/codeact_agent).
|
||||
Our main agent is the CodeAct agent. You can [see its prompts here](https://github.com/OpenHands/OpenHands/tree/main/openhands/agenthub/codeact_agent).
|
||||
|
||||
Changes to these prompts, and to the underlying behavior in Python, can have a huge impact on user experience.
|
||||
You can try modifying the prompts to see how they change the behavior of the agent as you use the app
|
||||
@@ -54,7 +54,7 @@ The agent needs a place to run code and commands. When you run OpenHands on your
|
||||
to do this by default. But there are other ways of creating a sandbox for the agent.
|
||||
|
||||
If you work for a company that provides a cloud-based runtime, you could help us add support for that runtime
|
||||
by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/base.py).
|
||||
by implementing the [interface specified here](https://github.com/OpenHands/OpenHands/blob/main/openhands/runtime/base.py).
|
||||
|
||||
#### Testing
|
||||
When you write code, it is also good to write tests. Please navigate to the [`./tests`](./tests) folder to see existing test suites.
|
||||
@@ -84,7 +84,7 @@ For example, a PR title could be:
|
||||
- `refactor: modify package path`
|
||||
- `feat(frontend): xxxx`, where `(frontend)` means that this PR mainly focuses on the frontend component.
|
||||
|
||||
You may also check out previous PRs in the [PR list](https://github.com/All-Hands-AI/OpenHands/pulls).
|
||||
You may also check out previous PRs in the [PR list](https://github.com/OpenHands/OpenHands/pulls).
|
||||
|
||||
### Pull Request description
|
||||
- If your PR is small (such as a typo fix), you can go brief.
|
||||
@@ -97,7 +97,7 @@ please include a short message that we can add to our changelog.
|
||||
|
||||
### Opening Issues
|
||||
|
||||
If you notice any bugs or have any feature requests please open them via the [issues page](https://github.com/All-Hands-AI/OpenHands/issues). We will triage based on how critical the bug is or how potentially useful the improvement is, discuss, and implement the ones that the community has interest/effort for.
|
||||
If you notice any bugs or have any feature requests please open them via the [issues page](https://github.com/OpenHands/OpenHands/issues). We will triage based on how critical the bug is or how potentially useful the improvement is, discuss, and implement the ones that the community has interest/effort for.
|
||||
|
||||
Further, if you see an issue you like, please leave a "thumbs-up" or a comment, which will help us prioritize.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## Contributors
|
||||
|
||||
We would like to thank all the [contributors](https://github.com/All-Hands-AI/OpenHands/graphs/contributors) who have helped make OpenHands possible. We greatly appreciate your dedication and hard work.
|
||||
We would like to thank all the [contributors](https://github.com/OpenHands/OpenHands/graphs/contributors) who have helped make OpenHands possible. We greatly appreciate your dedication and hard work.
|
||||
|
||||
## Open Source Projects
|
||||
|
||||
@@ -14,7 +14,7 @@ OpenHands includes and adapts the following open source projects. We are gratefu
|
||||
|
||||
#### [Aider](https://github.com/paul-gauthier/aider)
|
||||
- License: Apache License 2.0
|
||||
- Description: AI pair programming tool. OpenHands has adapted and integrated its linter module for code-related tasks in [`agentskills utilities`](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/agent_skills/utils/aider)
|
||||
- Description: AI pair programming tool. OpenHands has adapted and integrated its linter module for code-related tasks in [`agentskills utilities`](https://github.com/OpenHands/OpenHands/tree/main/openhands/runtime/plugins/agent_skills/utils/aider)
|
||||
|
||||
#### [BrowserGym](https://github.com/ServiceNow/BrowserGym)
|
||||
- License: Apache License 2.0
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This guide is for people working on OpenHands and editing the source code.
|
||||
If you wish to contribute your changes, check out the
|
||||
[CONTRIBUTING.md](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md)
|
||||
[CONTRIBUTING.md](https://github.com/OpenHands/OpenHands/blob/main/CONTRIBUTING.md)
|
||||
on how to clone and setup the project initially before moving on. Otherwise,
|
||||
you can clone the OpenHands project directly.
|
||||
|
||||
@@ -159,7 +159,7 @@ poetry run pytest ./tests/unit/test_*.py
|
||||
To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker
|
||||
container image by setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
|
||||
|
||||
Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.59-nikolaik`
|
||||
Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/openhands/runtime:0.60-nikolaik`
|
||||
|
||||
## Develop inside Docker container
|
||||
|
||||
@@ -193,7 +193,7 @@ Here's a guide to the important documentation files in the repository:
|
||||
- [/README.md](./README.md): Main project overview, features, and basic setup instructions
|
||||
- [/Development.md](./Development.md) (this file): Comprehensive guide for developers working on OpenHands
|
||||
- [/CONTRIBUTING.md](./CONTRIBUTING.md): Guidelines for contributing to the project, including code style and PR process
|
||||
- [/docs/DOC_STYLE_GUIDE.md](./docs/DOC_STYLE_GUIDE.md): Standards for writing and maintaining project documentation
|
||||
- [DOC_STYLE_GUIDE.md](https://github.com/All-Hands-AI/docs/blob/main/openhands/DOC_STYLE_GUIDE.md): Standards for writing and maintaining project documentation
|
||||
- [/openhands/README.md](./openhands/README.md): Details about the backend Python implementation
|
||||
- [/frontend/README.md](./frontend/README.md): Frontend React application setup and development guide
|
||||
- [/containers/README.md](./containers/README.md): Information about Docker containers and deployment
|
||||
|
||||
42
README.md
42
README.md
@@ -7,26 +7,26 @@
|
||||
|
||||
|
||||
<div align="center">
|
||||
<a href="https://github.com/All-Hands-AI/OpenHands/graphs/contributors"><img src="https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Contributors"></a>
|
||||
<a href="https://github.com/All-Hands-AI/OpenHands/stargazers"><img src="https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Stargazers"></a>
|
||||
<a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
|
||||
<a href="https://github.com/OpenHands/OpenHands/graphs/contributors"><img src="https://img.shields.io/github/contributors/OpenHands/OpenHands?style=for-the-badge&color=blue" alt="Contributors"></a>
|
||||
<a href="https://github.com/OpenHands/OpenHands/stargazers"><img src="https://img.shields.io/github/stars/OpenHands/OpenHands?style=for-the-badge&color=blue" alt="Stargazers"></a>
|
||||
<a href="https://github.com/OpenHands/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/OpenHands/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
|
||||
<br/>
|
||||
<a href="https://all-hands.dev/joinslack"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
|
||||
<a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
|
||||
<a href="https://github.com/OpenHands/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
|
||||
<br/>
|
||||
<a href="https://docs.all-hands.dev/usage/getting-started"><img src="https://img.shields.io/badge/Documentation-000?logo=googledocs&logoColor=FFE165&style=for-the-badge" alt="Check out the documentation"></a>
|
||||
<a href="https://arxiv.org/abs/2407.16741"><img src="https://img.shields.io/badge/Paper%20on%20Arxiv-000?logoColor=FFE165&logo=arxiv&style=for-the-badge" alt="Paper on Arxiv"></a>
|
||||
<a href="https://docs.google.com/spreadsheets/d/1wOUdFCMyY6Nt0AIqF705KN4JKOWgeI4wUGUP60krXXs/edit?gid=0#gid=0"><img src="https://img.shields.io/badge/Benchmark%20score-000?logoColor=FFE165&logo=huggingface&style=for-the-badge" alt="Evaluation Benchmark Score"></a>
|
||||
|
||||
<!-- Keep these links. Translations will automatically update with the README. -->
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=de">Deutsch</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=es">Español</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=fr">français</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=ja">日本語</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=ko">한국어</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=pt">Português</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=ru">Русский</a> |
|
||||
<a href="https://www.readme-i18n.com/All-Hands-AI/OpenHands?lang=zh">中文</a>
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=de">Deutsch</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=es">Español</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=fr">français</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=ja">日本語</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=ko">한국어</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=pt">Português</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=ru">Русский</a> |
|
||||
<a href="https://www.readme-i18n.com/OpenHands/OpenHands?lang=zh">中文</a>
|
||||
|
||||
<hr>
|
||||
</div>
|
||||
@@ -82,17 +82,17 @@ You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)
|
||||
You can also run OpenHands directly with Docker:
|
||||
|
||||
```bash
|
||||
docker pull docker.all-hands.dev/all-hands-ai/runtime:0.59-nikolaik
|
||||
docker pull docker.openhands.dev/openhands/runtime:0.60-nikolaik
|
||||
|
||||
docker run -it --rm --pull=always \
|
||||
-e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.59-nikolaik \
|
||||
-e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.openhands.dev/openhands/runtime:0.60-nikolaik \
|
||||
-e LOG_ALL_EVENTS=true \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v ~/.openhands:/.openhands \
|
||||
-p 3000:3000 \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name openhands-app \
|
||||
docker.all-hands.dev/all-hands-ai/openhands:0.59
|
||||
docker.openhands.dev/openhands/openhands:0.60
|
||||
```
|
||||
|
||||
</details>
|
||||
@@ -119,7 +119,7 @@ system requirements and more information.
|
||||
> It is not appropriate for multi-tenant deployments where multiple users share the same instance. There is no built-in authentication, isolation, or scalability.
|
||||
>
|
||||
> If you're interested in running OpenHands in a multi-tenant environment, check out the source-available, commercially-licensed
|
||||
> [OpenHands Cloud Helm Chart](https://github.com/all-Hands-AI/OpenHands-cloud)
|
||||
> [OpenHands Cloud Helm Chart](https://github.com/openHands/OpenHands-cloud)
|
||||
|
||||
You can [connect OpenHands to your local filesystem](https://docs.all-hands.dev/usage/runtimes/docker#connecting-to-your-filesystem),
|
||||
interact with it via a [friendly CLI](https://docs.all-hands.dev/usage/how-to/cli-mode),
|
||||
@@ -128,7 +128,7 @@ or run it on tagged issues with [a github action](https://docs.all-hands.dev/usa
|
||||
|
||||
Visit [Running OpenHands](https://docs.all-hands.dev/usage/installation) for more information and setup instructions.
|
||||
|
||||
If you want to modify the OpenHands source code, check out [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md).
|
||||
If you want to modify the OpenHands source code, check out [Development.md](https://github.com/OpenHands/OpenHands/blob/main/Development.md).
|
||||
|
||||
Having issues? The [Troubleshooting Guide](https://docs.all-hands.dev/usage/troubleshooting) can help.
|
||||
|
||||
@@ -146,17 +146,17 @@ OpenHands is a community-driven project, and we welcome contributions from every
|
||||
through Slack, so this is the best place to start, but we also are happy to have you contact us on Github:
|
||||
|
||||
- [Join our Slack workspace](https://all-hands.dev/joinslack) - Here we talk about research, architecture, and future development.
|
||||
- [Read or post Github Issues](https://github.com/All-Hands-AI/OpenHands/issues) - Check out the issues we're working on, or add your own ideas.
|
||||
- [Read or post Github Issues](https://github.com/OpenHands/OpenHands/issues) - Check out the issues we're working on, or add your own ideas.
|
||||
|
||||
See more about the community in [COMMUNITY.md](./COMMUNITY.md) or find details on contributing in [CONTRIBUTING.md](./CONTRIBUTING.md).
|
||||
|
||||
## 📈 Progress
|
||||
|
||||
See the monthly OpenHands roadmap [here](https://github.com/orgs/All-Hands-AI/projects/1) (updated at the maintainer's meeting at the end of each month).
|
||||
See the monthly OpenHands roadmap [here](https://github.com/orgs/OpenHands/projects/1) (updated at the maintainer's meeting at the end of each month).
|
||||
|
||||
<p align="center">
|
||||
<a href="https://star-history.com/#All-Hands-AI/OpenHands&Date">
|
||||
<img src="https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date" width="500" alt="Star History Chart">
|
||||
<a href="https://star-history.com/#OpenHands/OpenHands&Date">
|
||||
<img src="https://api.star-history.com/svg?repos=OpenHands/OpenHands&type=Date" width="500" alt="Star History Chart">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
|
||||
@@ -189,7 +189,7 @@ model = "gpt-4o"
|
||||
# Whether to use native tool calling if supported by the model. Can be true, false, or None by default, which chooses the model's default behavior based on the evaluation.
|
||||
# ATTENTION: Based on evaluation, enabling native function calling may lead to worse results
|
||||
# in some scenarios. Use with caution and consider testing with your specific use case.
|
||||
# https://github.com/All-Hands-AI/OpenHands/pull/4711
|
||||
# https://github.com/OpenHands/OpenHands/pull/4711
|
||||
#native_tool_calling = None
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
DOCKER_REGISTRY=ghcr.io
|
||||
DOCKER_ORG=all-hands-ai
|
||||
DOCKER_ORG=openhands
|
||||
DOCKER_IMAGE=openhands
|
||||
DOCKER_BASE_DIR="."
|
||||
|
||||
@@ -104,6 +104,9 @@ RUN apt-get update && apt-get install -y \
|
||||
&& apt-get clean \
|
||||
&& apt-get autoremove -y
|
||||
|
||||
# mark /app as safe git directory to avoid pre-commit errors
|
||||
RUN git config --system --add safe.directory /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# cache build dependencies
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
- SANDBOX_API_HOSTNAME=host.docker.internal
|
||||
- DOCKER_HOST_ADDR=host.docker.internal
|
||||
#
|
||||
- SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.59-nikolaik}
|
||||
- SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/openhands/runtime:0.60-nikolaik}
|
||||
- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
|
||||
- WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
|
||||
ports:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
DOCKER_REGISTRY=ghcr.io
|
||||
DOCKER_ORG=all-hands-ai
|
||||
DOCKER_ORG=openhands
|
||||
DOCKER_BASE_DIR="./containers/runtime"
|
||||
DOCKER_IMAGE=runtime
|
||||
# These variables will be appended by the runtime_build.py script
|
||||
|
||||
@@ -7,7 +7,7 @@ services:
|
||||
image: openhands:latest
|
||||
container_name: openhands-app-${DATE:-}
|
||||
environment:
|
||||
- SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.59-nikolaik}
|
||||
- SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.openhands.dev/openhands/runtime:0.60-nikolaik}
|
||||
#- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} # enable this only if you want a specific non-root sandbox user but you will have to manually adjust permissions of ~/.openhands for this user
|
||||
- WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
|
||||
ports:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG OPENHANDS_VERSION=latest
|
||||
ARG BASE="ghcr.io/all-hands-ai/openhands"
|
||||
ARG BASE="ghcr.io/openhands/openhands"
|
||||
FROM ${BASE}:${OPENHANDS_VERSION}
|
||||
|
||||
# Datadog labels
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
This directory contains the enterprise server used by [OpenHands Cloud](https://github.com/All-Hands-AI/OpenHands-Cloud/). The official, public version of OpenHands Cloud is available at
|
||||
[app.all-hands.dev](https://app.all-hands.dev).
|
||||
|
||||
You may also want to check out the MIT-licensed [OpenHands](https://github.com/All-Hands-AI/OpenHands)
|
||||
You may also want to check out the MIT-licensed [OpenHands](https://github.com/OpenHands/OpenHands)
|
||||
|
||||
## Extension of OpenHands (OSS)
|
||||
|
||||
@@ -16,7 +16,7 @@ The code in `/enterprise` directory builds on top of open source (OSS) code, ext
|
||||
|
||||
- Enterprise stacks on top of OSS. For example, the middleware in enterprise is stacked right on top of the middlewares in OSS. In `SAAS`, the middleware from BOTH repos will be present and running (which can sometimes cause conflicts)
|
||||
|
||||
- Enterprise overrides the implementation in OSS (only one is present at a time). For example, the server config SaasServerConfig which overrides [`ServerConfig`](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/server/config/server_config.py#L8) on OSS. This is done through dynamic imports ([see here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/server/config/server_config.py#L37-#L45))
|
||||
- Enterprise overrides the implementation in OSS (only one is present at a time). For example, the server config SaasServerConfig which overrides [`ServerConfig`](https://github.com/OpenHands/OpenHands/blob/main/openhands/server/config/server_config.py#L8) on OSS. This is done through dynamic imports ([see here](https://github.com/OpenHands/OpenHands/blob/main/openhands/server/config/server_config.py#L37-#L45))
|
||||
|
||||
Key areas that change on `SAAS` are
|
||||
|
||||
|
||||
@@ -0,0 +1,856 @@
|
||||
# OpenHands Enterprise Usage Telemetry Service
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Introduction](#1-introduction)
|
||||
- 1.1 [Problem Statement](#11-problem-statement)
|
||||
- 1.2 [Proposed Solution](#12-proposed-solution)
|
||||
2. [User Interface](#2-user-interface)
|
||||
- 2.1 [License Warning Banner](#21-license-warning-banner)
|
||||
- 2.2 [Administrator Experience](#22-administrator-experience)
|
||||
3. [Other Context](#3-other-context)
|
||||
- 3.1 [Replicated Platform Integration](#31-replicated-platform-integration)
|
||||
- 3.2 [Administrator Email Detection Strategy](#32-administrator-email-detection-strategy)
|
||||
- 3.3 [Metrics Collection Framework](#33-metrics-collection-framework)
|
||||
4. [Technical Design](#4-technical-design)
|
||||
- 4.1 [Database Schema](#41-database-schema)
|
||||
- 4.1.1 [Telemetry Metrics Table](#411-telemetry-metrics-table)
|
||||
- 4.1.2 [Telemetry Identity Table](#412-telemetry-identity-table)
|
||||
- 4.2 [Metrics Collection Framework](#42-metrics-collection-framework)
|
||||
- 4.2.1 [Base Collector Interface](#421-base-collector-interface)
|
||||
- 4.2.2 [Collector Registry](#422-collector-registry)
|
||||
- 4.2.3 [Example Collector Implementation](#423-example-collector-implementation)
|
||||
- 4.3 [Collection and Upload System](#43-collection-and-upload-system)
|
||||
- 4.3.1 [Metrics Collection Processor](#431-metrics-collection-processor)
|
||||
- 4.3.2 [Replicated Upload Processor](#432-replicated-upload-processor)
|
||||
- 4.4 [License Warning System](#44-license-warning-system)
|
||||
- 4.4.1 [License Status Endpoint](#441-license-status-endpoint)
|
||||
- 4.4.2 [UI Integration](#442-ui-integration)
|
||||
- 4.5 [Cronjob Configuration](#45-cronjob-configuration)
|
||||
- 4.5.1 [Collection Cronjob](#451-collection-cronjob)
|
||||
- 4.5.2 [Upload Cronjob](#452-upload-cronjob)
|
||||
5. [Implementation Plan](#5-implementation-plan)
|
||||
- 5.1 [Database Schema and Models (M1)](#51-database-schema-and-models-m1)
|
||||
- 5.1.1 [OpenHands - Database Migration](#511-openhands---database-migration)
|
||||
- 5.1.2 [OpenHands - Model Tests](#512-openhands---model-tests)
|
||||
- 5.2 [Metrics Collection Framework (M2)](#52-metrics-collection-framework-m2)
|
||||
- 5.2.1 [OpenHands - Core Collection Framework](#521-openhands---core-collection-framework)
|
||||
- 5.2.2 [OpenHands - Example Collectors](#522-openhands---example-collectors)
|
||||
- 5.2.3 [OpenHands - Framework Tests](#523-openhands---framework-tests)
|
||||
- 5.3 [Collection and Upload Processors (M3)](#53-collection-and-upload-processors-m3)
|
||||
- 5.3.1 [OpenHands - Collection Processor](#531-openhands---collection-processor)
|
||||
- 5.3.2 [OpenHands - Upload Processor](#532-openhands---upload-processor)
|
||||
- 5.3.3 [OpenHands - Integration Tests](#533-openhands---integration-tests)
|
||||
- 5.4 [License Warning API (M4)](#54-license-warning-api-m4)
|
||||
- 5.4.1 [OpenHands - License Status API](#541-openhands---license-status-api)
|
||||
- 5.4.2 [OpenHands - API Integration](#542-openhands---api-integration)
|
||||
- 5.5 [UI Warning Banner (M5)](#55-ui-warning-banner-m5)
|
||||
- 5.5.1 [OpenHands - UI Warning Banner](#551-openhands---ui-warning-banner)
|
||||
- 5.5.2 [OpenHands - UI Integration](#552-openhands---ui-integration)
|
||||
- 5.6 [Helm Chart Deployment Configuration (M6)](#56-helm-chart-deployment-configuration-m6)
|
||||
- 5.6.1 [OpenHands-Cloud - Cronjob Manifests](#561-openhands-cloud---cronjob-manifests)
|
||||
- 5.6.2 [OpenHands-Cloud - Configuration Management](#562-openhands-cloud---configuration-management)
|
||||
- 5.7 [Documentation and Enhanced Collectors (M7)](#57-documentation-and-enhanced-collectors-m7)
|
||||
- 5.7.1 [OpenHands - Advanced Collectors](#571-openhands---advanced-collectors)
|
||||
- 5.7.2 [OpenHands - Monitoring and Testing](#572-openhands---monitoring-and-testing)
|
||||
- 5.7.3 [OpenHands - Technical Documentation](#573-openhands---technical-documentation)
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
### 1.1 Problem Statement
|
||||
|
||||
OpenHands Enterprise (OHE) helm charts are publicly available but not open source, creating a visibility gap for the sales team. Unknown users can install and use OHE without the vendor's knowledge, preventing proper customer engagement and sales pipeline management. Without usage telemetry, the vendor cannot identify potential customers, track installation health, or proactively support users who may need assistance.
|
||||
|
||||
### 1.2 Proposed Solution
|
||||
|
||||
We propose implementing a comprehensive telemetry service that leverages the Replicated metrics platform and Python SDK to track OHE installations and usage. The solution provides automatic customer discovery, instance monitoring, and usage metrics collection while maintaining a clear license compliance pathway.
|
||||
|
||||
The system consists of three main components: (1) a pluggable metrics collection framework that allows developers to easily define and register custom metrics collectors, (2) automated cronjobs that periodically collect metrics and upload them to Replicated's vendor portal, and (3) a license compliance warning system that displays UI notifications when telemetry uploads fail, indicating potential license expiration.
|
||||
|
||||
The design ensures that telemetry cannot be easily disabled without breaking core OHE functionality by tying the warning system to environment variables that are essential for OHE operation. This approach balances user transparency with business requirements for customer visibility.
|
||||
|
||||
## 2. User Interface
|
||||
|
||||
### 2.1 License Warning Banner
|
||||
|
||||
When telemetry uploads fail for more than 4 days, users will see a prominent warning banner in the OpenHands Enterprise UI:
|
||||
|
||||
```
|
||||
⚠️ Your OpenHands Enterprise license will expire in 30 days. Please contact support if this issue persists.
|
||||
```
|
||||
|
||||
The banner appears at the top of all pages and cannot be permanently dismissed while the condition persists. Users can temporarily dismiss it, but it will reappear on page refresh until telemetry uploads resume successfully.
|
||||
|
||||
### 2.2 Administrator Experience
|
||||
|
||||
System administrators will not need to configure the telemetry system manually. The service automatically:
|
||||
|
||||
1. **Detects OHE installations** using existing required environment variables (`GITHUB_APP_CLIENT_ID`, `KEYCLOAK_SERVER_URL`, etc.)
|
||||
|
||||
2. **Generates unique customer identifiers** using administrator contact information:
|
||||
- Customer email: Determined by the following priority order:
|
||||
1. `OPENHANDS_ADMIN_EMAIL` environment variable (if set in helm values)
|
||||
2. Email of the first user who accepted Terms of Service (earliest `accepted_tos` timestamp)
|
||||
- Instance ID: Automatically generated by Replicated SDK using machine fingerprinting (IOPlatformUUID on macOS, D-Bus machine ID on Linux, Machine GUID on Windows)
|
||||
- **No Fallback**: If neither email source is available, telemetry collection is skipped until at least one user exists
|
||||
|
||||
3. **Collects and uploads metrics transparently** in the background via weekly collection and daily upload cronjobs
|
||||
|
||||
4. **Displays warnings only when necessary** for license compliance - no notifications appear during normal operation
|
||||
|
||||
## 3. Other Context
|
||||
|
||||
### 3.1 Replicated Platform Integration
|
||||
|
||||
The Replicated platform provides vendor-hosted infrastructure for collecting customer and instance telemetry. The Python SDK handles authentication, state management, and reliable metric delivery. Key concepts:
|
||||
|
||||
- **Customer**: Represents a unique OHE installation, identified by email or installation fingerprint
|
||||
- **Instance**: Represents a specific deployment of OHE for a customer
|
||||
- **Metrics**: Custom key-value data points collected from the installation
|
||||
- **Status**: Instance health indicators (running, degraded, updating, etc.)
|
||||
|
||||
The SDK automatically handles machine fingerprinting, local state caching, and retry logic for failed uploads.
|
||||
|
||||
### 3.2 Administrator Email Detection Strategy
|
||||
|
||||
To identify the appropriate administrator contact for sales outreach, the system uses a three-tier approach that avoids performance penalties on user authentication:
|
||||
|
||||
**Tier 1: Explicit Configuration** - The `OPENHANDS_ADMIN_EMAIL` environment variable allows administrators to explicitly specify the contact email during deployment.
|
||||
|
||||
**Tier 2: First Active User Detection** - If no explicit email is configured, the system identifies the first user who accepted Terms of Service (earliest `accepted_tos` timestamp with a valid email). This represents the first person to actively engage with the system and is very likely the administrator or installer.
|
||||
|
||||
**No Fallback Needed** - If neither email source is available, telemetry collection is skipped entirely. This ensures we only report meaningful usage data when there are actual active users.
|
||||
|
||||
**Performance Optimization**: The admin email determination is performed only during telemetry upload attempts, ensuring zero performance impact on user login flows.
|
||||
|
||||
### 3.3 Metrics Collection Framework
|
||||
|
||||
The proposed collector framework allows developers to define metrics in a single file change:
|
||||
|
||||
```python
|
||||
@register_collector("user_activity")
|
||||
class UserActivityCollector(MetricsCollector):
|
||||
def collect(self) -> Dict[str, Any]:
|
||||
# Query database and return metrics
|
||||
return {"active_users_7d": count, "conversations_created": total}
|
||||
```
|
||||
|
||||
Collectors are automatically discovered and executed by the collection cronjob, making the system extensible without modifying core collection logic.
|
||||
|
||||
## 4. Technical Design
|
||||
|
||||
### 4.1 Database Schema
|
||||
|
||||
#### 4.1.1 Telemetry Metrics Table
|
||||
|
||||
Stores collected metrics with transmission status tracking:
|
||||
|
||||
```sql
|
||||
CREATE TABLE telemetry_metrics (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
collected_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
metrics_data JSONB NOT NULL,
|
||||
uploaded_at TIMESTAMP WITH TIME ZONE NULL,
|
||||
upload_attempts INTEGER DEFAULT 0,
|
||||
last_upload_error TEXT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX idx_telemetry_metrics_collected_at ON telemetry_metrics(collected_at);
|
||||
CREATE INDEX idx_telemetry_metrics_uploaded_at ON telemetry_metrics(uploaded_at);
|
||||
```
|
||||
|
||||
#### 4.1.2 Telemetry Identity Table
|
||||
|
||||
Stores persistent identity information that must survive container restarts:
|
||||
|
||||
```sql
|
||||
CREATE TABLE telemetry_identity (
|
||||
id INTEGER PRIMARY KEY DEFAULT 1,
|
||||
customer_id VARCHAR(255) NULL,
|
||||
instance_id VARCHAR(255) NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT single_identity_row CHECK (id = 1)
|
||||
);
|
||||
```
|
||||
|
||||
**Design Rationale:**
|
||||
- **Separation of Concerns**: Identity data (customer_id, instance_id) is separated from operational data
|
||||
- **Persistent vs Computed**: Only data that cannot be reliably recomputed is persisted
|
||||
- **Upload Tracking**: Upload timestamps are tied directly to the metrics they represent
|
||||
- **Simplified Queries**: System state can be derived from metrics table (e.g., `MAX(uploaded_at)` for last successful upload)
|
||||
|
||||
### 4.2 Metrics Collection Framework
|
||||
|
||||
#### 4.2.1 Base Collector Interface
|
||||
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Any, List
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class MetricResult:
|
||||
key: str
|
||||
value: Any
|
||||
|
||||
class MetricsCollector(ABC):
|
||||
"""Base class for metrics collectors."""
|
||||
|
||||
@abstractmethod
|
||||
def collect(self) -> List[MetricResult]:
|
||||
"""Collect metrics and return results."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def collector_name(self) -> str:
|
||||
"""Unique name for this collector."""
|
||||
pass
|
||||
|
||||
def should_collect(self) -> bool:
|
||||
"""Override to add collection conditions."""
|
||||
return True
|
||||
```
|
||||
|
||||
#### 4.2.2 Collector Registry
|
||||
|
||||
```python
|
||||
from typing import Dict, Type, List
|
||||
import importlib
|
||||
import pkgutil
|
||||
|
||||
class CollectorRegistry:
|
||||
"""Registry for metrics collectors."""
|
||||
|
||||
def __init__(self):
|
||||
self._collectors: Dict[str, Type[MetricsCollector]] = {}
|
||||
|
||||
def register(self, collector_class: Type[MetricsCollector]) -> None:
|
||||
"""Register a collector class."""
|
||||
collector = collector_class()
|
||||
self._collectors[collector.collector_name] = collector_class
|
||||
|
||||
def get_all_collectors(self) -> List[MetricsCollector]:
|
||||
"""Get instances of all registered collectors."""
|
||||
return [cls() for cls in self._collectors.values()]
|
||||
|
||||
def discover_collectors(self, package_path: str) -> None:
|
||||
"""Auto-discover collectors in a package."""
|
||||
# Implementation to scan for @register_collector decorators
|
||||
pass
|
||||
|
||||
# Global registry instance
|
||||
collector_registry = CollectorRegistry()
|
||||
|
||||
def register_collector(name: str):
|
||||
"""Decorator to register a collector."""
|
||||
def decorator(cls: Type[MetricsCollector]) -> Type[MetricsCollector]:
|
||||
collector_registry.register(cls)
|
||||
return cls
|
||||
return decorator
|
||||
```
|
||||
|
||||
#### 4.2.3 Example Collector Implementation
|
||||
|
||||
```python
|
||||
@register_collector("system_metrics")
|
||||
class SystemMetricsCollector(MetricsCollector):
|
||||
"""Collects basic system and usage metrics."""
|
||||
|
||||
@property
|
||||
def collector_name(self) -> str:
|
||||
return "system_metrics"
|
||||
|
||||
def collect(self) -> List[MetricResult]:
|
||||
results = []
|
||||
|
||||
# Collect user count
|
||||
with session_maker() as session:
|
||||
user_count = session.query(UserSettings).count()
|
||||
results.append(MetricResult(
|
||||
key="total_users",
|
||||
value=user_count
|
||||
))
|
||||
|
||||
# Collect conversation count (last 30 days)
|
||||
thirty_days_ago = datetime.now(timezone.utc) - timedelta(days=30)
|
||||
conversation_count = session.query(StoredConversationMetadata)\
|
||||
.filter(StoredConversationMetadata.created_at >= thirty_days_ago)\
|
||||
.count()
|
||||
|
||||
results.append(MetricResult(
|
||||
key="conversations_30d",
|
||||
value=conversation_count
|
||||
))
|
||||
|
||||
return results
|
||||
```
|
||||
|
||||
### 4.3 Collection and Upload System
|
||||
|
||||
#### 4.3.1 Metrics Collection Processor
|
||||
|
||||
```python
|
||||
class TelemetryCollectionProcessor(MaintenanceTaskProcessor):
|
||||
"""Maintenance task processor for collecting metrics."""
|
||||
|
||||
collection_interval_days: int = 7
|
||||
|
||||
async def __call__(self, task: MaintenanceTask) -> dict:
|
||||
"""Collect metrics from all registered collectors."""
|
||||
|
||||
# Check if collection is needed
|
||||
if not self._should_collect():
|
||||
return {"status": "skipped", "reason": "too_recent"}
|
||||
|
||||
# Collect metrics from all registered collectors
|
||||
all_metrics = {}
|
||||
collector_results = {}
|
||||
|
||||
for collector in collector_registry.get_all_collectors():
|
||||
try:
|
||||
if collector.should_collect():
|
||||
results = collector.collect()
|
||||
for result in results:
|
||||
all_metrics[result.key] = result.value
|
||||
collector_results[collector.collector_name] = len(results)
|
||||
except Exception as e:
|
||||
logger.error(f"Collector {collector.collector_name} failed: {e}")
|
||||
collector_results[collector.collector_name] = f"error: {e}"
|
||||
|
||||
# Store metrics in database
|
||||
with session_maker() as session:
|
||||
telemetry_record = TelemetryMetrics(
|
||||
metrics_data=all_metrics,
|
||||
collected_at=datetime.now(timezone.utc)
|
||||
)
|
||||
session.add(telemetry_record)
|
||||
session.commit()
|
||||
|
||||
# Note: No need to track last_collection_at separately
|
||||
# Can be derived from MAX(collected_at) in telemetry_metrics
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"metrics_collected": len(all_metrics),
|
||||
"collectors_run": collector_results
|
||||
}
|
||||
|
||||
def _should_collect(self) -> bool:
|
||||
"""Check if collection is needed based on interval."""
|
||||
with session_maker() as session:
|
||||
# Get last collection time from metrics table
|
||||
last_collected = session.query(func.max(TelemetryMetrics.collected_at)).scalar()
|
||||
if not last_collected:
|
||||
return True
|
||||
|
||||
time_since_last = datetime.now(timezone.utc) - last_collected
|
||||
return time_since_last.days >= self.collection_interval_days
|
||||
```
|
||||
|
||||
#### 4.3.2 Replicated Upload Processor
|
||||
|
||||
```python
|
||||
from replicated import AsyncReplicatedClient, InstanceStatus
|
||||
|
||||
class TelemetryUploadProcessor(MaintenanceTaskProcessor):
|
||||
"""Maintenance task processor for uploading metrics to Replicated."""
|
||||
|
||||
replicated_publishable_key: str
|
||||
replicated_app_slug: str
|
||||
|
||||
async def __call__(self, task: MaintenanceTask) -> dict:
|
||||
"""Upload pending metrics to Replicated."""
|
||||
|
||||
# Get pending metrics
|
||||
with session_maker() as session:
|
||||
pending_metrics = session.query(TelemetryMetrics)\
|
||||
.filter(TelemetryMetrics.uploaded_at.is_(None))\
|
||||
.order_by(TelemetryMetrics.collected_at)\
|
||||
.all()
|
||||
|
||||
if not pending_metrics:
|
||||
return {"status": "no_pending_metrics"}
|
||||
|
||||
# Get admin email - skip if not available
|
||||
admin_email = self._get_admin_email()
|
||||
if not admin_email:
|
||||
logger.info("Skipping telemetry upload - no admin email available")
|
||||
return {
|
||||
"status": "skipped",
|
||||
"reason": "no_admin_email",
|
||||
"total_processed": 0
|
||||
}
|
||||
|
||||
uploaded_count = 0
|
||||
failed_count = 0
|
||||
|
||||
async with AsyncReplicatedClient(
|
||||
publishable_key=self.replicated_publishable_key,
|
||||
app_slug=self.replicated_app_slug
|
||||
) as client:
|
||||
|
||||
# Get or create customer and instance
|
||||
customer = await client.customer.get_or_create(
|
||||
email_address=admin_email
|
||||
)
|
||||
instance = await customer.get_or_create_instance()
|
||||
|
||||
# Store customer/instance IDs for future use
|
||||
await self._update_telemetry_identity(customer.customer_id, instance.instance_id)
|
||||
|
||||
# Upload each metric batch
|
||||
for metric_record in pending_metrics:
|
||||
try:
|
||||
# Send individual metrics
|
||||
for key, value in metric_record.metrics_data.items():
|
||||
await instance.send_metric(key, value)
|
||||
|
||||
# Update instance status
|
||||
await instance.set_status(InstanceStatus.RUNNING)
|
||||
|
||||
# Mark as uploaded
|
||||
with session_maker() as session:
|
||||
record = session.query(TelemetryMetrics)\
|
||||
.filter(TelemetryMetrics.id == metric_record.id)\
|
||||
.first()
|
||||
if record:
|
||||
record.uploaded_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
|
||||
uploaded_count += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload metrics {metric_record.id}: {e}")
|
||||
|
||||
# Update error info
|
||||
with session_maker() as session:
|
||||
record = session.query(TelemetryMetrics)\
|
||||
.filter(TelemetryMetrics.id == metric_record.id)\
|
||||
.first()
|
||||
if record:
|
||||
record.upload_attempts += 1
|
||||
record.last_upload_error = str(e)
|
||||
session.commit()
|
||||
|
||||
failed_count += 1
|
||||
|
||||
# Note: No need to track last_successful_upload_at separately
|
||||
# Can be derived from MAX(uploaded_at) in telemetry_metrics
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"uploaded": uploaded_count,
|
||||
"failed": failed_count,
|
||||
"total_processed": len(pending_metrics)
|
||||
}
|
||||
|
||||
def _get_admin_email(self) -> str | None:
|
||||
"""Get administrator email for customer identification."""
|
||||
# 1. Check environment variable first
|
||||
env_admin_email = os.getenv('OPENHANDS_ADMIN_EMAIL')
|
||||
if env_admin_email:
|
||||
logger.info("Using admin email from environment variable")
|
||||
return env_admin_email
|
||||
|
||||
# 2. Use first active user's email (earliest accepted_tos)
|
||||
with session_maker() as session:
|
||||
first_user = session.query(UserSettings)\
|
||||
.filter(UserSettings.email.isnot(None))\
|
||||
.filter(UserSettings.accepted_tos.isnot(None))\
|
||||
.order_by(UserSettings.accepted_tos.asc())\
|
||||
.first()
|
||||
|
||||
if first_user and first_user.email:
|
||||
logger.info(f"Using first active user email: {first_user.email}")
|
||||
return first_user.email
|
||||
|
||||
# No admin email available - skip telemetry
|
||||
logger.info("No admin email available - skipping telemetry collection")
|
||||
return None
|
||||
|
||||
async def _update_telemetry_identity(self, customer_id: str, instance_id: str) -> None:
|
||||
"""Update or create telemetry identity record."""
|
||||
with session_maker() as session:
|
||||
identity = session.query(TelemetryIdentity).first()
|
||||
if not identity:
|
||||
identity = TelemetryIdentity()
|
||||
session.add(identity)
|
||||
|
||||
identity.customer_id = customer_id
|
||||
identity.instance_id = instance_id
|
||||
session.commit()
|
||||
```
|
||||
|
||||
### 4.4 License Warning System
|
||||
|
||||
#### 4.4.1 License Status Endpoint
|
||||
|
||||
```python
|
||||
from fastapi import APIRouter
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
license_router = APIRouter()
|
||||
|
||||
@license_router.get("/license-status")
|
||||
async def get_license_status():
|
||||
"""Get license warning status for UI display."""
|
||||
|
||||
# Only show warnings for OHE installations
|
||||
if not _is_openhands_enterprise():
|
||||
return {"warn": False, "message": ""}
|
||||
|
||||
with session_maker() as session:
|
||||
# Get last successful upload time from metrics table
|
||||
last_upload = session.query(func.max(TelemetryMetrics.uploaded_at))\
|
||||
.filter(TelemetryMetrics.uploaded_at.isnot(None))\
|
||||
.scalar()
|
||||
|
||||
if not last_upload:
|
||||
# No successful uploads yet - show warning after 4 days
|
||||
return {
|
||||
"warn": True,
|
||||
"message": "OpenHands Enterprise license verification pending. Please ensure network connectivity."
|
||||
}
|
||||
|
||||
# Check if last successful upload was more than 4 days ago
|
||||
days_since_upload = (datetime.now(timezone.utc) - last_upload).days
|
||||
|
||||
if days_since_upload > 4:
|
||||
# Find oldest unsent batch
|
||||
oldest_unsent = session.query(TelemetryMetrics)\
|
||||
.filter(TelemetryMetrics.uploaded_at.is_(None))\
|
||||
.order_by(TelemetryMetrics.collected_at)\
|
||||
.first()
|
||||
|
||||
if oldest_unsent:
|
||||
# Calculate expiration date (oldest unsent + 34 days)
|
||||
expiration_date = oldest_unsent.collected_at + timedelta(days=34)
|
||||
days_until_expiration = (expiration_date - datetime.now(timezone.utc)).days
|
||||
|
||||
if days_until_expiration <= 0:
|
||||
message = "Your OpenHands Enterprise license has expired. Please contact support immediately."
|
||||
else:
|
||||
message = f"Your OpenHands Enterprise license will expire in {days_until_expiration} days. Please contact support if this issue persists."
|
||||
|
||||
return {"warn": True, "message": message}
|
||||
|
||||
return {"warn": False, "message": ""}
|
||||
|
||||
def _is_openhands_enterprise() -> bool:
|
||||
"""Detect if this is an OHE installation."""
|
||||
# Check for required OHE environment variables
|
||||
required_vars = [
|
||||
'GITHUB_APP_CLIENT_ID',
|
||||
'KEYCLOAK_SERVER_URL',
|
||||
'KEYCLOAK_REALM_NAME'
|
||||
]
|
||||
|
||||
return all(os.getenv(var) for var in required_vars)
|
||||
```
|
||||
|
||||
#### 4.4.2 UI Integration
|
||||
|
||||
The frontend will poll the license status endpoint and display warnings using the existing banner component pattern:
|
||||
|
||||
```typescript
|
||||
// New component: LicenseWarningBanner.tsx
|
||||
interface LicenseStatus {
|
||||
warn: boolean;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export function LicenseWarningBanner() {
|
||||
const [licenseStatus, setLicenseStatus] = useState<LicenseStatus>({ warn: false, message: "" });
|
||||
|
||||
useEffect(() => {
|
||||
const checkLicenseStatus = async () => {
|
||||
try {
|
||||
const response = await fetch('/api/license-status');
|
||||
const status = await response.json();
|
||||
setLicenseStatus(status);
|
||||
} catch (error) {
|
||||
console.error('Failed to check license status:', error);
|
||||
}
|
||||
};
|
||||
|
||||
// Check immediately and then every hour
|
||||
checkLicenseStatus();
|
||||
const interval = setInterval(checkLicenseStatus, 60 * 60 * 1000);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
if (!licenseStatus.warn) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="bg-red-600 text-white p-4 rounded flex items-center justify-between">
|
||||
<div className="flex items-center">
|
||||
<FaExclamationTriangle className="mr-3" />
|
||||
<span>{licenseStatus.message}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### 4.5 Cronjob Configuration
|
||||
|
||||
The cronjob configurations will be deployed via the OpenHands-Cloud helm charts.
|
||||
|
||||
#### 4.5.1 Collection Cronjob
|
||||
|
||||
The collection cronjob runs weekly to gather metrics:
|
||||
|
||||
```yaml
|
||||
# charts/openhands/templates/telemetry-collection-cronjob.yaml
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: {{ include "openhands.fullname" . }}-telemetry-collection
|
||||
labels:
|
||||
{{- include "openhands.labels" . | nindent 4 }}
|
||||
spec:
|
||||
schedule: "0 2 * * 0" # Weekly on Sunday at 2 AM
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: telemetry-collector
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
env:
|
||||
{{- include "openhands.env" . | nindent 12 }}
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- |
|
||||
from enterprise.storage.maintenance_task import MaintenanceTask, MaintenanceTaskStatus
|
||||
from enterprise.storage.database import session_maker
|
||||
from enterprise.server.telemetry.collection_processor import TelemetryCollectionProcessor
|
||||
|
||||
# Create collection task
|
||||
processor = TelemetryCollectionProcessor()
|
||||
task = MaintenanceTask()
|
||||
task.set_processor(processor)
|
||||
task.status = MaintenanceTaskStatus.PENDING
|
||||
|
||||
with session_maker() as session:
|
||||
session.add(task)
|
||||
session.commit()
|
||||
restartPolicy: OnFailure
|
||||
```
|
||||
|
||||
#### 4.5.2 Upload Cronjob
|
||||
|
||||
The upload cronjob runs daily to send metrics to Replicated:
|
||||
|
||||
```yaml
|
||||
# charts/openhands/templates/telemetry-upload-cronjob.yaml
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: {{ include "openhands.fullname" . }}-telemetry-upload
|
||||
labels:
|
||||
{{- include "openhands.labels" . | nindent 4 }}
|
||||
spec:
|
||||
schedule: "0 3 * * *" # Daily at 3 AM
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: telemetry-uploader
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
env:
|
||||
{{- include "openhands.env" . | nindent 12 }}
|
||||
- name: REPLICATED_PUBLISHABLE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "openhands.fullname" . }}-replicated-config
|
||||
key: publishable-key
|
||||
- name: REPLICATED_APP_SLUG
|
||||
value: {{ .Values.telemetry.replicatedAppSlug | default "openhands-enterprise" | quote }}
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- |
|
||||
from enterprise.storage.maintenance_task import MaintenanceTask, MaintenanceTaskStatus
|
||||
from enterprise.storage.database import session_maker
|
||||
from enterprise.server.telemetry.upload_processor import TelemetryUploadProcessor
|
||||
import os
|
||||
|
||||
# Create upload task
|
||||
processor = TelemetryUploadProcessor(
|
||||
replicated_publishable_key=os.getenv('REPLICATED_PUBLISHABLE_KEY'),
|
||||
replicated_app_slug=os.getenv('REPLICATED_APP_SLUG', 'openhands-enterprise')
|
||||
)
|
||||
task = MaintenanceTask()
|
||||
task.set_processor(processor)
|
||||
task.status = MaintenanceTaskStatus.PENDING
|
||||
|
||||
with session_maker() as session:
|
||||
session.add(task)
|
||||
session.commit()
|
||||
restartPolicy: OnFailure
|
||||
```
|
||||
|
||||
## 5. Implementation Plan
|
||||
|
||||
All implementation must pass existing lints and tests. New functionality requires comprehensive unit tests with >90% coverage. Integration tests should verify end-to-end telemetry flow including collection, storage, upload, and warning display.
|
||||
|
||||
### 5.1 Database Schema and Models (M1)
|
||||
|
||||
**Repository**: OpenHands
|
||||
Establish the foundational database schema and SQLAlchemy models for telemetry data storage.
|
||||
|
||||
#### 5.1.1 OpenHands - Database Migration
|
||||
|
||||
- [ ] `enterprise/migrations/versions/077_create_telemetry_tables.py`
|
||||
- [ ] `enterprise/storage/telemetry_metrics.py`
|
||||
- [ ] `enterprise/storage/telemetry_config.py`
|
||||
|
||||
#### 5.1.2 OpenHands - Model Tests
|
||||
|
||||
- [ ] `enterprise/tests/unit/storage/test_telemetry_metrics.py`
|
||||
- [ ] `enterprise/tests/unit/storage/test_telemetry_config.py`
|
||||
|
||||
**Demo**: Database tables created and models can store/retrieve telemetry data.
|
||||
|
||||
### 5.2 Metrics Collection Framework (M2)
|
||||
|
||||
**Repository**: OpenHands
|
||||
Implement the pluggable metrics collection system with registry and base classes.
|
||||
|
||||
#### 5.2.1 OpenHands - Core Collection Framework
|
||||
|
||||
- [ ] `enterprise/server/telemetry/__init__.py`
|
||||
- [ ] `enterprise/server/telemetry/collector_base.py`
|
||||
- [ ] `enterprise/server/telemetry/collector_registry.py`
|
||||
- [ ] `enterprise/server/telemetry/decorators.py`
|
||||
|
||||
#### 5.2.2 OpenHands - Example Collectors
|
||||
|
||||
- [ ] `enterprise/server/telemetry/collectors/__init__.py`
|
||||
- [ ] `enterprise/server/telemetry/collectors/system_metrics.py`
|
||||
- [ ] `enterprise/server/telemetry/collectors/user_activity.py`
|
||||
|
||||
#### 5.2.3 OpenHands - Framework Tests
|
||||
|
||||
- [ ] `enterprise/tests/unit/telemetry/test_collector_base.py`
|
||||
- [ ] `enterprise/tests/unit/telemetry/test_collector_registry.py`
|
||||
- [ ] `enterprise/tests/unit/telemetry/test_system_metrics.py`
|
||||
|
||||
**Demo**: Developers can create new collectors with a single file change using the @register_collector decorator.
|
||||
|
||||
### 5.3 Collection and Upload Processors (M3)
|
||||
|
||||
**Repository**: OpenHands
|
||||
Implement maintenance task processors for collecting metrics and uploading to Replicated.
|
||||
|
||||
#### 5.3.1 OpenHands - Collection Processor
|
||||
|
||||
- [ ] `enterprise/server/telemetry/collection_processor.py`
|
||||
- [ ] `enterprise/tests/unit/telemetry/test_collection_processor.py`
|
||||
|
||||
#### 5.3.2 OpenHands - Upload Processor
|
||||
|
||||
- [ ] `enterprise/server/telemetry/upload_processor.py`
|
||||
- [ ] `enterprise/tests/unit/telemetry/test_upload_processor.py`
|
||||
|
||||
#### 5.3.3 OpenHands - Integration Tests
|
||||
|
||||
- [ ] `enterprise/tests/integration/test_telemetry_flow.py`
|
||||
|
||||
**Demo**: Metrics are automatically collected weekly and uploaded daily to Replicated vendor portal.
|
||||
|
||||
### 5.4 License Warning API (M4)
|
||||
|
||||
**Repository**: OpenHands
|
||||
Implement the license status endpoint for the warning system.
|
||||
|
||||
#### 5.4.1 OpenHands - License Status API
|
||||
|
||||
- [ ] `enterprise/server/routes/license.py`
|
||||
- [ ] `enterprise/tests/unit/routes/test_license.py`
|
||||
|
||||
#### 5.4.2 OpenHands - API Integration
|
||||
|
||||
- [ ] Update `enterprise/saas_server.py` to include license router
|
||||
|
||||
**Demo**: License status API returns warning status based on telemetry upload success.
|
||||
|
||||
### 5.5 UI Warning Banner (M5)
|
||||
|
||||
**Repository**: OpenHands
|
||||
Implement the frontend warning banner component and integration.
|
||||
|
||||
#### 5.5.1 OpenHands - UI Warning Banner
|
||||
|
||||
- [ ] `frontend/src/components/features/license/license-warning-banner.tsx`
|
||||
- [ ] `frontend/src/components/features/license/license-warning-banner.test.tsx`
|
||||
|
||||
#### 5.5.2 OpenHands - UI Integration
|
||||
|
||||
- [ ] Update main UI layout to include license warning banner
|
||||
- [ ] Add license status polling service
|
||||
|
||||
**Demo**: License warnings appear in UI when telemetry uploads fail for >4 days, with accurate expiration countdown.
|
||||
|
||||
### 5.6 Helm Chart Deployment Configuration (M6)
|
||||
|
||||
**Repository**: OpenHands-Cloud
|
||||
Create Kubernetes cronjob configurations and deployment scripts.
|
||||
|
||||
#### 5.6.1 OpenHands-Cloud - Cronjob Manifests
|
||||
|
||||
- [ ] `charts/openhands/templates/telemetry-collection-cronjob.yaml`
|
||||
- [ ] `charts/openhands/templates/telemetry-upload-cronjob.yaml`
|
||||
|
||||
#### 5.6.2 OpenHands-Cloud - Configuration Management
|
||||
|
||||
- [ ] `charts/openhands/templates/replicated-secret.yaml`
|
||||
- [ ] Update `charts/openhands/values.yaml` with telemetry configuration options:
|
||||
```yaml
|
||||
# Add to values.yaml
|
||||
telemetry:
|
||||
enabled: true
|
||||
replicatedAppSlug: "openhands-enterprise"
|
||||
adminEmail: "" # Optional: admin email for customer identification
|
||||
|
||||
# Add to deployment environment variables
|
||||
env:
|
||||
OPENHANDS_ADMIN_EMAIL: "{{ .Values.telemetry.adminEmail }}"
|
||||
```
|
||||
|
||||
**Demo**: Complete telemetry system deployed via helm chart with configurable collection intervals and Replicated integration.
|
||||
|
||||
### 5.7 Documentation and Enhanced Collectors (M7)
|
||||
|
||||
**Repository**: OpenHands
|
||||
Add comprehensive metrics collectors, monitoring capabilities, and documentation.
|
||||
|
||||
#### 5.7.1 OpenHands - Advanced Collectors
|
||||
|
||||
- [ ] `enterprise/server/telemetry/collectors/conversation_metrics.py`
|
||||
- [ ] `enterprise/server/telemetry/collectors/integration_usage.py`
|
||||
- [ ] `enterprise/server/telemetry/collectors/performance_metrics.py`
|
||||
|
||||
#### 5.7.2 OpenHands - Monitoring and Testing
|
||||
|
||||
- [ ] `enterprise/server/telemetry/monitoring.py`
|
||||
- [ ] `enterprise/tests/e2e/test_telemetry_system.py`
|
||||
- [ ] Performance tests for large-scale metric collection
|
||||
|
||||
#### 5.7.3 OpenHands - Technical Documentation
|
||||
|
||||
- [ ] `enterprise/server/telemetry/README.md`
|
||||
- [ ] Update deployment documentation with telemetry configuration instructions
|
||||
- [ ] Add troubleshooting guide for telemetry issues
|
||||
|
||||
**Demo**: Rich telemetry data flowing to vendor portal with comprehensive monitoring, alerting for system health, and complete documentation.
|
||||
274
enterprise/enterprise_local/README.md
Normal file
274
enterprise/enterprise_local/README.md
Normal file
@@ -0,0 +1,274 @@
|
||||
# Instructions for developing SAAS locally
|
||||
|
||||
You have a few options here, which are expanded on below:
|
||||
|
||||
- A simple local development setup, with live reloading for both OSS and this repo
|
||||
- A more complex setup that includes Redis
|
||||
- An even more complex setup that includes GitHub events
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, make sure you have the following tools installed:
|
||||
|
||||
### Required for all options:
|
||||
|
||||
- [gcloud CLI](https://cloud.google.com/sdk/docs/install) - For authentication and secrets management
|
||||
- [sops](https://github.com/mozilla/sops) - For secrets decryption
|
||||
- macOS: `brew install sops`
|
||||
- Linux: `sudo apt-get install sops` or download from GitHub releases
|
||||
- Windows: Install via Chocolatey `choco install sops` or download from GitHub releases
|
||||
|
||||
### Additional requirements for enabling GitHub webhook events
|
||||
|
||||
- make
|
||||
- Python development tools (build-essential, python3-dev)
|
||||
- [ngrok](https://ngrok.com/download) - For creating tunnels to localhost
|
||||
|
||||
## Option 1: Simple local development
|
||||
|
||||
This option will allow you to modify the both the OSS code and the code in this repo,
|
||||
and see the changes in real-time.
|
||||
|
||||
This option works best for most scenarios. The only thing it's missing is
|
||||
the GitHub events webhook, which is not necessary for most development.
|
||||
|
||||
### 1. OpenHands location
|
||||
|
||||
The open source OpenHands repo should be cloned as a sibling directory,
|
||||
in `../OpenHands`. This is hard-coded in the pyproject.toml (edit if necessary)
|
||||
|
||||
If you're doing this the first time, you may need to run
|
||||
|
||||
```
|
||||
poetry update openhands-ai
|
||||
```
|
||||
|
||||
### 2. Set up env
|
||||
|
||||
First run this to retrieve Github App secrets
|
||||
|
||||
```
|
||||
gcloud auth application-default login
|
||||
gcloud config set project global-432717
|
||||
local/decrypt_env.sh
|
||||
```
|
||||
|
||||
Now run this to generate a `.env` file, which will used to run SAAS locally
|
||||
|
||||
```
|
||||
python -m pip install PyYAML
|
||||
export LITE_LLM_API_KEY=<your LLM API key>
|
||||
python enterprise_local/convert_to_env.py
|
||||
```
|
||||
|
||||
You'll also need to set up the runtime image, so that the dev server doesn't try to rebuild it.
|
||||
|
||||
```
|
||||
export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/openhands/runtime:main-nikolaik
|
||||
docker pull $SANDBOX_RUNTIME_CONTAINER_IMAGE
|
||||
```
|
||||
|
||||
By default the application will log in json, you can override.
|
||||
|
||||
```
|
||||
export LOG_PLAIN_TEXT=1
|
||||
```
|
||||
|
||||
### 3. Start the OpenHands frontend
|
||||
|
||||
Start the frontend like you normally would in the open source OpenHands repo.
|
||||
|
||||
### 4. Start the SaaS backend
|
||||
|
||||
```
|
||||
make build
|
||||
|
||||
make start-backend
|
||||
```
|
||||
|
||||
You should have a server running on `localhost:3000`, similar to the open source backend.
|
||||
Oauth should work properly.
|
||||
|
||||
## Option 2: With Redis
|
||||
|
||||
Follow all the steps above, then setup redis:
|
||||
|
||||
```bash
|
||||
docker run -p 6379:6379 --name openhands-redis -d redis
|
||||
export REDIS_HOST=host.docker.internal # you may want this to be localhost
|
||||
export REDIS_PORT=6379
|
||||
```
|
||||
|
||||
## Option 3: Work with GitHub events
|
||||
|
||||
### 1. Setup env file
|
||||
|
||||
(see above)
|
||||
|
||||
### 2. Build OSS Openhands
|
||||
|
||||
Develop on [Openhands](https://github.com/All-Hands-AI/OpenHands) locally. When ready, run the following inside Openhands repo (not the Deploy repo)
|
||||
|
||||
```
|
||||
docker build -f containers/app/Dockerfile -t openhands .
|
||||
```
|
||||
|
||||
### 3. Build SAAS Openhands
|
||||
|
||||
Build the SAAS image locally inside Deploy repo. Note that `openhands` is the name of the image built in Step 2
|
||||
|
||||
```
|
||||
docker build -t openhands-saas ./app/ --build-arg BASE="openhands"
|
||||
```
|
||||
|
||||
### 4. Create a tunnel
|
||||
|
||||
Run in a separate terminal
|
||||
|
||||
```
|
||||
ngrok http 3000
|
||||
```
|
||||
|
||||
There will be a line
|
||||
|
||||
```
|
||||
Forwarding https://bc71-2603-7000-5000-1575-e4a6-697b-589e-5801.ngrok-free.app
|
||||
```
|
||||
|
||||
Remember this URL as it will be used in Step 5 and 6
|
||||
|
||||
### 5. Setup Staging Github App callback/webhook urls
|
||||
|
||||
Using the URL found in Step 4, add another callback URL (`https://bc71-2603-7000-5000-1575-e4a6-697b-589e-5801.ngrok-free.app/oauth/github/callback`)
|
||||
|
||||
### 6. Run
|
||||
|
||||
This is the last step! Run SAAS openhands locally using
|
||||
|
||||
```
|
||||
docker run --env-file ./app/.env -p 3000:3000 openhands-saas
|
||||
```
|
||||
|
||||
Note `--env-file` is what injects the `.env` file created in Step 1
|
||||
|
||||
Visit the tunnel domain found in Step 4 to run the app (`https://bc71-2603-7000-5000-1575-e4a6-697b-589e-5801.ngrok-free.app`)
|
||||
|
||||
### Local Debugging with VSCode
|
||||
|
||||
Local Development necessitates running a version of OpenHands that is as similar as possible to the version running in the SAAS Environment. Before running these steps, it is assumed you have a local development version of the OSS OpenHands project running.
|
||||
|
||||
#### Redis
|
||||
|
||||
A Local redis instance is required for clustered communication between server nodes. The standard docker instance will suffice.
|
||||
`docker run -it -p 6379:6379 --name my-redis -d redis`
|
||||
|
||||
#### Postgres
|
||||
|
||||
A Local postgres instance is required. I used the official docker image:
|
||||
`docker run -p 5432:5432 --name my-postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=openhands -d postgres`
|
||||
Run the alembic migrations:
|
||||
`poetry run alembic upgrade head `
|
||||
|
||||
#### VSCode launch.json
|
||||
|
||||
The VSCode launch.json below sets up 2 servers to test clustering, running independently on localhost:3030 and localhost:3031. Running only the server on 3030 is usually sufficient unless tests of the clustered functionality are required. Secrets may be harvested directly from staging by connecting...
|
||||
`kubectl exec --stdin --tty <POD_NAME> -n <NAMESPACE> -- /bin/bash`
|
||||
And then invoking `printenv`. NOTE: _DO NOT DO THIS WITH PROD!!!_ (Hopefully by the time you read this, nobody will have access.)
|
||||
|
||||
```
|
||||
{
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Python Debugger: Python File",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${file}"
|
||||
},
|
||||
{
|
||||
"name": "OpenHands Deploy",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"module": "uvicorn",
|
||||
"args": [
|
||||
"saas_server:app",
|
||||
"--reload",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
"3030"
|
||||
],
|
||||
"env": {
|
||||
"DEBUG": "1",
|
||||
"FILE_STORE": "local",
|
||||
"REDIS_HOST": "localhost:6379",
|
||||
"OPENHANDS": "<YOUR LOCAL OSS OPENHANDS DIR>",
|
||||
"FRONTEND_DIRECTORY": "<YOUR LOCAL OSS OPENHANDS DIR>/frontend/build",
|
||||
"SANDBOX_RUNTIME_CONTAINER_IMAGE": "ghcr.io/openhands/runtime:main-nikolaik",
|
||||
"FILE_STORE_PATH": "<YOUR HOME DIRECTORY>>/.openhands-state",
|
||||
"OPENHANDS_CONFIG_CLS": "server.config.SaaSServerConfig",
|
||||
"GITHUB_APP_ID": "1062351",
|
||||
"GITHUB_APP_PRIVATE_KEY": "<GITHUB PRIVATE KEY>",
|
||||
"GITHUB_APP_CLIENT_ID": "Iv23lis7eUWDQHIq8US0",
|
||||
"GITHUB_APP_CLIENT_SECRET": "<GITHUB CLIENT SECRET>",
|
||||
"POSTHOG_CLIENT_KEY": "<POSTHOG CLIENT KEY>",
|
||||
"LITE_LLM_API_URL": "https://llm-proxy.staging.all-hands.dev",
|
||||
"LITE_LLM_TEAM_ID": "62ea39c4-8886-44f3-b7ce-07ed4fe42d2c",
|
||||
"LITE_LLM_API_KEY": "<LITE LLM API KEY>"
|
||||
},
|
||||
"justMyCode": false,
|
||||
"cwd": "${workspaceFolder}/app"
|
||||
},
|
||||
{
|
||||
"name": "OpenHands Deploy 2",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"module": "uvicorn",
|
||||
"args": [
|
||||
"saas_server:app",
|
||||
"--reload",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
"3031"
|
||||
],
|
||||
"env": {
|
||||
"DEBUG": "1",
|
||||
"FILE_STORE": "local",
|
||||
"REDIS_HOST": "localhost:6379",
|
||||
"OPENHANDS": "<YOUR LOCAL OSS OPENHANDS DIR>",
|
||||
"FRONTEND_DIRECTORY": "<YOUR LOCAL OSS OPENHANDS DIR>/frontend/build",
|
||||
"SANDBOX_RUNTIME_CONTAINER_IMAGE": "ghcr.io/openhands/runtime:main-nikolaik",
|
||||
"FILE_STORE_PATH": "<YOUR HOME DIRECTORY>>/.openhands-state",
|
||||
"OPENHANDS_CONFIG_CLS": "server.config.SaaSServerConfig",
|
||||
"GITHUB_APP_ID": "1062351",
|
||||
"GITHUB_APP_PRIVATE_KEY": "<GITHUB PRIVATE KEY>",
|
||||
"GITHUB_APP_CLIENT_ID": "Iv23lis7eUWDQHIq8US0",
|
||||
"GITHUB_APP_CLIENT_SECRET": "<GITHUB CLIENT SECRET>",
|
||||
"POSTHOG_CLIENT_KEY": "<POSTHOG CLIENT KEY>",
|
||||
"LITE_LLM_API_URL": "https://llm-proxy.staging.all-hands.dev",
|
||||
"LITE_LLM_TEAM_ID": "62ea39c4-8886-44f3-b7ce-07ed4fe42d2c",
|
||||
"LITE_LLM_API_KEY": "<LITE LLM API KEY>"
|
||||
},
|
||||
"justMyCode": false,
|
||||
"cwd": "${workspaceFolder}/app"
|
||||
},
|
||||
{
|
||||
"name": "Unit Tests",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"module": "pytest",
|
||||
"args": [
|
||||
"./tests/unit",
|
||||
//"./tests/unit/test_clustered_conversation_manager.py",
|
||||
"--durations=0"
|
||||
],
|
||||
"env": {
|
||||
"DEBUG": "1"
|
||||
},
|
||||
"justMyCode": false,
|
||||
"cwd": "${workspaceFolder}/app"
|
||||
},
|
||||
// set working directory...
|
||||
]
|
||||
}
|
||||
```
|
||||
127
enterprise/enterprise_local/convert_to_env.py
Normal file
127
enterprise/enterprise_local/convert_to_env.py
Normal file
@@ -0,0 +1,127 @@
|
||||
import base64
|
||||
import os
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def convert_yaml_to_env(yaml_file, target_parameters, output_env_file, prefix):
|
||||
"""Converts a YAML file into .env file format for specified target parameters under 'stringData' and 'data'.
|
||||
|
||||
:param yaml_file: Path to the YAML file.
|
||||
:param target_parameters: List of keys to extract from the YAML file.
|
||||
:param output_env_file: Path to the output .env file.
|
||||
:param prefix: Prefix for environment variables.
|
||||
"""
|
||||
try:
|
||||
# Load the YAML file
|
||||
with open(yaml_file, 'r') as file:
|
||||
yaml_data = yaml.safe_load(file)
|
||||
|
||||
# Extract sections
|
||||
string_data = yaml_data.get('stringData', None)
|
||||
data = yaml_data.get('data', None)
|
||||
|
||||
if string_data:
|
||||
env_source = string_data
|
||||
process_base64 = False
|
||||
elif data:
|
||||
env_source = data
|
||||
process_base64 = True
|
||||
else:
|
||||
print(
|
||||
"Error: Neither 'stringData' nor 'data' section found in the YAML file."
|
||||
)
|
||||
return
|
||||
|
||||
env_lines = []
|
||||
|
||||
for param in target_parameters:
|
||||
if param in env_source:
|
||||
value = env_source[param]
|
||||
if process_base64:
|
||||
try:
|
||||
decoded_value = base64.b64decode(value).decode('utf-8')
|
||||
formatted_value = (
|
||||
decoded_value.replace('\n', '\\n')
|
||||
if '\n' in decoded_value
|
||||
else decoded_value
|
||||
)
|
||||
except Exception as decode_error:
|
||||
print(f"Error decoding base64 for '{param}': {decode_error}")
|
||||
continue
|
||||
else:
|
||||
formatted_value = (
|
||||
value.replace('\n', '\\n')
|
||||
if isinstance(value, str) and '\n' in value
|
||||
else value
|
||||
)
|
||||
|
||||
new_key = prefix + param.upper().replace('-', '_')
|
||||
env_lines.append(f'{new_key}={formatted_value}')
|
||||
else:
|
||||
print(
|
||||
f"Warning: Parameter '{param}' not found in the selected section."
|
||||
)
|
||||
|
||||
# Write to the .env file
|
||||
with open(output_env_file, 'a') as env_file:
|
||||
env_file.write('\n'.join(env_lines) + '\n')
|
||||
|
||||
except Exception as e:
|
||||
print(f'Error: {e}')
|
||||
|
||||
|
||||
lite_llm_api_key = os.getenv('LITE_LLM_API_KEY')
|
||||
if not lite_llm_api_key:
|
||||
print('Set the LITE_LLM_API_KEY environment variable to your API key')
|
||||
sys.exit(1)
|
||||
|
||||
yaml_file = 'github_decrypted.yaml'
|
||||
target_parameters = ['client-id', 'client-secret', 'webhook-secret', 'private-key']
|
||||
output_env_file = './enterprise/.env'
|
||||
|
||||
if os.path.exists(output_env_file):
|
||||
os.remove(output_env_file)
|
||||
convert_yaml_to_env(yaml_file, target_parameters, output_env_file, 'GITHUB_APP_')
|
||||
os.remove(yaml_file)
|
||||
|
||||
yaml_file = 'keycloak_realm_decrypted.yaml'
|
||||
target_parameters = ['client-id', 'client-secret', 'provider-name', 'realm-name']
|
||||
convert_yaml_to_env(yaml_file, target_parameters, output_env_file, 'KEYCLOAK_')
|
||||
os.remove(yaml_file)
|
||||
|
||||
yaml_file = 'keycloak_admin_decrypted.yaml'
|
||||
target_parameters = ['admin-password']
|
||||
convert_yaml_to_env(yaml_file, target_parameters, output_env_file, 'KEYCLOAK_')
|
||||
os.remove(yaml_file)
|
||||
|
||||
lines = []
|
||||
lines.append('KEYCLOAK_SERVER_URL=https://auth.staging.all-hands.dev/')
|
||||
lines.append('KEYCLOAK_SERVER_URL_EXT=https://auth.staging.all-hands.dev/')
|
||||
lines.append('OPENHANDS_CONFIG_CLS=server.config.SaaSServerConfig')
|
||||
lines.append(
|
||||
'OPENHANDS_GITHUB_SERVICE_CLS=integrations.github.github_service.SaaSGitHubService'
|
||||
)
|
||||
lines.append(
|
||||
'OPENHANDS_GITLAB_SERVICE_CLS=integrations.gitlab.gitlab_service.SaaSGitLabService'
|
||||
)
|
||||
lines.append(
|
||||
'OPENHANDS_BITBUCKET_SERVICE_CLS=integrations.bitbucket.bitbucket_service.SaaSBitBucketService'
|
||||
)
|
||||
lines.append(
|
||||
'OPENHANDS_CONVERSATION_VALIDATOR_CLS=storage.saas_conversation_validator.SaasConversationValidator'
|
||||
)
|
||||
lines.append('POSTHOG_CLIENT_KEY=test')
|
||||
lines.append('ENABLE_PROACTIVE_CONVERSATION_STARTERS=true')
|
||||
lines.append('MAX_CONCURRENT_CONVERSATIONS=10')
|
||||
lines.append('LITE_LLM_API_URL=https://llm-proxy.eval.all-hands.dev')
|
||||
lines.append('LITELLM_DEFAULT_MODEL=litellm_proxy/claude-sonnet-4-20250514')
|
||||
lines.append(f'LITE_LLM_API_KEY={lite_llm_api_key}')
|
||||
lines.append('LOCAL_DEPLOYMENT=true')
|
||||
lines.append('DB_HOST=localhost')
|
||||
|
||||
with open(output_env_file, 'a') as env_file:
|
||||
env_file.write('\n'.join(lines))
|
||||
|
||||
print(f'.env file created at: {output_env_file}')
|
||||
27
enterprise/enterprise_local/decrypt_env.sh
Normal file
27
enterprise/enterprise_local/decrypt_env.sh
Normal file
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Check if DEPLOY_DIR argument was provided
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: $0 <DEPLOY_DIR>"
|
||||
echo "Example: $0 /path/to/deploy"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Normalize path (remove trailing slash)
|
||||
DEPLOY_DIR="${DEPLOY_DIR%/}"
|
||||
|
||||
# Function to decrypt and rename
|
||||
decrypt_and_move() {
|
||||
local secret_path="$1"
|
||||
local output_name="$2"
|
||||
|
||||
${DEPLOY_DIR}/scripts/decrypt.sh "${DEPLOY_DIR}/${secret_path}"
|
||||
mv decrypted.yaml "${output_name}"
|
||||
echo "Moved decrypted.yaml to ${output_name}"
|
||||
}
|
||||
|
||||
# Decrypt each secret file
|
||||
decrypt_and_move "openhands/envs/feature/secrets/github-app.yaml" "github_decrypted.yaml"
|
||||
decrypt_and_move "openhands/envs/staging/secrets/keycloak-realm.yaml" "keycloak_realm_decrypted.yaml"
|
||||
decrypt_and_move "openhands/envs/staging/secrets/keycloak-admin.yaml" "keycloak_admin_decrypted.yaml"
|
||||
@@ -1,18 +1,47 @@
|
||||
from uuid import UUID
|
||||
|
||||
from experiments.constants import (
|
||||
ENABLE_EXPERIMENT_MANAGER,
|
||||
EXPERIMENT_SYSTEM_PROMPT_EXPERIMENT,
|
||||
)
|
||||
from experiments.experiment_versions import (
|
||||
handle_condenser_max_step_experiment,
|
||||
handle_system_prompt_experiment,
|
||||
)
|
||||
from experiments.experiment_versions._004_condenser_max_step_experiment import (
|
||||
handle_condenser_max_step_experiment__v1,
|
||||
)
|
||||
|
||||
from openhands.core.config.openhands_config import OpenHandsConfig
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.experiments.experiment_manager import ExperimentManager
|
||||
from openhands.sdk import Agent
|
||||
from openhands.server.session.conversation_init_data import ConversationInitData
|
||||
|
||||
|
||||
class SaaSExperimentManager(ExperimentManager):
|
||||
@staticmethod
|
||||
def run_agent_variant_tests__v1(
|
||||
user_id: str | None, conversation_id: UUID, agent: Agent
|
||||
) -> Agent:
|
||||
if not ENABLE_EXPERIMENT_MANAGER:
|
||||
logger.info(
|
||||
'experiment_manager:run_conversation_variant_test:skipped',
|
||||
extra={'reason': 'experiment_manager_disabled'},
|
||||
)
|
||||
return agent
|
||||
|
||||
agent = handle_condenser_max_step_experiment__v1(
|
||||
user_id, conversation_id, agent
|
||||
)
|
||||
|
||||
if EXPERIMENT_SYSTEM_PROMPT_EXPERIMENT:
|
||||
agent = agent.model_copy(
|
||||
update={'system_prompt_filename': 'system_prompt_long_horizon.j2'}
|
||||
)
|
||||
|
||||
return agent
|
||||
|
||||
@staticmethod
|
||||
def run_conversation_variant_test(
|
||||
user_id, conversation_id, conversation_settings
|
||||
|
||||
@@ -5,12 +5,18 @@ This module contains the handler for the condenser max step experiment that test
|
||||
different max_size values for the condenser configuration.
|
||||
"""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
import posthog
|
||||
from experiments.constants import EXPERIMENT_CONDENSER_MAX_STEP
|
||||
from server.constants import IS_FEATURE_ENV
|
||||
from storage.experiment_assignment_store import ExperimentAssignmentStore
|
||||
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.sdk import Agent
|
||||
from openhands.sdk.context.condenser import (
|
||||
LLMSummarizingCondenser,
|
||||
)
|
||||
from openhands.server.session.conversation_init_data import ConversationInitData
|
||||
|
||||
|
||||
@@ -190,3 +196,37 @@ def handle_condenser_max_step_experiment(
|
||||
return conversation_settings
|
||||
|
||||
return conversation_settings
|
||||
|
||||
|
||||
def handle_condenser_max_step_experiment__v1(
|
||||
user_id: str | None,
|
||||
conversation_id: UUID,
|
||||
agent: Agent,
|
||||
) -> Agent:
|
||||
enabled_variant = _get_condenser_max_step_variant(user_id, str(conversation_id))
|
||||
|
||||
if enabled_variant is None:
|
||||
return agent
|
||||
|
||||
if enabled_variant == 'control':
|
||||
condenser_max_size = 120
|
||||
elif enabled_variant == 'treatment':
|
||||
condenser_max_size = 80
|
||||
else:
|
||||
logger.error(
|
||||
'condenser_max_step_experiment:unknown_variant',
|
||||
extra={
|
||||
'user_id': user_id,
|
||||
'convo_id': conversation_id,
|
||||
'variant': enabled_variant,
|
||||
'reason': 'unknown variant; returning original conversation settings',
|
||||
},
|
||||
)
|
||||
return agent
|
||||
|
||||
condenser_llm = agent.llm.model_copy(update={'usage_id': 'condenser'})
|
||||
condenser = LLMSummarizingCondenser(
|
||||
llm=condenser_llm, max_size=condenser_max_size, keep_first=4
|
||||
)
|
||||
|
||||
return agent.model_copy(update={'condenser': condenser})
|
||||
|
||||
@@ -31,7 +31,7 @@ from server.utils.conversation_callback_utils import register_callback_processor
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.integrations.provider import ProviderToken, ProviderType
|
||||
from openhands.server.types import LLMAuthenticationError, MissingSettingsError
|
||||
from openhands.storage.data_models.user_secrets import UserSecrets
|
||||
from openhands.storage.data_models.secrets import Secrets
|
||||
from openhands.utils.async_utils import call_sync_from_async
|
||||
|
||||
|
||||
@@ -250,7 +250,7 @@ class GithubManager(Manager):
|
||||
f'[GitHub] Creating new conversation for user {user_info.username}'
|
||||
)
|
||||
|
||||
secret_store = UserSecrets(
|
||||
secret_store = Secrets(
|
||||
provider_tokens=MappingProxyType(
|
||||
{
|
||||
ProviderType.GITHUB: ProviderToken(
|
||||
|
||||
@@ -24,7 +24,7 @@ from server.config import get_config
|
||||
from storage.database import session_maker
|
||||
from storage.proactive_conversation_store import ProactiveConversationStore
|
||||
from storage.saas_secrets_store import SaasSecretsStore
|
||||
from storage.user_settings import UserSettings
|
||||
from storage.saas_settings_store import SaasSettingsStore
|
||||
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.integrations.github.github_service import GithubServiceImpl
|
||||
@@ -61,20 +61,19 @@ async def get_user_proactive_conversation_setting(user_id: str | None) -> bool:
|
||||
if not user_id:
|
||||
return False
|
||||
|
||||
def _get_setting():
|
||||
with session_maker() as session:
|
||||
settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == user_id)
|
||||
.first()
|
||||
)
|
||||
config = get_config()
|
||||
settings_store = SaasSettingsStore(
|
||||
user_id=user_id, session_maker=session_maker, config=config
|
||||
)
|
||||
|
||||
if not settings or settings.enable_proactive_conversation_starters is None:
|
||||
return False
|
||||
settings = await call_sync_from_async(
|
||||
settings_store.get_user_settings_by_keycloak_id, user_id
|
||||
)
|
||||
|
||||
return settings.enable_proactive_conversation_starters
|
||||
if not settings or settings.enable_proactive_conversation_starters is None:
|
||||
return False
|
||||
|
||||
return await call_sync_from_async(_get_setting)
|
||||
return settings.enable_proactive_conversation_starters
|
||||
|
||||
|
||||
# =================================================
|
||||
|
||||
@@ -25,7 +25,7 @@ from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.integrations.gitlab.gitlab_service import GitLabServiceImpl
|
||||
from openhands.integrations.provider import ProviderToken, ProviderType
|
||||
from openhands.server.types import LLMAuthenticationError, MissingSettingsError
|
||||
from openhands.storage.data_models.user_secrets import UserSecrets
|
||||
from openhands.storage.data_models.secrets import Secrets
|
||||
|
||||
|
||||
class GitlabManager(Manager):
|
||||
@@ -198,7 +198,7 @@ class GitlabManager(Manager):
|
||||
f'[GitLab] Creating new conversation for user {user_info.username}'
|
||||
)
|
||||
|
||||
secret_store = UserSecrets(
|
||||
secret_store = Secrets(
|
||||
provider_tokens=MappingProxyType(
|
||||
{
|
||||
ProviderType.GITLAB: ProviderToken(
|
||||
|
||||
@@ -32,6 +32,7 @@ from openhands.integrations.service_types import Repository
|
||||
from openhands.server.shared import server_config
|
||||
from openhands.server.types import LLMAuthenticationError, MissingSettingsError
|
||||
from openhands.server.user_auth.user_auth import UserAuth
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
JIRA_CLOUD_API_URL = 'https://api.atlassian.com/ex/jira'
|
||||
|
||||
@@ -408,7 +409,7 @@ class JiraManager(Manager):
|
||||
svc_acc_api_key: str,
|
||||
) -> Tuple[str, str]:
|
||||
url = f'{JIRA_CLOUD_API_URL}/{jira_cloud_id}/rest/api/2/issue/{job_context.issue_key}'
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.get(url, auth=(svc_acc_email, svc_acc_api_key))
|
||||
response.raise_for_status()
|
||||
issue_payload = response.json()
|
||||
@@ -443,7 +444,7 @@ class JiraManager(Manager):
|
||||
f'{JIRA_CLOUD_API_URL}/{jira_cloud_id}/rest/api/2/issue/{issue_key}/comment'
|
||||
)
|
||||
data = {'body': message.message}
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(
|
||||
url, auth=(svc_acc_email, svc_acc_api_key), json=data
|
||||
)
|
||||
|
||||
@@ -57,7 +57,7 @@ class JiraNewConversationView(JiraViewInterface):
|
||||
raise StartingConvoException('No repository selected for this conversation')
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
user_secrets = await self.saas_user_auth.get_user_secrets()
|
||||
user_secrets = await self.saas_user_auth.get_secrets()
|
||||
instructions, user_msg = self._get_instructions(jinja_env)
|
||||
|
||||
try:
|
||||
@@ -132,8 +132,10 @@ class JiraExistingConversationView(JiraViewInterface):
|
||||
conversation_store = await ConversationStoreImpl.get_instance(
|
||||
config, user_id
|
||||
)
|
||||
metadata = await conversation_store.get_metadata(self.conversation_id)
|
||||
if not metadata:
|
||||
|
||||
try:
|
||||
await conversation_store.get_metadata(self.conversation_id)
|
||||
except FileNotFoundError:
|
||||
raise StartingConvoException('Conversation no longer exists.')
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
|
||||
@@ -34,6 +34,7 @@ from openhands.integrations.service_types import Repository
|
||||
from openhands.server.shared import server_config
|
||||
from openhands.server.types import LLMAuthenticationError, MissingSettingsError
|
||||
from openhands.server.user_auth.user_auth import UserAuth
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
|
||||
class JiraDcManager(Manager):
|
||||
@@ -422,7 +423,7 @@ class JiraDcManager(Manager):
|
||||
"""Get issue details from Jira DC API."""
|
||||
url = f'{job_context.base_api_url}/rest/api/2/issue/{job_context.issue_key}'
|
||||
headers = {'Authorization': f'Bearer {svc_acc_api_key}'}
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
issue_payload = response.json()
|
||||
@@ -452,7 +453,7 @@ class JiraDcManager(Manager):
|
||||
url = f'{base_api_url}/rest/api/2/issue/{issue_key}/comment'
|
||||
headers = {'Authorization': f'Bearer {svc_acc_api_key}'}
|
||||
data = {'body': message.message}
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(url, headers=headers, json=data)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
@@ -60,7 +60,7 @@ class JiraDcNewConversationView(JiraDcViewInterface):
|
||||
raise StartingConvoException('No repository selected for this conversation')
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
user_secrets = await self.saas_user_auth.get_user_secrets()
|
||||
user_secrets = await self.saas_user_auth.get_secrets()
|
||||
instructions, user_msg = self._get_instructions(jinja_env)
|
||||
|
||||
try:
|
||||
@@ -135,8 +135,10 @@ class JiraDcExistingConversationView(JiraDcViewInterface):
|
||||
conversation_store = await ConversationStoreImpl.get_instance(
|
||||
config, user_id
|
||||
)
|
||||
metadata = await conversation_store.get_metadata(self.conversation_id)
|
||||
if not metadata:
|
||||
|
||||
try:
|
||||
await conversation_store.get_metadata(self.conversation_id)
|
||||
except FileNotFoundError:
|
||||
raise StartingConvoException('Conversation no longer exists.')
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
|
||||
@@ -31,6 +31,7 @@ from openhands.integrations.service_types import Repository
|
||||
from openhands.server.shared import server_config
|
||||
from openhands.server.types import LLMAuthenticationError, MissingSettingsError
|
||||
from openhands.server.user_auth.user_auth import UserAuth
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
|
||||
class LinearManager(Manager):
|
||||
@@ -408,7 +409,7 @@ class LinearManager(Manager):
|
||||
async def _query_api(self, query: str, variables: Dict, api_key: str) -> Dict:
|
||||
"""Query Linear GraphQL API."""
|
||||
headers = {'Authorization': api_key}
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(
|
||||
self.api_url,
|
||||
headers=headers,
|
||||
|
||||
@@ -57,7 +57,7 @@ class LinearNewConversationView(LinearViewInterface):
|
||||
raise StartingConvoException('No repository selected for this conversation')
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
user_secrets = await self.saas_user_auth.get_user_secrets()
|
||||
user_secrets = await self.saas_user_auth.get_secrets()
|
||||
instructions, user_msg = self._get_instructions(jinja_env)
|
||||
|
||||
try:
|
||||
@@ -132,8 +132,10 @@ class LinearExistingConversationView(LinearViewInterface):
|
||||
conversation_store = await ConversationStoreImpl.get_instance(
|
||||
config, user_id
|
||||
)
|
||||
metadata = await conversation_store.get_metadata(self.conversation_id)
|
||||
if not metadata:
|
||||
|
||||
try:
|
||||
await conversation_store.get_metadata(self.conversation_id)
|
||||
except FileNotFoundError:
|
||||
raise StartingConvoException('Conversation no longer exists.')
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
|
||||
@@ -87,7 +87,7 @@ class SlackManager(Manager):
|
||||
return slack_user, saas_user_auth
|
||||
|
||||
def _infer_repo_from_message(self, user_msg: str) -> str | None:
|
||||
# Regular expression to match patterns like "All-Hands-AI/OpenHands" or "deploy repo"
|
||||
# Regular expression to match patterns like "OpenHands/OpenHands" or "deploy repo"
|
||||
pattern = r'([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+)|([a-zA-Z0-9_-]+)(?=\s+repo)'
|
||||
match = re.search(pattern, user_msg)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.schema.agent import AgentState
|
||||
from openhands.events.action import MessageAction
|
||||
from openhands.events.serialization.event import event_to_dict
|
||||
from openhands.integrations.provider import ProviderHandler
|
||||
from openhands.server.services.conversation_service import (
|
||||
create_new_conversation,
|
||||
setup_init_conversation_settings,
|
||||
@@ -185,22 +186,30 @@ class SlackNewConversationView(SlackViewInterface):
|
||||
self._verify_necessary_values_are_set()
|
||||
|
||||
provider_tokens = await self.saas_user_auth.get_provider_tokens()
|
||||
user_secrets = await self.saas_user_auth.get_user_secrets()
|
||||
user_secrets = await self.saas_user_auth.get_secrets()
|
||||
user_instructions, conversation_instructions = self._get_instructions(jinja)
|
||||
|
||||
# Determine git provider from repository
|
||||
git_provider = None
|
||||
if self.selected_repo and provider_tokens:
|
||||
provider_handler = ProviderHandler(provider_tokens)
|
||||
repository = await provider_handler.verify_repo_provider(self.selected_repo)
|
||||
git_provider = repository.git_provider
|
||||
|
||||
agent_loop_info = await create_new_conversation(
|
||||
user_id=self.slack_to_openhands_user.keycloak_user_id,
|
||||
git_provider_tokens=provider_tokens,
|
||||
selected_repository=self.selected_repo,
|
||||
selected_branch=None,
|
||||
initial_user_msg=user_instructions,
|
||||
conversation_instructions=conversation_instructions
|
||||
if conversation_instructions
|
||||
else None,
|
||||
conversation_instructions=(
|
||||
conversation_instructions if conversation_instructions else None
|
||||
),
|
||||
image_urls=None,
|
||||
replay_json=None,
|
||||
conversation_trigger=ConversationTrigger.SLACK,
|
||||
custom_secrets=user_secrets.custom_secrets if user_secrets else None,
|
||||
git_provider=git_provider,
|
||||
)
|
||||
|
||||
self.conversation_id = agent_loop_info.conversation_id
|
||||
@@ -263,8 +272,10 @@ class SlackUpdateExistingConversationView(SlackNewConversationView):
|
||||
# Check if conversation has been deleted
|
||||
# Update logic when soft delete is implemented
|
||||
conversation_store = await ConversationStoreImpl.get_instance(config, user_id)
|
||||
metadata = await conversation_store.get_metadata(self.conversation_id)
|
||||
if not metadata:
|
||||
|
||||
try:
|
||||
await conversation_store.get_metadata(self.conversation_id)
|
||||
except FileNotFoundError:
|
||||
raise StartingConvoException('Conversation no longer exists.')
|
||||
|
||||
provider_tokens = await saas_user_auth.get_provider_tokens()
|
||||
|
||||
@@ -381,7 +381,7 @@ def infer_repo_from_message(user_msg: str) -> list[str]:
|
||||
# Captures: protocol, domain, owner, repo (with optional .git extension)
|
||||
git_url_pattern = r'https?://(?:github\.com|gitlab\.com|bitbucket\.org)/([a-zA-Z0-9_.-]+)/([a-zA-Z0-9_.-]+?)(?:\.git)?(?:[/?#].*?)?(?=\s|$|[^\w.-])'
|
||||
|
||||
# Pattern to match direct owner/repo mentions (e.g., "All-Hands-AI/OpenHands")
|
||||
# Pattern to match direct owner/repo mentions (e.g., "OpenHands/OpenHands")
|
||||
# Must be surrounded by word boundaries or specific characters to avoid false positives
|
||||
direct_pattern = (
|
||||
r'(?:^|\s|[\[\(\'"])([a-zA-Z0-9_.-]+)/([a-zA-Z0-9_.-]+)(?=\s|$|[\]\)\'",.])'
|
||||
|
||||
27
enterprise/migrations/versions/077_drop_settings_table.py
Normal file
27
enterprise/migrations/versions/077_drop_settings_table.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""drop settings table
|
||||
|
||||
Revision ID: 077
|
||||
Revises: 076
|
||||
Create Date: 2025-10-21 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '077'
|
||||
down_revision: Union[str, None] = '076'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Drop the deprecated settings table."""
|
||||
op.execute('DROP TABLE IF EXISTS settings')
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op downgrade since the settings table is deprecated."""
|
||||
pass
|
||||
129
enterprise/migrations/versions/078_create_telemetry_tables.py
Normal file
129
enterprise/migrations/versions/078_create_telemetry_tables.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""create telemetry tables
|
||||
|
||||
Revision ID: 078
|
||||
Revises: 077
|
||||
Create Date: 2025-10-21
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '078'
|
||||
down_revision: Union[str, None] = '077'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create telemetry tables for metrics collection and configuration."""
|
||||
# Create telemetry_metrics table
|
||||
op.create_table(
|
||||
'telemetry_metrics',
|
||||
sa.Column(
|
||||
'id',
|
||||
sa.String(), # UUID as string
|
||||
nullable=False,
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column(
|
||||
'collected_at',
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text('CURRENT_TIMESTAMP'),
|
||||
),
|
||||
sa.Column(
|
||||
'metrics_data',
|
||||
sa.JSON(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
'uploaded_at',
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
'upload_attempts',
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default='0',
|
||||
),
|
||||
sa.Column(
|
||||
'last_upload_error',
|
||||
sa.Text(),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
'created_at',
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text('CURRENT_TIMESTAMP'),
|
||||
),
|
||||
sa.Column(
|
||||
'updated_at',
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text('CURRENT_TIMESTAMP'),
|
||||
),
|
||||
)
|
||||
|
||||
# Create indexes for telemetry_metrics
|
||||
op.create_index(
|
||||
'ix_telemetry_metrics_collected_at', 'telemetry_metrics', ['collected_at']
|
||||
)
|
||||
op.create_index(
|
||||
'ix_telemetry_metrics_uploaded_at', 'telemetry_metrics', ['uploaded_at']
|
||||
)
|
||||
|
||||
# Create telemetry_replicated_identity table (minimal persistent identity data)
|
||||
op.create_table(
|
||||
'telemetry_replicated_identity',
|
||||
sa.Column(
|
||||
'id',
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
primary_key=True,
|
||||
server_default='1',
|
||||
),
|
||||
sa.Column(
|
||||
'customer_id',
|
||||
sa.String(255),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
'instance_id',
|
||||
sa.String(255),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
'created_at',
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text('CURRENT_TIMESTAMP'),
|
||||
),
|
||||
sa.Column(
|
||||
'updated_at',
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text('CURRENT_TIMESTAMP'),
|
||||
),
|
||||
)
|
||||
|
||||
# Add constraint to ensure single row in telemetry_replicated_identity
|
||||
op.create_check_constraint(
|
||||
'single_identity_row', 'telemetry_replicated_identity', 'id = 1'
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop telemetry tables."""
|
||||
# Drop indexes first
|
||||
op.drop_index('ix_telemetry_metrics_uploaded_at', 'telemetry_metrics')
|
||||
op.drop_index('ix_telemetry_metrics_collected_at', 'telemetry_metrics')
|
||||
|
||||
# Drop tables
|
||||
op.drop_table('telemetry_replicated_identity')
|
||||
op.drop_table('telemetry_metrics')
|
||||
@@ -0,0 +1,39 @@
|
||||
"""rename user_secrets table to custom_secrets
|
||||
|
||||
Revision ID: 079
|
||||
Revises: 078
|
||||
Create Date: 2025-10-27 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '079'
|
||||
down_revision: Union[str, None] = '078'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Rename the table from user_secrets to custom_secrets
|
||||
op.rename_table('user_secrets', 'custom_secrets')
|
||||
|
||||
# Rename the index to match the new table name
|
||||
op.drop_index('idx_user_secrets_keycloak_user_id', 'custom_secrets')
|
||||
op.create_index(
|
||||
'idx_custom_secrets_keycloak_user_id', 'custom_secrets', ['keycloak_user_id']
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Rename the index back to the original name
|
||||
op.drop_index('idx_custom_secrets_keycloak_user_id', 'custom_secrets')
|
||||
op.create_index(
|
||||
'idx_user_secrets_keycloak_user_id', 'custom_secrets', ['keycloak_user_id']
|
||||
)
|
||||
|
||||
# Rename the table back from custom_secrets to user_secrets
|
||||
op.rename_table('custom_secrets', 'user_secrets')
|
||||
3947
enterprise/poetry.lock
generated
3947
enterprise/poetry.lock
generated
File diff suppressed because one or more lines are too long
@@ -11,7 +11,7 @@ description = "Deploy OpenHands"
|
||||
authors = [ "OpenHands" ]
|
||||
license = "POLYFORM"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/All-Hands-AI/OpenHands"
|
||||
repository = "https://github.com/OpenHands/OpenHands"
|
||||
packages = [
|
||||
{ include = "server" },
|
||||
{ include = "storage" },
|
||||
|
||||
@@ -31,7 +31,7 @@ from openhands.integrations.provider import (
|
||||
)
|
||||
from openhands.server.settings import Settings
|
||||
from openhands.server.user_auth.user_auth import AuthType, UserAuth
|
||||
from openhands.storage.data_models.user_secrets import UserSecrets
|
||||
from openhands.storage.data_models.secrets import Secrets
|
||||
from openhands.storage.settings.settings_store import SettingsStore
|
||||
|
||||
token_manager = TokenManager()
|
||||
@@ -52,7 +52,7 @@ class SaasUserAuth(UserAuth):
|
||||
settings_store: SaasSettingsStore | None = None
|
||||
secrets_store: SaasSecretsStore | None = None
|
||||
_settings: Settings | None = None
|
||||
_user_secrets: UserSecrets | None = None
|
||||
_secrets: Secrets | None = None
|
||||
accepted_tos: bool | None = None
|
||||
auth_type: AuthType = AuthType.COOKIE
|
||||
|
||||
@@ -119,13 +119,13 @@ class SaasUserAuth(UserAuth):
|
||||
self.secrets_store = secrets_store
|
||||
return secrets_store
|
||||
|
||||
async def get_user_secrets(self):
|
||||
user_secrets = self._user_secrets
|
||||
async def get_secrets(self):
|
||||
user_secrets = self._secrets
|
||||
if user_secrets:
|
||||
return user_secrets
|
||||
secrets_store = await self.get_secrets_store()
|
||||
user_secrets = await secrets_store.load()
|
||||
self._user_secrets = user_secrets
|
||||
self._secrets = user_secrets
|
||||
return user_secrets
|
||||
|
||||
async def get_access_token(self) -> SecretStr | None:
|
||||
@@ -148,7 +148,7 @@ class SaasUserAuth(UserAuth):
|
||||
if not access_token:
|
||||
raise AuthError()
|
||||
|
||||
user_secrets = await self.get_user_secrets()
|
||||
user_secrets = await self.get_secrets()
|
||||
|
||||
try:
|
||||
# TODO: I think we can do this in a single request if we refactor
|
||||
|
||||
@@ -37,6 +37,7 @@ from storage.offline_token_store import OfflineTokenStore
|
||||
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt
|
||||
|
||||
from openhands.integrations.service_types import ProviderType
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
|
||||
def _before_sleep_callback(retry_state: RetryCallState) -> None:
|
||||
@@ -191,7 +192,7 @@ class TokenManager:
|
||||
access_token: str,
|
||||
idp: ProviderType,
|
||||
) -> dict[str, str | int]:
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
base_url = KEYCLOAK_SERVER_URL_EXT if self.external else KEYCLOAK_SERVER_URL
|
||||
url = f'{base_url}/realms/{KEYCLOAK_REALM_NAME}/broker/{idp.value}/token'
|
||||
headers = {
|
||||
@@ -293,11 +294,12 @@ class TokenManager:
|
||||
refresh_token_expires_at: int,
|
||||
) -> dict[str, str | int] | None:
|
||||
current_time = int(time.time())
|
||||
# expire access_token ten minutes before actual expiration
|
||||
# expire access_token four hours before actual expiration
|
||||
# This ensures tokens are refreshed on resume to have at least 4 hours validity
|
||||
access_expired = (
|
||||
False
|
||||
if access_token_expires_at == 0
|
||||
else access_token_expires_at < current_time + 600
|
||||
else access_token_expires_at < current_time + 14400
|
||||
)
|
||||
refresh_expired = (
|
||||
False
|
||||
@@ -349,7 +351,7 @@ class TokenManager:
|
||||
'refresh_token': refresh_token,
|
||||
'grant_type': 'refresh_token',
|
||||
}
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(url, data=payload)
|
||||
response.raise_for_status()
|
||||
logger.info('Successfully refreshed GitHub token')
|
||||
@@ -375,7 +377,7 @@ class TokenManager:
|
||||
'refresh_token': refresh_token,
|
||||
'grant_type': 'refresh_token',
|
||||
}
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(url, data=payload)
|
||||
response.raise_for_status()
|
||||
logger.info('Successfully refreshed GitLab token')
|
||||
@@ -403,7 +405,7 @@ class TokenManager:
|
||||
'refresh_token': refresh_token,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(url, data=data, headers=headers)
|
||||
response.raise_for_status()
|
||||
logger.info('Successfully refreshed Bitbucket token')
|
||||
|
||||
@@ -3,43 +3,45 @@ from datetime import UTC, datetime
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, field_validator
|
||||
from server.config import get_config
|
||||
from server.constants import LITE_LLM_API_KEY, LITE_LLM_API_URL
|
||||
from storage.api_key_store import ApiKeyStore
|
||||
from storage.database import session_maker
|
||||
from storage.user_settings import UserSettings
|
||||
from storage.saas_settings_store import SaasSettingsStore
|
||||
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.server.user_auth import get_user_id
|
||||
from openhands.utils.async_utils import call_sync_from_async
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
|
||||
# Helper functions for BYOR API key management
|
||||
async def get_byor_key_from_db(user_id: str) -> str | None:
|
||||
"""Get the BYOR key from the database for a user."""
|
||||
config = get_config()
|
||||
settings_store = SaasSettingsStore(
|
||||
user_id=user_id, session_maker=session_maker, config=config
|
||||
)
|
||||
|
||||
def _get_byor_key():
|
||||
with session_maker() as session:
|
||||
user_db_settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == user_id)
|
||||
.first()
|
||||
)
|
||||
if user_db_settings and user_db_settings.llm_api_key_for_byor:
|
||||
return user_db_settings.llm_api_key_for_byor
|
||||
return None
|
||||
|
||||
return await call_sync_from_async(_get_byor_key)
|
||||
user_db_settings = await call_sync_from_async(
|
||||
settings_store.get_user_settings_by_keycloak_id, user_id
|
||||
)
|
||||
if user_db_settings and user_db_settings.llm_api_key_for_byor:
|
||||
return user_db_settings.llm_api_key_for_byor
|
||||
return None
|
||||
|
||||
|
||||
async def store_byor_key_in_db(user_id: str, key: str) -> None:
|
||||
"""Store the BYOR key in the database for a user."""
|
||||
config = get_config()
|
||||
settings_store = SaasSettingsStore(
|
||||
user_id=user_id, session_maker=session_maker, config=config
|
||||
)
|
||||
|
||||
def _update_user_settings():
|
||||
with session_maker() as session:
|
||||
user_db_settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == user_id)
|
||||
.first()
|
||||
user_db_settings = settings_store.get_user_settings_by_keycloak_id(
|
||||
user_id, session
|
||||
)
|
||||
if user_db_settings:
|
||||
user_db_settings.llm_api_key_for_byor = key
|
||||
@@ -67,9 +69,10 @@ async def generate_byor_key(user_id: str) -> str | None:
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={
|
||||
'x-goog-api-key': LITE_LLM_API_KEY,
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
response = await client.post(
|
||||
f'{LITE_LLM_API_URL}/key/generate',
|
||||
@@ -119,9 +122,10 @@ async def delete_byor_key_from_litellm(user_id: str, byor_key: str) -> bool:
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={
|
||||
'x-goog-api-key': LITE_LLM_API_KEY,
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
# Delete the key directly using the key value
|
||||
delete_url = f'{LITE_LLM_API_URL}/key/delete'
|
||||
|
||||
@@ -16,10 +16,11 @@ from server.auth.constants import (
|
||||
from server.auth.gitlab_sync import schedule_gitlab_repo_sync
|
||||
from server.auth.saas_user_auth import SaasUserAuth
|
||||
from server.auth.token_manager import TokenManager
|
||||
from server.config import sign_token
|
||||
from server.config import get_config, sign_token
|
||||
from server.constants import IS_FEATURE_ENV
|
||||
from server.routes.event_webhook import _get_session_api_key, _get_user_id
|
||||
from storage.database import session_maker
|
||||
from storage.saas_settings_store import SaasSettingsStore
|
||||
from storage.user_settings import UserSettings
|
||||
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
@@ -212,16 +213,14 @@ async def keycloak_callback(
|
||||
f'&state={state}'
|
||||
)
|
||||
|
||||
has_accepted_tos = False
|
||||
with session_maker() as session:
|
||||
user_settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == user_id)
|
||||
.first()
|
||||
)
|
||||
has_accepted_tos = (
|
||||
user_settings is not None and user_settings.accepted_tos is not None
|
||||
)
|
||||
config = get_config()
|
||||
settings_store = SaasSettingsStore(
|
||||
user_id=user_id, session_maker=session_maker, config=config
|
||||
)
|
||||
user_settings = settings_store.get_user_settings_by_keycloak_id(user_id)
|
||||
has_accepted_tos = (
|
||||
user_settings is not None and user_settings.accepted_tos is not None
|
||||
)
|
||||
|
||||
# If the user hasn't accepted the TOS, redirect to the TOS page
|
||||
if not has_accepted_tos:
|
||||
|
||||
@@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
from integrations import stripe_service
|
||||
from pydantic import BaseModel
|
||||
from server.config import get_config
|
||||
from server.constants import (
|
||||
LITE_LLM_API_KEY,
|
||||
LITE_LLM_API_URL,
|
||||
@@ -22,15 +23,47 @@ from server.constants import (
|
||||
from server.logger import logger
|
||||
from storage.billing_session import BillingSession
|
||||
from storage.database import session_maker
|
||||
from storage.saas_settings_store import SaasSettingsStore
|
||||
from storage.subscription_access import SubscriptionAccess
|
||||
from storage.user_settings import UserSettings
|
||||
|
||||
from openhands.server.user_auth import get_user_id
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
stripe.api_key = STRIPE_API_KEY
|
||||
billing_router = APIRouter(prefix='/api/billing')
|
||||
|
||||
|
||||
# TODO: Add a new app_mode named "ON_PREM" to support self-hosted customers instead of doing this
|
||||
# and members should comment out the "validate_saas_environment" function if they are developing and testing locally.
|
||||
def is_all_hands_saas_environment(request: Request) -> bool:
|
||||
"""Check if the current domain is an All Hands SaaS environment.
|
||||
|
||||
Args:
|
||||
request: FastAPI Request object
|
||||
|
||||
Returns:
|
||||
True if the current domain contains "all-hands.dev" or "openhands.dev" postfix
|
||||
"""
|
||||
hostname = request.url.hostname or ''
|
||||
return hostname.endswith('all-hands.dev') or hostname.endswith('openhands.dev')
|
||||
|
||||
|
||||
def validate_saas_environment(request: Request) -> None:
|
||||
"""Validate that the request is coming from an All Hands SaaS environment.
|
||||
|
||||
Args:
|
||||
request: FastAPI Request object
|
||||
|
||||
Raises:
|
||||
HTTPException: If the request is not from an All Hands SaaS environment
|
||||
"""
|
||||
if not is_all_hands_saas_environment(request):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail='Checkout sessions are only available for All Hands SaaS environments',
|
||||
)
|
||||
|
||||
|
||||
class BillingSessionType(Enum):
|
||||
DIRECT_PAYMENT = 'DIRECT_PAYMENT'
|
||||
MONTHLY_SUBSCRIPTION = 'MONTHLY_SUBSCRIPTION'
|
||||
@@ -78,7 +111,7 @@ def calculate_credits(user_info: LiteLlmUserInfo) -> float:
|
||||
async def get_credits(user_id: str = Depends(get_user_id)) -> GetCreditsResponse:
|
||||
if not stripe_service.STRIPE_API_KEY:
|
||||
return GetCreditsResponse()
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
user_json = await _get_litellm_user(client, user_id)
|
||||
credits = calculate_credits(user_json['user_info'])
|
||||
return GetCreditsResponse(credits=Decimal('{:.2f}'.format(credits)))
|
||||
@@ -196,6 +229,8 @@ async def cancel_subscription(user_id: str = Depends(get_user_id)) -> JSONRespon
|
||||
async def create_customer_setup_session(
|
||||
request: Request, user_id: str = Depends(get_user_id)
|
||||
) -> CreateBillingSessionResponse:
|
||||
validate_saas_environment(request)
|
||||
|
||||
customer_id = await stripe_service.find_or_create_customer(user_id)
|
||||
checkout_session = await stripe.checkout.Session.create_async(
|
||||
customer=customer_id,
|
||||
@@ -214,6 +249,8 @@ async def create_checkout_session(
|
||||
request: Request,
|
||||
user_id: str = Depends(get_user_id),
|
||||
) -> CreateBillingSessionResponse:
|
||||
validate_saas_environment(request)
|
||||
|
||||
customer_id = await stripe_service.find_or_create_customer(user_id)
|
||||
checkout_session = await stripe.checkout.Session.create_async(
|
||||
customer=customer_id,
|
||||
@@ -268,6 +305,8 @@ async def create_subscription_checkout_session(
|
||||
billing_session_type: BillingSessionType = BillingSessionType.MONTHLY_SUBSCRIPTION,
|
||||
user_id: str = Depends(get_user_id),
|
||||
) -> CreateBillingSessionResponse:
|
||||
validate_saas_environment(request)
|
||||
|
||||
# Prevent duplicate subscriptions for the same user
|
||||
with session_maker() as session:
|
||||
now = datetime.now(UTC)
|
||||
@@ -343,6 +382,8 @@ async def create_subscription_checkout_session_via_get(
|
||||
user_id: str = Depends(get_user_id),
|
||||
) -> RedirectResponse:
|
||||
"""Create a subscription checkout session using a GET request (For easier copy / paste to URL bar)."""
|
||||
validate_saas_environment(request)
|
||||
|
||||
response = await create_subscription_checkout_session(
|
||||
request, billing_session_type, user_id
|
||||
)
|
||||
@@ -390,7 +431,7 @@ async def success_callback(session_id: str, request: Request):
|
||||
)
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
# Update max budget in litellm
|
||||
user_json = await _get_litellm_user(client, billing_session.user_id)
|
||||
amount_subtotal = stripe_session.amount_subtotal or 0
|
||||
@@ -578,11 +619,14 @@ async def stripe_webhook(request: Request) -> JSONResponse:
|
||||
|
||||
def reset_user_to_free_tier_settings(user_id: str) -> None:
|
||||
"""Reset user settings to free tier defaults when subscription ends."""
|
||||
config = get_config()
|
||||
settings_store = SaasSettingsStore(
|
||||
user_id=user_id, session_maker=session_maker, config=config
|
||||
)
|
||||
|
||||
with session_maker() as session:
|
||||
user_settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == user_id)
|
||||
.first()
|
||||
user_settings = settings_store.get_user_settings_by_keycloak_id(
|
||||
user_id, session
|
||||
)
|
||||
|
||||
if user_settings:
|
||||
|
||||
@@ -11,6 +11,7 @@ from fastapi.responses import RedirectResponse
|
||||
from server.logger import logger
|
||||
|
||||
from openhands.server.shared import config
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
GITHUB_PROXY_ENDPOINTS = bool(os.environ.get('GITHUB_PROXY_ENDPOINTS'))
|
||||
|
||||
@@ -87,7 +88,7 @@ def add_github_proxy_routes(app: FastAPI):
|
||||
]
|
||||
body = urlencode(query_params, doseq=True)
|
||||
url = 'https://github.com/login/oauth/access_token'
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(url, content=body)
|
||||
return Response(
|
||||
response.content,
|
||||
@@ -101,7 +102,7 @@ def add_github_proxy_routes(app: FastAPI):
|
||||
logger.info(f'github_proxy_post:1:{path}')
|
||||
body = await request.body()
|
||||
url = f'https://github.com/{path}'
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(verify=httpx_verify_option()) as client:
|
||||
response = await client.post(url, content=body, headers=request.headers)
|
||||
return Response(
|
||||
response.content,
|
||||
|
||||
@@ -52,6 +52,7 @@ from openhands.storage.locations import (
|
||||
get_conversation_events_dir,
|
||||
)
|
||||
from openhands.utils.async_utils import call_sync_from_async
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
from openhands.utils.import_utils import get_impl
|
||||
from openhands.utils.shutdown_listener import should_continue
|
||||
from openhands.utils.utils import create_registry_and_conversation_stats
|
||||
@@ -266,9 +267,10 @@ class SaasNestedConversationManager(ConversationManager):
|
||||
):
|
||||
logger.info('starting_nested_conversation', extra={'sid': sid})
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={
|
||||
'X-Session-API-Key': session_api_key,
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
await self._setup_nested_settings(client, api_url, settings)
|
||||
await self._setup_provider_tokens(client, api_url, settings)
|
||||
@@ -484,9 +486,10 @@ class SaasNestedConversationManager(ConversationManager):
|
||||
raise ValueError(f'no_such_conversation:{sid}')
|
||||
nested_url = self._get_nested_url_for_runtime(runtime['runtime_id'], sid)
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={
|
||||
'X-Session-API-Key': runtime['session_api_key'],
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
response = await client.post(f'{nested_url}/events', json=data)
|
||||
response.raise_for_status()
|
||||
@@ -551,9 +554,10 @@ class SaasNestedConversationManager(ConversationManager):
|
||||
return None
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={
|
||||
'X-Session-API-Key': session_api_key,
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
# Query the nested runtime for conversation info
|
||||
response = await client.get(nested_url)
|
||||
@@ -828,6 +832,7 @@ class SaasNestedConversationManager(ConversationManager):
|
||||
@contextlib.asynccontextmanager
|
||||
async def _httpx_client(self):
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={'X-API-Key': self.config.sandbox.api_key or ''},
|
||||
timeout=_HTTP_TIMEOUT,
|
||||
) as client:
|
||||
|
||||
@@ -7,11 +7,11 @@ from dataclasses import dataclass
|
||||
from cryptography.fernet import Fernet
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from storage.database import session_maker
|
||||
from storage.stored_user_secrets import StoredUserSecrets
|
||||
from storage.stored_custom_secrets import StoredCustomSecrets
|
||||
|
||||
from openhands.core.config.openhands_config import OpenHandsConfig
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.storage.data_models.user_secrets import UserSecrets
|
||||
from openhands.storage.data_models.secrets import Secrets
|
||||
from openhands.storage.secrets.secrets_store import SecretsStore
|
||||
|
||||
|
||||
@@ -21,20 +21,20 @@ class SaasSecretsStore(SecretsStore):
|
||||
session_maker: sessionmaker
|
||||
config: OpenHandsConfig
|
||||
|
||||
async def load(self) -> UserSecrets | None:
|
||||
async def load(self) -> Secrets | None:
|
||||
if not self.user_id:
|
||||
return None
|
||||
|
||||
with self.session_maker() as session:
|
||||
# Fetch all secrets for the given user ID
|
||||
settings = (
|
||||
session.query(StoredUserSecrets)
|
||||
.filter(StoredUserSecrets.keycloak_user_id == self.user_id)
|
||||
session.query(StoredCustomSecrets)
|
||||
.filter(StoredCustomSecrets.keycloak_user_id == self.user_id)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not settings:
|
||||
return UserSecrets()
|
||||
return Secrets()
|
||||
|
||||
kwargs = {}
|
||||
for secret in settings:
|
||||
@@ -45,14 +45,14 @@ class SaasSecretsStore(SecretsStore):
|
||||
|
||||
self._decrypt_kwargs(kwargs)
|
||||
|
||||
return UserSecrets(custom_secrets=kwargs) # type: ignore[arg-type]
|
||||
return Secrets(custom_secrets=kwargs) # type: ignore[arg-type]
|
||||
|
||||
async def store(self, item: UserSecrets):
|
||||
async def store(self, item: Secrets):
|
||||
with self.session_maker() as session:
|
||||
# Incoming secrets are always the most updated ones
|
||||
# Delete all existing records and override with incoming ones
|
||||
session.query(StoredUserSecrets).filter(
|
||||
StoredUserSecrets.keycloak_user_id == self.user_id
|
||||
session.query(StoredCustomSecrets).filter(
|
||||
StoredCustomSecrets.keycloak_user_id == self.user_id
|
||||
).delete()
|
||||
|
||||
# Prepare the new secrets data
|
||||
@@ -74,7 +74,7 @@ class SaasSecretsStore(SecretsStore):
|
||||
|
||||
# Add the new secrets
|
||||
for secret_name, secret_value, description in secret_tuples:
|
||||
new_secret = StoredUserSecrets(
|
||||
new_secret = StoredCustomSecrets(
|
||||
keycloak_user_id=self.user_id,
|
||||
secret_name=secret_name,
|
||||
secret_value=secret_value,
|
||||
|
||||
@@ -24,7 +24,6 @@ from server.constants import (
|
||||
from server.logger import logger
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from storage.database import session_maker
|
||||
from storage.stored_settings import StoredSettings
|
||||
from storage.user_settings import UserSettings
|
||||
|
||||
from openhands.core.config.openhands_config import OpenHandsConfig
|
||||
@@ -32,6 +31,7 @@ from openhands.server.settings import Settings
|
||||
from openhands.storage import get_file_store
|
||||
from openhands.storage.settings.settings_store import SettingsStore
|
||||
from openhands.utils.async_utils import call_sync_from_async
|
||||
from openhands.utils.http_session import httpx_verify_option
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,15 +40,46 @@ class SaasSettingsStore(SettingsStore):
|
||||
session_maker: sessionmaker
|
||||
config: OpenHandsConfig
|
||||
|
||||
def get_user_settings_by_keycloak_id(
|
||||
self, keycloak_user_id: str, session=None
|
||||
) -> UserSettings | None:
|
||||
"""
|
||||
Get UserSettings by keycloak_user_id.
|
||||
|
||||
Args:
|
||||
keycloak_user_id: The keycloak user ID to search for
|
||||
session: Optional existing database session. If not provided, creates a new one.
|
||||
|
||||
Returns:
|
||||
UserSettings object if found, None otherwise
|
||||
"""
|
||||
if not keycloak_user_id:
|
||||
return None
|
||||
|
||||
def _get_settings():
|
||||
if session:
|
||||
# Use provided session
|
||||
return (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == keycloak_user_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
# Create new session
|
||||
with self.session_maker() as new_session:
|
||||
return (
|
||||
new_session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == keycloak_user_id)
|
||||
.first()
|
||||
)
|
||||
|
||||
return _get_settings()
|
||||
|
||||
async def load(self) -> Settings | None:
|
||||
if not self.user_id:
|
||||
return None
|
||||
with self.session_maker() as session:
|
||||
settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == self.user_id)
|
||||
.first()
|
||||
)
|
||||
settings = self.get_user_settings_by_keycloak_id(self.user_id, session)
|
||||
|
||||
if not settings or settings.user_version != CURRENT_USER_SETTINGS_VERSION:
|
||||
logger.info(
|
||||
@@ -72,12 +103,8 @@ class SaasSettingsStore(SettingsStore):
|
||||
if item:
|
||||
kwargs = item.model_dump(context={'expose_secrets': True})
|
||||
self._encrypt_kwargs(kwargs)
|
||||
query = session.query(UserSettings).filter(
|
||||
UserSettings.keycloak_user_id == self.user_id
|
||||
)
|
||||
|
||||
# First check if we have an existing entry in the new table
|
||||
existing = query.first()
|
||||
existing = self.get_user_settings_by_keycloak_id(self.user_id, session)
|
||||
|
||||
kwargs = {
|
||||
key: value
|
||||
@@ -144,33 +171,6 @@ class SaasSettingsStore(SettingsStore):
|
||||
await self.store(settings)
|
||||
return settings
|
||||
|
||||
def load_legacy_db_settings(self, github_user_id: str) -> Settings | None:
|
||||
if not github_user_id:
|
||||
return None
|
||||
|
||||
with self.session_maker() as session:
|
||||
settings = (
|
||||
session.query(StoredSettings)
|
||||
.filter(StoredSettings.id == github_user_id)
|
||||
.first()
|
||||
)
|
||||
if settings is None:
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
'saas_settings_store:load_legacy_db_settings:found',
|
||||
extra={'github_user_id': github_user_id},
|
||||
)
|
||||
kwargs = {
|
||||
c.name: getattr(settings, c.name)
|
||||
for c in StoredSettings.__table__.columns
|
||||
if c.name in Settings.model_fields
|
||||
}
|
||||
self._decrypt_kwargs(kwargs)
|
||||
del kwargs['secrets_store']
|
||||
settings = Settings(**kwargs)
|
||||
return settings
|
||||
|
||||
async def load_legacy_file_store_settings(self, github_user_id: str):
|
||||
if not github_user_id:
|
||||
return None
|
||||
@@ -216,9 +216,10 @@ class SaasSettingsStore(SettingsStore):
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
verify=httpx_verify_option(),
|
||||
headers={
|
||||
'x-goog-api-key': LITE_LLM_API_KEY,
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
# Get the previous max budget to prevent accidental loss
|
||||
# In Litellm a get always succeeds, regardless of whether the user actually exists
|
||||
@@ -235,10 +236,8 @@ class SaasSettingsStore(SettingsStore):
|
||||
spend = user_info.get('spend') or 0
|
||||
|
||||
with session_maker() as session:
|
||||
user_settings = (
|
||||
session.query(UserSettings)
|
||||
.filter(UserSettings.keycloak_user_id == self.user_id)
|
||||
.first()
|
||||
user_settings = self.get_user_settings_by_keycloak_id(
|
||||
self.user_id, session
|
||||
)
|
||||
# In upgrade to V4, we no longer use billing margin, but instead apply this directly
|
||||
# in litellm. The default billing marign was 2 before this (hence the magic numbers below)
|
||||
|
||||
@@ -2,8 +2,8 @@ from sqlalchemy import Column, Identity, Integer, String
|
||||
from storage.base import Base
|
||||
|
||||
|
||||
class StoredUserSecrets(Base): # type: ignore
|
||||
__tablename__ = 'user_secrets'
|
||||
class StoredCustomSecrets(Base): # type: ignore
|
||||
__tablename__ = 'custom_secrets'
|
||||
id = Column(Integer, Identity(), primary_key=True)
|
||||
keycloak_user_id = Column(String, nullable=True, index=True)
|
||||
secret_name = Column(String, nullable=False)
|
||||
@@ -1,29 +0,0 @@
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import JSON, Boolean, Column, Float, Integer, String
|
||||
from storage.base import Base
|
||||
|
||||
|
||||
class StoredSettings(Base): # type: ignore
|
||||
"""
|
||||
Legacy user settings storage. This should be considered deprecated - use UserSettings isntead
|
||||
"""
|
||||
|
||||
__tablename__ = 'settings'
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
language = Column(String, nullable=True)
|
||||
agent = Column(String, nullable=True)
|
||||
max_iterations = Column(Integer, nullable=True)
|
||||
security_analyzer = Column(String, nullable=True)
|
||||
confirmation_mode = Column(Boolean, nullable=True, default=False)
|
||||
llm_model = Column(String, nullable=True)
|
||||
llm_api_key = Column(String, nullable=True)
|
||||
llm_base_url = Column(String, nullable=True)
|
||||
remote_runtime_resource_factor = Column(Integer, nullable=True)
|
||||
enable_default_condenser = Column(Boolean, nullable=False, default=True)
|
||||
user_consents_to_analytics = Column(Boolean, nullable=True)
|
||||
margin = Column(Float, nullable=True)
|
||||
enable_sound_notifications = Column(Boolean, nullable=True, default=False)
|
||||
sandbox_base_container_image = Column(String, nullable=True)
|
||||
sandbox_runtime_container_image = Column(String, nullable=True)
|
||||
secrets_store = Column(JSON, nullable=True)
|
||||
98
enterprise/storage/telemetry_identity.py
Normal file
98
enterprise/storage/telemetry_identity.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""SQLAlchemy model for telemetry identity.
|
||||
|
||||
This model stores persistent identity information that must survive container restarts
|
||||
for the OpenHands Enterprise Telemetry Service.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import CheckConstraint, Column, DateTime, Integer, String
|
||||
from storage.base import Base
|
||||
|
||||
|
||||
class TelemetryIdentity(Base): # type: ignore
|
||||
"""Stores persistent identity information for telemetry.
|
||||
|
||||
This table is designed to contain exactly one row (enforced by database constraint)
|
||||
that maintains only the identity data that cannot be reliably recomputed:
|
||||
- customer_id: Established relationship with Replicated
|
||||
- instance_id: Generated once, must remain stable
|
||||
|
||||
Operational data like timestamps are derived from the telemetry_metrics table.
|
||||
"""
|
||||
|
||||
__tablename__ = 'telemetry_replicated_identity'
|
||||
__table_args__ = (CheckConstraint('id = 1', name='single_identity_row'),)
|
||||
|
||||
id = Column(Integer, primary_key=True, default=1)
|
||||
customer_id = Column(String(255), nullable=True)
|
||||
instance_id = Column(String(255), nullable=True)
|
||||
created_at = Column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
customer_id: Optional[str] = None,
|
||||
instance_id: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize telemetry identity.
|
||||
|
||||
Args:
|
||||
customer_id: Unique identifier for the customer
|
||||
instance_id: Unique identifier for this OpenHands instance
|
||||
**kwargs: Additional keyword arguments for SQLAlchemy
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Set defaults for fields that would normally be set by SQLAlchemy
|
||||
now = datetime.now(UTC)
|
||||
if not hasattr(self, 'created_at') or self.created_at is None:
|
||||
self.created_at = now
|
||||
if not hasattr(self, 'updated_at') or self.updated_at is None:
|
||||
self.updated_at = now
|
||||
|
||||
# Force id to be 1 to maintain single-row constraint
|
||||
self.id = 1
|
||||
self.customer_id = customer_id
|
||||
self.instance_id = instance_id
|
||||
|
||||
def set_customer_info(
|
||||
self,
|
||||
customer_id: Optional[str] = None,
|
||||
instance_id: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Update customer and instance identification information.
|
||||
|
||||
Args:
|
||||
customer_id: Unique identifier for the customer
|
||||
instance_id: Unique identifier for this OpenHands instance
|
||||
"""
|
||||
if customer_id is not None:
|
||||
self.customer_id = customer_id
|
||||
if instance_id is not None:
|
||||
self.instance_id = instance_id
|
||||
|
||||
@property
|
||||
def has_customer_info(self) -> bool:
|
||||
"""Check if customer identification information is configured."""
|
||||
return bool(self.customer_id and self.instance_id)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"<TelemetryIdentity(customer_id='{self.customer_id}', "
|
||||
f"instance_id='{self.instance_id}')>"
|
||||
)
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
112
enterprise/storage/telemetry_metrics.py
Normal file
112
enterprise/storage/telemetry_metrics.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""SQLAlchemy model for telemetry metrics data.
|
||||
|
||||
This model stores individual metric collection records with upload tracking
|
||||
and retry logic for the OpenHands Enterprise Telemetry Service.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from sqlalchemy import JSON, Column, DateTime, Integer, String, Text
|
||||
from storage.base import Base
|
||||
|
||||
|
||||
class TelemetryMetrics(Base): # type: ignore
|
||||
"""Stores collected telemetry metrics with upload tracking.
|
||||
|
||||
Each record represents a single metrics collection event with associated
|
||||
metadata for upload status and retry logic.
|
||||
"""
|
||||
|
||||
__tablename__ = 'telemetry_metrics'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
collected_at = Column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
index=True,
|
||||
)
|
||||
metrics_data = Column(JSON, nullable=False)
|
||||
uploaded_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
upload_attempts = Column(Integer, nullable=False, default=0)
|
||||
last_upload_error = Column(Text, nullable=True)
|
||||
created_at = Column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metrics_data: Dict[str, Any],
|
||||
collected_at: Optional[datetime] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize a new telemetry metrics record.
|
||||
|
||||
Args:
|
||||
metrics_data: Dictionary containing the collected metrics
|
||||
collected_at: Timestamp when metrics were collected (defaults to now)
|
||||
**kwargs: Additional keyword arguments for SQLAlchemy
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Set defaults for fields that would normally be set by SQLAlchemy
|
||||
now = datetime.now(UTC)
|
||||
if not hasattr(self, 'id') or self.id is None:
|
||||
self.id = str(uuid.uuid4())
|
||||
if not hasattr(self, 'upload_attempts') or self.upload_attempts is None:
|
||||
self.upload_attempts = 0
|
||||
if not hasattr(self, 'created_at') or self.created_at is None:
|
||||
self.created_at = now
|
||||
if not hasattr(self, 'updated_at') or self.updated_at is None:
|
||||
self.updated_at = now
|
||||
|
||||
self.metrics_data = metrics_data
|
||||
if collected_at:
|
||||
self.collected_at = collected_at
|
||||
elif not hasattr(self, 'collected_at') or self.collected_at is None:
|
||||
self.collected_at = now
|
||||
|
||||
def mark_uploaded(self) -> None:
|
||||
"""Mark this metrics record as successfully uploaded."""
|
||||
self.uploaded_at = datetime.now(UTC)
|
||||
self.last_upload_error = None
|
||||
|
||||
def mark_upload_failed(self, error_message: str) -> None:
|
||||
"""Mark this metrics record as having failed upload.
|
||||
|
||||
Args:
|
||||
error_message: Description of the upload failure
|
||||
"""
|
||||
self.upload_attempts += 1
|
||||
self.last_upload_error = error_message
|
||||
self.uploaded_at = None
|
||||
|
||||
@property
|
||||
def is_uploaded(self) -> bool:
|
||||
"""Check if this metrics record has been successfully uploaded."""
|
||||
return self.uploaded_at is not None
|
||||
|
||||
@property
|
||||
def needs_retry(self) -> bool:
|
||||
"""Check if this metrics record needs upload retry (failed but not too many attempts)."""
|
||||
return not self.is_uploaded and self.upload_attempts < 3
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"<TelemetryMetrics(id='{self.id}', "
|
||||
f"collected_at='{self.collected_at}', "
|
||||
f'uploaded={self.is_uploaded})>'
|
||||
)
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -17,7 +17,6 @@ from storage.github_app_installation import GithubAppInstallation
|
||||
from storage.maintenance_task import MaintenanceTask, MaintenanceTaskStatus
|
||||
from storage.stored_conversation_metadata import StoredConversationMetadata
|
||||
from storage.stored_offline_token import StoredOfflineToken
|
||||
from storage.stored_settings import StoredSettings
|
||||
from storage.stripe_customer import StripeCustomer
|
||||
from storage.user_settings import UserSettings
|
||||
|
||||
@@ -85,7 +84,7 @@ def add_minimal_fixtures(session_maker):
|
||||
updated_at=datetime.fromisoformat('2025-03-08'),
|
||||
)
|
||||
)
|
||||
session.add(StoredSettings(id='mock-user-id', user_consents_to_analytics=True))
|
||||
|
||||
session.add(
|
||||
StripeCustomer(
|
||||
keycloak_user_id='mock-user-id',
|
||||
|
||||
1
enterprise/tests/unit/experiments/__init__.py
Normal file
1
enterprise/tests/unit/experiments/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Unit tests for experiments module."""
|
||||
@@ -0,0 +1,137 @@
|
||||
# tests/test_condenser_max_step_experiment_v1.py
|
||||
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
from experiments.experiment_manager import SaaSExperimentManager
|
||||
|
||||
# SUT imports (update the module path if needed)
|
||||
from experiments.experiment_versions._004_condenser_max_step_experiment import (
|
||||
handle_condenser_max_step_experiment__v1,
|
||||
)
|
||||
from pydantic import SecretStr
|
||||
|
||||
from openhands.sdk import LLM, Agent
|
||||
from openhands.sdk.context.condenser import LLMSummarizingCondenser
|
||||
|
||||
|
||||
def make_agent() -> Agent:
|
||||
"""Build a minimal valid Agent."""
|
||||
llm = LLM(
|
||||
usage_id='primary-llm',
|
||||
model='provider/model',
|
||||
api_key=SecretStr('sk-test'),
|
||||
)
|
||||
return Agent(llm=llm)
|
||||
|
||||
|
||||
def _patch_variant(monkeypatch, return_value):
|
||||
"""Patch the internal variant getter to return a specific value."""
|
||||
monkeypatch.setattr(
|
||||
'experiments.experiment_versions._004_condenser_max_step_experiment._get_condenser_max_step_variant',
|
||||
lambda user_id, conv_id: return_value,
|
||||
raising=True,
|
||||
)
|
||||
|
||||
|
||||
def test_control_variant_sets_condenser_with_max_size_120(monkeypatch):
|
||||
_patch_variant(monkeypatch, 'control')
|
||||
agent = make_agent()
|
||||
conv_id = uuid4()
|
||||
|
||||
result = handle_condenser_max_step_experiment__v1('user-1', conv_id, agent)
|
||||
|
||||
# Should be a new Agent instance with a condenser installed
|
||||
assert result is not agent
|
||||
assert isinstance(result.condenser, LLMSummarizingCondenser)
|
||||
|
||||
# The condenser should have its own LLM (usage_id overridden to "condenser")
|
||||
assert result.condenser.llm.usage_id == 'condenser'
|
||||
# The original agent LLM remains unchanged
|
||||
assert agent.llm.usage_id == 'primary-llm'
|
||||
|
||||
# Control: max_size = 120, keep_first = 4
|
||||
assert result.condenser.max_size == 120
|
||||
assert result.condenser.keep_first == 4
|
||||
|
||||
|
||||
def test_treatment_variant_sets_condenser_with_max_size_80(monkeypatch):
|
||||
_patch_variant(monkeypatch, 'treatment')
|
||||
agent = make_agent()
|
||||
conv_id = uuid4()
|
||||
|
||||
result = handle_condenser_max_step_experiment__v1('user-2', conv_id, agent)
|
||||
|
||||
assert result is not agent
|
||||
assert isinstance(result.condenser, LLMSummarizingCondenser)
|
||||
assert result.condenser.llm.usage_id == 'condenser'
|
||||
assert result.condenser.max_size == 80
|
||||
assert result.condenser.keep_first == 4
|
||||
|
||||
|
||||
def test_none_variant_returns_original_agent_without_changes(monkeypatch):
|
||||
_patch_variant(monkeypatch, None)
|
||||
agent = make_agent()
|
||||
conv_id = uuid4()
|
||||
|
||||
result = handle_condenser_max_step_experiment__v1('user-3', conv_id, agent)
|
||||
|
||||
# No changes—same instance and no condenser attribute added
|
||||
assert result is agent
|
||||
assert getattr(result, 'condenser', None) is None
|
||||
|
||||
|
||||
def test_unknown_variant_returns_original_agent_without_changes(monkeypatch):
|
||||
_patch_variant(monkeypatch, 'weird-variant')
|
||||
agent = make_agent()
|
||||
conv_id = uuid4()
|
||||
|
||||
result = handle_condenser_max_step_experiment__v1('user-4', conv_id, agent)
|
||||
|
||||
assert result is agent
|
||||
assert getattr(result, 'condenser', None) is None
|
||||
|
||||
|
||||
@patch('experiments.experiment_manager.handle_condenser_max_step_experiment__v1')
|
||||
@patch('experiments.experiment_manager.ENABLE_EXPERIMENT_MANAGER', False)
|
||||
def test_run_agent_variant_tests_v1_noop_when_manager_disabled(
|
||||
mock_handle_condenser,
|
||||
):
|
||||
"""If ENABLE_EXPERIMENT_MANAGER is False, the method returns the exact same agent and does not call the handler."""
|
||||
agent = make_agent()
|
||||
conv_id = uuid4()
|
||||
|
||||
result = SaaSExperimentManager.run_agent_variant_tests__v1(
|
||||
user_id='user-123',
|
||||
conversation_id=conv_id,
|
||||
agent=agent,
|
||||
)
|
||||
|
||||
# Same object returned (no copy)
|
||||
assert result is agent
|
||||
# Handler should not have been called
|
||||
mock_handle_condenser.assert_not_called()
|
||||
|
||||
|
||||
@patch('experiments.experiment_manager.ENABLE_EXPERIMENT_MANAGER', True)
|
||||
@patch('experiments.experiment_manager.EXPERIMENT_SYSTEM_PROMPT_EXPERIMENT', True)
|
||||
def test_run_agent_variant_tests_v1_calls_handler_and_sets_system_prompt(monkeypatch):
|
||||
"""When enabled, it should call the condenser experiment handler and set the long-horizon system prompt."""
|
||||
agent = make_agent()
|
||||
conv_id = uuid4()
|
||||
|
||||
_patch_variant(monkeypatch, 'treatment')
|
||||
|
||||
result: Agent = SaaSExperimentManager.run_agent_variant_tests__v1(
|
||||
user_id='user-abc',
|
||||
conversation_id=conv_id,
|
||||
agent=agent,
|
||||
)
|
||||
|
||||
# Should be a different instance than the original (copied after handler runs)
|
||||
assert result is not agent
|
||||
assert result.system_prompt_filename == 'system_prompt_long_horizon.j2'
|
||||
|
||||
# The condenser returned by the handler must be preserved after the system-prompt override copy
|
||||
assert isinstance(result.condenser, LLMSummarizingCondenser)
|
||||
assert result.condenser.max_size == 80
|
||||
@@ -137,7 +137,9 @@ class TestJiraExistingConversationView:
|
||||
):
|
||||
"""Test conversation update with no metadata"""
|
||||
mock_store = AsyncMock()
|
||||
mock_store.get_metadata.return_value = None
|
||||
mock_store.get_metadata.side_effect = FileNotFoundError(
|
||||
'No such file or directory'
|
||||
)
|
||||
mock_store_impl.return_value = mock_store
|
||||
|
||||
with pytest.raises(
|
||||
@@ -307,7 +309,7 @@ class TestJiraViewEdgeCases:
|
||||
mock_agent_loop_info,
|
||||
):
|
||||
"""Test conversation creation when user has no secrets"""
|
||||
new_conversation_view.saas_user_auth.get_user_secrets.return_value = None
|
||||
new_conversation_view.saas_user_auth.get_secrets.return_value = None
|
||||
mock_create_conversation.return_value = mock_agent_loop_info
|
||||
mock_store.create_conversation = AsyncMock()
|
||||
|
||||
|
||||
@@ -137,7 +137,9 @@ class TestJiraDcExistingConversationView:
|
||||
):
|
||||
"""Test conversation update with no metadata"""
|
||||
mock_store = AsyncMock()
|
||||
mock_store.get_metadata.return_value = None
|
||||
mock_store.get_metadata.side_effect = FileNotFoundError(
|
||||
'No such file or directory'
|
||||
)
|
||||
mock_store_impl.return_value = mock_store
|
||||
|
||||
with pytest.raises(
|
||||
@@ -307,7 +309,7 @@ class TestJiraDcViewEdgeCases:
|
||||
mock_agent_loop_info,
|
||||
):
|
||||
"""Test conversation creation when user has no secrets"""
|
||||
new_conversation_view.saas_user_auth.get_user_secrets.return_value = None
|
||||
new_conversation_view.saas_user_auth.get_secrets.return_value = None
|
||||
mock_create_conversation.return_value = mock_agent_loop_info
|
||||
mock_store.create_conversation = AsyncMock()
|
||||
|
||||
|
||||
@@ -137,7 +137,9 @@ class TestLinearExistingConversationView:
|
||||
):
|
||||
"""Test conversation update with no metadata"""
|
||||
mock_store = AsyncMock()
|
||||
mock_store.get_metadata.return_value = None
|
||||
mock_store.get_metadata.side_effect = FileNotFoundError(
|
||||
'No such file or directory'
|
||||
)
|
||||
mock_store_impl.return_value = mock_store
|
||||
|
||||
with pytest.raises(
|
||||
@@ -307,7 +309,7 @@ class TestLinearViewEdgeCases:
|
||||
mock_agent_loop_info,
|
||||
):
|
||||
"""Test conversation creation when user has no secrets"""
|
||||
new_conversation_view.saas_user_auth.get_user_secrets.return_value = None
|
||||
new_conversation_view.saas_user_auth.get_secrets.return_value = None
|
||||
mock_create_conversation.return_value = mock_agent_loop_info
|
||||
mock_store.create_conversation = AsyncMock()
|
||||
|
||||
|
||||
1
enterprise/tests/unit/storage/__init__.py
Normal file
1
enterprise/tests/unit/storage/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# Storage unit tests
|
||||
129
enterprise/tests/unit/storage/test_telemetry_identity.py
Normal file
129
enterprise/tests/unit/storage/test_telemetry_identity.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""Unit tests for TelemetryIdentity model.
|
||||
|
||||
Tests the persistent identity storage for the OpenHands Enterprise Telemetry Service.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from storage.telemetry_identity import TelemetryIdentity
|
||||
|
||||
|
||||
class TestTelemetryIdentity:
|
||||
"""Test cases for TelemetryIdentity model."""
|
||||
|
||||
def test_create_identity_with_defaults(self):
|
||||
"""Test creating identity with default values."""
|
||||
identity = TelemetryIdentity()
|
||||
|
||||
assert identity.id == 1
|
||||
assert identity.customer_id is None
|
||||
assert identity.instance_id is None
|
||||
assert isinstance(identity.created_at, datetime)
|
||||
assert isinstance(identity.updated_at, datetime)
|
||||
|
||||
def test_create_identity_with_values(self):
|
||||
"""Test creating identity with specific values."""
|
||||
customer_id = 'cust_123'
|
||||
instance_id = 'inst_456'
|
||||
|
||||
identity = TelemetryIdentity(customer_id=customer_id, instance_id=instance_id)
|
||||
|
||||
assert identity.id == 1
|
||||
assert identity.customer_id == customer_id
|
||||
assert identity.instance_id == instance_id
|
||||
|
||||
def test_set_customer_info(self):
|
||||
"""Test updating customer information."""
|
||||
identity = TelemetryIdentity()
|
||||
|
||||
# Update customer info
|
||||
identity.set_customer_info(
|
||||
customer_id='new_customer', instance_id='new_instance'
|
||||
)
|
||||
|
||||
assert identity.customer_id == 'new_customer'
|
||||
assert identity.instance_id == 'new_instance'
|
||||
|
||||
def test_set_customer_info_partial(self):
|
||||
"""Test partial updates of customer information."""
|
||||
identity = TelemetryIdentity(
|
||||
customer_id='original_customer', instance_id='original_instance'
|
||||
)
|
||||
|
||||
# Update only customer_id
|
||||
identity.set_customer_info(customer_id='updated_customer')
|
||||
assert identity.customer_id == 'updated_customer'
|
||||
assert identity.instance_id == 'original_instance'
|
||||
|
||||
# Update only instance_id
|
||||
identity.set_customer_info(instance_id='updated_instance')
|
||||
assert identity.customer_id == 'updated_customer'
|
||||
assert identity.instance_id == 'updated_instance'
|
||||
|
||||
def test_set_customer_info_with_none(self):
|
||||
"""Test that None values don't overwrite existing data."""
|
||||
identity = TelemetryIdentity(
|
||||
customer_id='existing_customer', instance_id='existing_instance'
|
||||
)
|
||||
|
||||
# Call with None values - should not change existing data
|
||||
identity.set_customer_info(customer_id=None, instance_id=None)
|
||||
assert identity.customer_id == 'existing_customer'
|
||||
assert identity.instance_id == 'existing_instance'
|
||||
|
||||
def test_has_customer_info_property(self):
|
||||
"""Test has_customer_info property logic."""
|
||||
identity = TelemetryIdentity()
|
||||
|
||||
# Initially false (both None)
|
||||
assert not identity.has_customer_info
|
||||
|
||||
# Still false with only customer_id
|
||||
identity.customer_id = 'customer_123'
|
||||
assert not identity.has_customer_info
|
||||
|
||||
# Still false with only instance_id
|
||||
identity.customer_id = None
|
||||
identity.instance_id = 'instance_456'
|
||||
assert not identity.has_customer_info
|
||||
|
||||
# True when both are set
|
||||
identity.customer_id = 'customer_123'
|
||||
identity.instance_id = 'instance_456'
|
||||
assert identity.has_customer_info
|
||||
|
||||
def test_has_customer_info_with_empty_strings(self):
|
||||
"""Test has_customer_info with empty strings."""
|
||||
identity = TelemetryIdentity(customer_id='', instance_id='')
|
||||
|
||||
# Empty strings should be falsy
|
||||
assert not identity.has_customer_info
|
||||
|
||||
def test_repr_method(self):
|
||||
"""Test string representation of identity."""
|
||||
identity = TelemetryIdentity(
|
||||
customer_id='test_customer', instance_id='test_instance'
|
||||
)
|
||||
|
||||
repr_str = repr(identity)
|
||||
assert 'TelemetryIdentity' in repr_str
|
||||
assert 'test_customer' in repr_str
|
||||
assert 'test_instance' in repr_str
|
||||
|
||||
def test_id_forced_to_one(self):
|
||||
"""Test that ID is always forced to 1."""
|
||||
identity = TelemetryIdentity()
|
||||
assert identity.id == 1
|
||||
|
||||
# Even if we try to set a different ID in constructor
|
||||
identity2 = TelemetryIdentity(customer_id='test')
|
||||
assert identity2.id == 1
|
||||
|
||||
def test_timestamps_are_set(self):
|
||||
"""Test that timestamps are properly set."""
|
||||
identity = TelemetryIdentity()
|
||||
|
||||
assert identity.created_at is not None
|
||||
assert identity.updated_at is not None
|
||||
assert isinstance(identity.created_at, datetime)
|
||||
assert isinstance(identity.updated_at, datetime)
|
||||
190
enterprise/tests/unit/storage/test_telemetry_metrics.py
Normal file
190
enterprise/tests/unit/storage/test_telemetry_metrics.py
Normal file
@@ -0,0 +1,190 @@
|
||||
"""Unit tests for TelemetryMetrics model."""
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from storage.telemetry_metrics import TelemetryMetrics
|
||||
|
||||
|
||||
class TestTelemetryMetrics:
|
||||
"""Test cases for TelemetryMetrics model."""
|
||||
|
||||
def test_init_with_metrics_data(self):
|
||||
"""Test initialization with metrics data."""
|
||||
metrics_data = {
|
||||
'cpu_usage': 75.5,
|
||||
'memory_usage': 1024,
|
||||
'active_sessions': 5,
|
||||
}
|
||||
|
||||
metrics = TelemetryMetrics(metrics_data=metrics_data)
|
||||
|
||||
assert metrics.metrics_data == metrics_data
|
||||
assert metrics.upload_attempts == 0
|
||||
assert metrics.uploaded_at is None
|
||||
assert metrics.last_upload_error is None
|
||||
assert metrics.collected_at is not None
|
||||
assert metrics.created_at is not None
|
||||
assert metrics.updated_at is not None
|
||||
|
||||
def test_init_with_custom_collected_at(self):
|
||||
"""Test initialization with custom collected_at timestamp."""
|
||||
metrics_data = {'test': 'value'}
|
||||
custom_time = datetime(2023, 1, 1, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
metrics = TelemetryMetrics(metrics_data=metrics_data, collected_at=custom_time)
|
||||
|
||||
assert metrics.collected_at == custom_time
|
||||
|
||||
def test_mark_uploaded(self):
|
||||
"""Test marking metrics as uploaded."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
|
||||
# Initially not uploaded
|
||||
assert not metrics.is_uploaded
|
||||
assert metrics.uploaded_at is None
|
||||
|
||||
# Mark as uploaded
|
||||
metrics.mark_uploaded()
|
||||
|
||||
assert metrics.is_uploaded
|
||||
|
||||
def test_mark_upload_failed(self):
|
||||
"""Test marking upload as failed."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
error_message = 'Network timeout'
|
||||
|
||||
# Initially no failures
|
||||
assert metrics.upload_attempts == 0
|
||||
assert metrics.last_upload_error is None
|
||||
|
||||
# Mark as failed
|
||||
metrics.mark_upload_failed(error_message)
|
||||
|
||||
assert metrics.upload_attempts == 1
|
||||
assert metrics.last_upload_error == error_message
|
||||
assert metrics.uploaded_at is None
|
||||
assert not metrics.is_uploaded
|
||||
|
||||
def test_multiple_upload_failures(self):
|
||||
"""Test multiple upload failures increment attempts."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
|
||||
metrics.mark_upload_failed('Error 1')
|
||||
assert metrics.upload_attempts == 1
|
||||
|
||||
metrics.mark_upload_failed('Error 2')
|
||||
assert metrics.upload_attempts == 2
|
||||
assert metrics.last_upload_error == 'Error 2'
|
||||
|
||||
def test_is_uploaded_property(self):
|
||||
"""Test is_uploaded property."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
|
||||
# Initially not uploaded
|
||||
assert not metrics.is_uploaded
|
||||
|
||||
# After marking uploaded
|
||||
metrics.mark_uploaded()
|
||||
assert metrics.is_uploaded
|
||||
|
||||
def test_needs_retry_property(self):
|
||||
"""Test needs_retry property logic."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
|
||||
# Initially needs retry (0 attempts, not uploaded)
|
||||
assert metrics.needs_retry
|
||||
|
||||
# After 1 failure, still needs retry
|
||||
metrics.mark_upload_failed('Error 1')
|
||||
assert metrics.needs_retry
|
||||
|
||||
# After 2 failures, still needs retry
|
||||
metrics.mark_upload_failed('Error 2')
|
||||
assert metrics.needs_retry
|
||||
|
||||
# After 3 failures, no more retries
|
||||
metrics.mark_upload_failed('Error 3')
|
||||
assert not metrics.needs_retry
|
||||
|
||||
# Reset and test successful upload
|
||||
metrics2 = TelemetryMetrics(metrics_data={'test': 'data'}) # type: ignore[unreachable]
|
||||
metrics2.mark_uploaded()
|
||||
# After upload, needs_retry should be False since is_uploaded is True
|
||||
|
||||
def test_upload_failure_clears_uploaded_at(self):
|
||||
"""Test that upload failure clears uploaded_at timestamp."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
|
||||
# Mark as uploaded first
|
||||
metrics.mark_uploaded()
|
||||
assert metrics.uploaded_at is not None
|
||||
|
||||
# Mark as failed - should clear uploaded_at
|
||||
metrics.mark_upload_failed('Network error')
|
||||
assert metrics.uploaded_at is None
|
||||
|
||||
def test_successful_upload_clears_error(self):
|
||||
"""Test that successful upload clears error message."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
|
||||
# Mark as failed first
|
||||
metrics.mark_upload_failed('Network error')
|
||||
assert metrics.last_upload_error == 'Network error'
|
||||
|
||||
# Mark as uploaded - should clear error
|
||||
metrics.mark_uploaded()
|
||||
assert metrics.last_upload_error is None
|
||||
|
||||
def test_uuid_generation(self):
|
||||
"""Test that each instance gets a unique UUID."""
|
||||
metrics1 = TelemetryMetrics(metrics_data={'test': 'data1'})
|
||||
metrics2 = TelemetryMetrics(metrics_data={'test': 'data2'})
|
||||
|
||||
assert metrics1.id != metrics2.id
|
||||
assert isinstance(uuid.UUID(metrics1.id), uuid.UUID)
|
||||
assert isinstance(uuid.UUID(metrics2.id), uuid.UUID)
|
||||
|
||||
def test_repr(self):
|
||||
"""Test string representation."""
|
||||
metrics = TelemetryMetrics(metrics_data={'test': 'data'})
|
||||
repr_str = repr(metrics)
|
||||
|
||||
assert 'TelemetryMetrics' in repr_str
|
||||
assert metrics.id in repr_str
|
||||
assert str(metrics.collected_at) in repr_str
|
||||
assert 'uploaded=False' in repr_str
|
||||
|
||||
# Test after upload
|
||||
metrics.mark_uploaded()
|
||||
repr_str = repr(metrics)
|
||||
assert 'uploaded=True' in repr_str
|
||||
|
||||
def test_complex_metrics_data(self):
|
||||
"""Test with complex nested metrics data."""
|
||||
complex_data = {
|
||||
'system': {
|
||||
'cpu': {'usage': 75.5, 'cores': 8},
|
||||
'memory': {'total': 16384, 'used': 8192},
|
||||
},
|
||||
'sessions': [
|
||||
{'id': 'session1', 'duration': 3600},
|
||||
{'id': 'session2', 'duration': 1800},
|
||||
],
|
||||
'timestamp': '2023-01-01T12:00:00Z',
|
||||
}
|
||||
|
||||
metrics = TelemetryMetrics(metrics_data=complex_data)
|
||||
|
||||
assert metrics.metrics_data == complex_data
|
||||
|
||||
def test_empty_metrics_data(self):
|
||||
"""Test with empty metrics data."""
|
||||
metrics = TelemetryMetrics(metrics_data={})
|
||||
|
||||
assert metrics.metrics_data == {}
|
||||
|
||||
def test_config_class(self):
|
||||
"""Test that Config class is properly set."""
|
||||
assert hasattr(TelemetryMetrics, 'Config')
|
||||
assert TelemetryMetrics.Config.from_attributes is True
|
||||
@@ -36,6 +36,46 @@ def session_maker(engine):
|
||||
return sessionmaker(bind=engine)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request():
|
||||
"""Create a mock request object with proper URL structure for testing."""
|
||||
return Request(
|
||||
scope={
|
||||
'type': 'http',
|
||||
'path': '/api/billing/test',
|
||||
'server': ('test.com', 80),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_checkout_request():
|
||||
"""Create a mock request object for checkout session tests."""
|
||||
request = Request(
|
||||
scope={
|
||||
'type': 'http',
|
||||
'path': '/api/billing/create-checkout-session',
|
||||
'server': ('test.com', 80),
|
||||
}
|
||||
)
|
||||
request._base_url = URL('http://test.com/')
|
||||
return request
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_subscription_request():
|
||||
"""Create a mock request object for subscription checkout session tests."""
|
||||
request = Request(
|
||||
scope={
|
||||
'type': 'http',
|
||||
'path': '/api/billing/subscription-checkout-session',
|
||||
'server': ('test.com', 80),
|
||||
}
|
||||
)
|
||||
request._base_url = URL('http://test.com/')
|
||||
return request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_credits_lite_llm_error():
|
||||
mock_request = Request(scope={'type': 'http', 'state': {'user_id': 'mock_user'}})
|
||||
@@ -90,14 +130,10 @@ async def test_get_credits_success():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_checkout_session_stripe_error(session_maker):
|
||||
async def test_create_checkout_session_stripe_error(
|
||||
session_maker, mock_checkout_request
|
||||
):
|
||||
"""Test handling of Stripe API errors."""
|
||||
mock_request = Request(
|
||||
scope={
|
||||
'type': 'http',
|
||||
}
|
||||
)
|
||||
mock_request._base_url = URL('http://test.com/')
|
||||
|
||||
mock_customer = stripe.Customer(
|
||||
id='mock-customer', metadata={'user_id': 'mock-user'}
|
||||
@@ -118,17 +154,16 @@ async def test_create_checkout_session_stripe_error(session_maker):
|
||||
'server.auth.token_manager.TokenManager.get_user_info_from_user_id',
|
||||
AsyncMock(return_value={'email': 'testy@tester.com'}),
|
||||
),
|
||||
patch('server.routes.billing.validate_saas_environment'),
|
||||
):
|
||||
await create_checkout_session(
|
||||
CreateCheckoutSessionRequest(amount=25), mock_request, 'mock_user'
|
||||
CreateCheckoutSessionRequest(amount=25), mock_checkout_request, 'mock_user'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_checkout_session_success(session_maker):
|
||||
async def test_create_checkout_session_success(session_maker, mock_checkout_request):
|
||||
"""Test successful creation of checkout session."""
|
||||
mock_request = Request(scope={'type': 'http'})
|
||||
mock_request._base_url = URL('http://test.com/')
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.url = 'https://checkout.stripe.com/test-session'
|
||||
@@ -152,12 +187,13 @@ async def test_create_checkout_session_success(session_maker):
|
||||
'server.auth.token_manager.TokenManager.get_user_info_from_user_id',
|
||||
AsyncMock(return_value={'email': 'testy@tester.com'}),
|
||||
),
|
||||
patch('server.routes.billing.validate_saas_environment'),
|
||||
):
|
||||
mock_db_session = MagicMock()
|
||||
mock_session_maker.return_value.__enter__.return_value = mock_db_session
|
||||
|
||||
result = await create_checkout_session(
|
||||
CreateCheckoutSessionRequest(amount=25), mock_request, 'mock_user'
|
||||
CreateCheckoutSessionRequest(amount=25), mock_checkout_request, 'mock_user'
|
||||
)
|
||||
|
||||
assert isinstance(result, CreateBillingSessionResponse)
|
||||
@@ -590,7 +626,9 @@ async def test_cancel_subscription_stripe_error():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_subscription_checkout_session_duplicate_prevention():
|
||||
async def test_create_subscription_checkout_session_duplicate_prevention(
|
||||
mock_subscription_request,
|
||||
):
|
||||
"""Test that creating a subscription when user already has active subscription raises error."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
@@ -609,11 +647,9 @@ async def test_create_subscription_checkout_session_duplicate_prevention():
|
||||
cancelled_at=None,
|
||||
)
|
||||
|
||||
mock_request = Request(scope={'type': 'http'})
|
||||
mock_request._base_url = URL('http://test.com/')
|
||||
|
||||
with (
|
||||
patch('server.routes.billing.session_maker') as mock_session_maker,
|
||||
patch('server.routes.billing.validate_saas_environment'),
|
||||
):
|
||||
# Setup mock session to return existing active subscription
|
||||
mock_session = MagicMock()
|
||||
@@ -623,7 +659,7 @@ async def test_create_subscription_checkout_session_duplicate_prevention():
|
||||
# Call the function and expect HTTPException
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await create_subscription_checkout_session(
|
||||
mock_request, user_id='test_user'
|
||||
mock_subscription_request, user_id='test_user'
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
@@ -634,10 +670,10 @@ async def test_create_subscription_checkout_session_duplicate_prevention():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_subscription_checkout_session_allows_after_cancellation():
|
||||
async def test_create_subscription_checkout_session_allows_after_cancellation(
|
||||
mock_subscription_request,
|
||||
):
|
||||
"""Test that creating a subscription is allowed when previous subscription was cancelled."""
|
||||
mock_request = Request(scope={'type': 'http'})
|
||||
mock_request._base_url = URL('http://test.com/')
|
||||
|
||||
mock_session_obj = MagicMock()
|
||||
mock_session_obj.url = 'https://checkout.stripe.com/test-session'
|
||||
@@ -657,6 +693,7 @@ async def test_create_subscription_checkout_session_allows_after_cancellation():
|
||||
'server.routes.billing.SUBSCRIPTION_PRICE_DATA',
|
||||
{'MONTHLY_SUBSCRIPTION': {'unit_amount': 2000}},
|
||||
),
|
||||
patch('server.routes.billing.validate_saas_environment'),
|
||||
):
|
||||
# Setup mock session - the query should return None because cancelled subscriptions are filtered out
|
||||
mock_session = MagicMock()
|
||||
@@ -665,7 +702,7 @@ async def test_create_subscription_checkout_session_allows_after_cancellation():
|
||||
|
||||
# Should succeed
|
||||
result = await create_subscription_checkout_session(
|
||||
mock_request, user_id='test_user'
|
||||
mock_subscription_request, user_id='test_user'
|
||||
)
|
||||
|
||||
assert isinstance(result, CreateBillingSessionResponse)
|
||||
@@ -673,10 +710,10 @@ async def test_create_subscription_checkout_session_allows_after_cancellation():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_subscription_checkout_session_success_no_existing():
|
||||
async def test_create_subscription_checkout_session_success_no_existing(
|
||||
mock_subscription_request,
|
||||
):
|
||||
"""Test successful subscription creation when no existing subscription."""
|
||||
mock_request = Request(scope={'type': 'http'})
|
||||
mock_request._base_url = URL('http://test.com/')
|
||||
|
||||
mock_session_obj = MagicMock()
|
||||
mock_session_obj.url = 'https://checkout.stripe.com/test-session'
|
||||
@@ -696,6 +733,7 @@ async def test_create_subscription_checkout_session_success_no_existing():
|
||||
'server.routes.billing.SUBSCRIPTION_PRICE_DATA',
|
||||
{'MONTHLY_SUBSCRIPTION': {'unit_amount': 2000}},
|
||||
),
|
||||
patch('server.routes.billing.validate_saas_environment'),
|
||||
):
|
||||
# Setup mock session to return no existing subscription
|
||||
mock_session = MagicMock()
|
||||
@@ -704,7 +742,7 @@ async def test_create_subscription_checkout_session_success_no_existing():
|
||||
|
||||
# Should succeed
|
||||
result = await create_subscription_checkout_session(
|
||||
mock_request, user_id='test_user'
|
||||
mock_subscription_request, user_id='test_user'
|
||||
)
|
||||
|
||||
assert isinstance(result, CreateBillingSessionResponse)
|
||||
|
||||
@@ -8,8 +8,8 @@ pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
# Mock the call_sync_from_async function to return the result of the function directly
|
||||
def mock_call_sync_from_async(func):
|
||||
return func()
|
||||
def mock_call_sync_from_async(func, *args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -5,11 +5,11 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
from pydantic import SecretStr
|
||||
from storage.saas_secrets_store import SaasSecretsStore
|
||||
from storage.stored_user_secrets import StoredUserSecrets
|
||||
from storage.stored_custom_secrets import StoredCustomSecrets
|
||||
|
||||
from openhands.core.config.openhands_config import OpenHandsConfig
|
||||
from openhands.integrations.provider import CustomSecret
|
||||
from openhands.storage.data_models.user_secrets import UserSecrets
|
||||
from openhands.storage.data_models.secrets import Secrets
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -27,8 +27,8 @@ def secrets_store(session_maker, mock_config):
|
||||
class TestSaasSecretsStore:
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_and_load(self, secrets_store):
|
||||
# Create a UserSecrets object with some test data
|
||||
user_secrets = UserSecrets(
|
||||
# Create a Secrets object with some test data
|
||||
user_secrets = Secrets(
|
||||
custom_secrets=MappingProxyType(
|
||||
{
|
||||
'api_token': CustomSecret.from_value(
|
||||
@@ -60,8 +60,8 @@ class TestSaasSecretsStore:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encryption_decryption(self, secrets_store):
|
||||
# Create a UserSecrets object with sensitive data
|
||||
user_secrets = UserSecrets(
|
||||
# Create a Secrets object with sensitive data
|
||||
user_secrets = Secrets(
|
||||
custom_secrets=MappingProxyType(
|
||||
{
|
||||
'api_token': CustomSecret.from_value(
|
||||
@@ -87,8 +87,8 @@ class TestSaasSecretsStore:
|
||||
# Verify the data is encrypted in the database
|
||||
with secrets_store.session_maker() as session:
|
||||
stored = (
|
||||
session.query(StoredUserSecrets)
|
||||
.filter(StoredUserSecrets.keycloak_user_id == 'user-id')
|
||||
session.query(StoredCustomSecrets)
|
||||
.filter(StoredCustomSecrets.keycloak_user_id == 'user-id')
|
||||
.first()
|
||||
)
|
||||
|
||||
@@ -154,7 +154,7 @@ class TestSaasSecretsStore:
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_existing_secrets(self, secrets_store):
|
||||
# Create and store initial secrets
|
||||
initial_secrets = UserSecrets(
|
||||
initial_secrets = Secrets(
|
||||
custom_secrets=MappingProxyType(
|
||||
{
|
||||
'api_token': CustomSecret.from_value(
|
||||
@@ -169,7 +169,7 @@ class TestSaasSecretsStore:
|
||||
await secrets_store.store(initial_secrets)
|
||||
|
||||
# Create and store updated secrets
|
||||
updated_secrets = UserSecrets(
|
||||
updated_secrets = Secrets(
|
||||
custom_secrets=MappingProxyType(
|
||||
{
|
||||
'api_token': CustomSecret.from_value(
|
||||
|
||||
@@ -8,7 +8,6 @@ from server.constants import (
|
||||
LITE_LLM_TEAM_ID,
|
||||
)
|
||||
from storage.saas_settings_store import SaasSettingsStore
|
||||
from storage.stored_settings import StoredSettings
|
||||
from storage.user_settings import UserSettings
|
||||
|
||||
from openhands.core.config.openhands_config import OpenHandsConfig
|
||||
@@ -303,26 +302,6 @@ async def test_create_default_settings_require_payment_disabled(
|
||||
assert settings.language == 'en'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_default_settings_with_existing_llm_key(
|
||||
settings_store, mock_stripe, mock_github_user, mock_litellm_api, session_maker
|
||||
):
|
||||
# Test that existing llm_api_key is preserved and not overwritten with litellm default
|
||||
with (
|
||||
patch('storage.saas_settings_store.REQUIRE_PAYMENT', False),
|
||||
patch('storage.saas_settings_store.LITE_LLM_API_KEY', 'mock-api-key'),
|
||||
patch('storage.saas_settings_store.session_maker', session_maker),
|
||||
):
|
||||
with settings_store.session_maker() as session:
|
||||
kwargs = {'id': '12345', 'language': 'en', 'llm_api_key': 'existing_key'}
|
||||
settings_store._encrypt_kwargs(kwargs)
|
||||
session.merge(StoredSettings(**kwargs))
|
||||
session.commit()
|
||||
updated_settings = await settings_store.create_default_settings(None)
|
||||
assert updated_settings is not None
|
||||
assert updated_settings.llm_api_key.get_secret_value() == 'test_api_key'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_default_lite_llm_settings_no_api_config(settings_store):
|
||||
with (
|
||||
|
||||
@@ -14,7 +14,7 @@ def slack_manager():
|
||||
@pytest.mark.parametrize(
|
||||
'message,expected',
|
||||
[
|
||||
('All-Hands-AI/Openhands', 'All-Hands-AI/Openhands'),
|
||||
('OpenHands/Openhands', 'OpenHands/Openhands'),
|
||||
('deploy repo', 'deploy'),
|
||||
('use hello world', None),
|
||||
],
|
||||
|
||||
@@ -13,7 +13,6 @@ from integrations.stripe_service import (
|
||||
)
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from storage.stored_settings import Base as StoredBase
|
||||
from storage.stripe_customer import Base as StripeCustomerBase
|
||||
from storage.stripe_customer import StripeCustomer
|
||||
from storage.user_settings import Base as UserBase
|
||||
@@ -22,7 +21,7 @@ from storage.user_settings import Base as UserBase
|
||||
@pytest.fixture
|
||||
def engine():
|
||||
engine = create_engine('sqlite:///:memory:')
|
||||
StoredBase.metadata.create_all(engine)
|
||||
|
||||
UserBase.metadata.create_all(engine)
|
||||
StripeCustomerBase.metadata.create_all(engine)
|
||||
return engine
|
||||
|
||||
@@ -74,8 +74,8 @@ def test_infer_repo_from_message():
|
||||
# Single GitHub URLs
|
||||
('Clone https://github.com/demo123/demo1.git', ['demo123/demo1']),
|
||||
(
|
||||
'Check out https://github.com/All-Hands-AI/OpenHands.git for details',
|
||||
['All-Hands-AI/OpenHands'],
|
||||
'Check out https://github.com/OpenHands/OpenHands.git for details',
|
||||
['OpenHands/OpenHands'],
|
||||
),
|
||||
('Visit https://github.com/microsoft/vscode', ['microsoft/vscode']),
|
||||
# Single GitLab URLs
|
||||
@@ -92,7 +92,7 @@ def test_infer_repo_from_message():
|
||||
['atlassian/atlassian-connect-express'],
|
||||
),
|
||||
# Single direct owner/repo mentions
|
||||
('Please deploy the All-Hands-AI/OpenHands repo', ['All-Hands-AI/OpenHands']),
|
||||
('Please deploy the OpenHands/OpenHands repo', ['OpenHands/OpenHands']),
|
||||
('I need help with the microsoft/vscode repository', ['microsoft/vscode']),
|
||||
('Check facebook/react for examples', ['facebook/react']),
|
||||
('The torvalds/linux kernel', ['torvalds/linux']),
|
||||
|
||||
@@ -6,14 +6,14 @@ This folder contains code and resources to run experiments and evaluations.
|
||||
|
||||
### Setup
|
||||
|
||||
Before starting evaluation, follow the instructions [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
|
||||
Before starting evaluation, follow the instructions [here](https://github.com/OpenHands/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
|
||||
|
||||
Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the [evaluation directory](#supported-benchmarks).
|
||||
Generally these will involve running `run_infer.py` to perform inference with the agents.
|
||||
|
||||
### Implementing and Evaluating an Agent
|
||||
|
||||
To add an agent to OpenHands, you will need to implement it in the [agenthub directory](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/agenthub). There is a README there with more information.
|
||||
To add an agent to OpenHands, you will need to implement it in the [agenthub directory](https://github.com/OpenHands/OpenHands/tree/main/openhands/agenthub). There is a README there with more information.
|
||||
|
||||
To evaluate an agent, you can provide the agent's name to the `run_infer.py` program.
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
|
||||
@@ -124,7 +124,7 @@ if __name__ == '__main__':
|
||||
)
|
||||
args, _ = parser.parse_known_args()
|
||||
|
||||
# Check https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/swe_bench/README.md#configure-openhands-and-your-llm
|
||||
# Check https://github.com/OpenHands/OpenHands/blob/main/evaluation/swe_bench/README.md#configure-openhands-and-your-llm
|
||||
# for details of how to set `llm_config`
|
||||
if args.llm_config:
|
||||
specified_llm_config = get_llm_config_arg(args.llm_config)
|
||||
|
||||
@@ -36,8 +36,8 @@ We use it to train strong LM agents that achieve state-of-the-art open results o
|
||||
The process of running SWE-Gym is very similar to how you'd run SWE-Bench evaluation.
|
||||
|
||||
|
||||
1. First, clone OpenHands repo `git clone https://github.com/All-Hands-AI/OpenHands.git`
|
||||
2. Then setup the repo following [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md)
|
||||
1. First, clone OpenHands repo `git clone https://github.com/OpenHands/OpenHands.git`
|
||||
2. Then setup the repo following [Development.md](https://github.com/OpenHands/OpenHands/blob/main/Development.md)
|
||||
3. Then you can simply serve your own model as an OpenAI compatible endpoint, put those info in config.toml. You can do this by following instruction [here](../../README.md#setup).
|
||||
4. And then simply do the following to sample for 16x parallelism:
|
||||
|
||||
|
||||
79
evaluation/benchmarks/multi_swe_bench/compute_skip_ids.py
Normal file
79
evaluation/benchmarks/multi_swe_bench/compute_skip_ids.py
Normal file
@@ -0,0 +1,79 @@
|
||||
import argparse
|
||||
import fnmatch
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def find_final_reports(base_dir, pattern=None):
|
||||
base_path = Path(base_dir)
|
||||
if not base_path.exists():
|
||||
raise FileNotFoundError(f'Base directory does not exist: {base_dir}')
|
||||
|
||||
# Find all final_report.json files
|
||||
all_reports = list(base_path.rglob('final_report.json'))
|
||||
|
||||
if pattern is None:
|
||||
return all_reports
|
||||
|
||||
# Filter by pattern
|
||||
filtered_reports = []
|
||||
for report in all_reports:
|
||||
# Get relative path from base_dir for matching
|
||||
rel_path = report.relative_to(base_path)
|
||||
if fnmatch.fnmatch(str(rel_path), pattern):
|
||||
filtered_reports.append(report)
|
||||
|
||||
return filtered_reports
|
||||
|
||||
|
||||
def collect_resolved_ids(report_files):
|
||||
id_counter = Counter()
|
||||
|
||||
for report_file in report_files:
|
||||
with open(report_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
if 'resolved_ids' not in data:
|
||||
raise KeyError(f"'resolved_ids' key not found in {report_file}")
|
||||
resolved_ids = data['resolved_ids']
|
||||
id_counter.update(resolved_ids)
|
||||
|
||||
return id_counter
|
||||
|
||||
|
||||
def get_skip_ids(id_counter, threshold):
|
||||
return [id_str for id_str, count in id_counter.items() if count >= threshold]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Compute SKIP_IDS from resolved IDs in final_report.json files'
|
||||
)
|
||||
parser.add_argument(
|
||||
'threshold',
|
||||
type=int,
|
||||
help='Minimum number of times an ID must be resolved to be skipped',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--base-dir',
|
||||
default='evaluation/evaluation_outputs/outputs',
|
||||
help='Base directory to search for final_report.json files (default: evaluation/evaluation_outputs/outputs)',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--pattern',
|
||||
default=None,
|
||||
help='Glob pattern to filter paths (e.g., "*Multi-SWE-RL*/**/*gpt*")',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
report_files = find_final_reports(args.base_dir, args.pattern)
|
||||
id_counter = collect_resolved_ids(report_files)
|
||||
|
||||
skip_ids = get_skip_ids(id_counter, args.threshold)
|
||||
skip_ids = [s.replace('/', '__').replace(':pr-', '-') for s in skip_ids]
|
||||
skip_ids = ','.join(sorted(skip_ids))
|
||||
print(skip_ids)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -80,7 +80,7 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = base_container_image
|
||||
|
||||
@@ -316,7 +316,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
@@ -747,10 +747,14 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
|
||||
subset = dataset[dataset[filter_column].isin(selected_ids)]
|
||||
logger.info(f'Retained {subset.shape[0]} tasks after filtering')
|
||||
return subset
|
||||
skip_ids = os.environ.get('SKIP_IDS', '').split(',')
|
||||
skip_ids = [id for id in os.environ.get('SKIP_IDS', '').split(',') if id]
|
||||
if len(skip_ids) > 0:
|
||||
logger.info(f'Dataset size before filtering: {dataset.shape[0]} tasks')
|
||||
logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')
|
||||
return dataset[~dataset[filter_column].isin(skip_ids)]
|
||||
logger.info(f'SKIP_IDS:\n{skip_ids}')
|
||||
filtered_dataset = dataset[~dataset[filter_column].isin(skip_ids)]
|
||||
logger.info(f'Dataset size after filtering: {filtered_dataset.shape[0]} tasks')
|
||||
return filtered_dataset
|
||||
return dataset
|
||||
|
||||
|
||||
@@ -768,6 +772,11 @@ if __name__ == '__main__':
|
||||
default='test',
|
||||
help='split to evaluate on',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--filter_dataset_after_sampling',
|
||||
action='store_true',
|
||||
help='if provided, filter dataset after sampling instead of before',
|
||||
)
|
||||
args, _ = parser.parse_known_args()
|
||||
|
||||
# NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
|
||||
@@ -777,10 +786,24 @@ if __name__ == '__main__':
|
||||
logger.info(f'Loading dataset {args.dataset} with split {args.split} ')
|
||||
dataset = load_dataset('json', data_files=args.dataset)
|
||||
dataset = dataset[args.split]
|
||||
swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')
|
||||
logger.info(
|
||||
f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'
|
||||
)
|
||||
swe_bench_tests = dataset.to_pandas()
|
||||
|
||||
# Determine filter strategy based on flag
|
||||
filter_func = None
|
||||
if args.filter_dataset_after_sampling:
|
||||
# Pass filter as callback to apply after sampling
|
||||
def filter_func(df):
|
||||
return filter_dataset(df, 'instance_id')
|
||||
|
||||
logger.info(
|
||||
f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks (filtering will occur after sampling)'
|
||||
)
|
||||
else:
|
||||
# Apply filter before sampling
|
||||
swe_bench_tests = filter_dataset(swe_bench_tests, 'instance_id')
|
||||
logger.info(
|
||||
f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'
|
||||
)
|
||||
|
||||
llm_config = None
|
||||
if args.llm_config:
|
||||
@@ -810,7 +833,9 @@ if __name__ == '__main__':
|
||||
|
||||
output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
|
||||
print(f'### OUTPUT FILE: {output_file} ###')
|
||||
instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
|
||||
instances = prepare_dataset(
|
||||
swe_bench_tests, output_file, args.eval_n_limit, filter_func=filter_func
|
||||
)
|
||||
|
||||
if len(instances) > 0 and not isinstance(
|
||||
instances['FAIL_TO_PASS'][instances['FAIL_TO_PASS'].index[0]], str
|
||||
|
||||
@@ -8,8 +8,14 @@
|
||||
MODEL=$1 # eg your llm config name in config.toml (eg: "llm.claude-3-5-sonnet-20241022-t05")
|
||||
EXP_NAME=$2 # "train-t05"
|
||||
EVAL_DATASET=$3 # path to original dataset (jsonl file)
|
||||
N_WORKERS=${4:-64}
|
||||
N_RUNS=${5:-1}
|
||||
MAX_ITER=$4
|
||||
N_WORKERS=${5:-64}
|
||||
N_RUNS=${6:-1}
|
||||
EVAL_LIMIT=${7:-}
|
||||
SKIP_IDS_THRESHOLD=$8
|
||||
SKIP_IDS_PATTERN=$9
|
||||
INPUT_SKIP_IDS=${10}
|
||||
FILTER_DATASET_AFTER_SAMPLING=${11:-}
|
||||
|
||||
export EXP_NAME=$EXP_NAME
|
||||
# use 2x resources for rollout since some codebases are pretty resource-intensive
|
||||
@@ -17,6 +23,7 @@ export DEFAULT_RUNTIME_RESOURCE_FACTOR=2
|
||||
echo "MODEL: $MODEL"
|
||||
echo "EXP_NAME: $EXP_NAME"
|
||||
echo "EVAL_DATASET: $EVAL_DATASET"
|
||||
echo "INPUT_SKIP_IDS: $INPUT_SKIP_IDS"
|
||||
# Generate DATASET path by adding _with_runtime_ before .jsonl extension
|
||||
DATASET="${EVAL_DATASET%.jsonl}_with_runtime_.jsonl" # path to converted dataset
|
||||
|
||||
@@ -35,9 +42,6 @@ else
|
||||
export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
|
||||
fi
|
||||
|
||||
#EVAL_LIMIT=3000
|
||||
MAX_ITER=100
|
||||
|
||||
|
||||
# ===== Run inference =====
|
||||
source "evaluation/utils/version_control.sh"
|
||||
@@ -69,17 +73,52 @@ function run_eval() {
|
||||
--dataset $DATASET \
|
||||
--split $SPLIT"
|
||||
|
||||
# Conditionally add filter flag
|
||||
if [ "$FILTER_DATASET_AFTER_SAMPLING" = "true" ]; then
|
||||
COMMAND="$COMMAND --filter_dataset_after_sampling"
|
||||
fi
|
||||
|
||||
echo "Running command: $COMMAND"
|
||||
if [ -n "$EVAL_LIMIT" ]; then
|
||||
echo "EVAL_LIMIT: $EVAL_LIMIT"
|
||||
COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
|
||||
fi
|
||||
|
||||
# Run the command
|
||||
eval $COMMAND
|
||||
}
|
||||
|
||||
for run_idx in $(seq 1 $N_RUNS); do
|
||||
if [ -n "$SKIP_IDS_THRESHOLD" ]; then
|
||||
echo "Computing SKIP_IDS for run $run_idx..."
|
||||
SKIP_CMD="poetry run python evaluation/benchmarks/multi_swe_bench/compute_skip_ids.py $SKIP_IDS_THRESHOLD"
|
||||
if [ -n "$SKIP_IDS_PATTERN" ]; then
|
||||
SKIP_CMD="$SKIP_CMD --pattern \"$SKIP_IDS_PATTERN\""
|
||||
fi
|
||||
COMPUTED_SKIP_IDS=$(eval $SKIP_CMD)
|
||||
SKIP_STATUS=$?
|
||||
if [ $SKIP_STATUS -ne 0 ]; then
|
||||
echo "ERROR: Skip IDs computation failed with exit code $SKIP_STATUS"
|
||||
exit $SKIP_STATUS
|
||||
fi
|
||||
echo "COMPUTED_SKIP_IDS: $COMPUTED_SKIP_IDS"
|
||||
else
|
||||
echo "SKIP_IDS_THRESHOLD not provided, skipping SKIP_IDS computation"
|
||||
COMPUTED_SKIP_IDS=""
|
||||
fi
|
||||
|
||||
# Concatenate COMPUTED_SKIP_IDS and INPUT_SKIP_IDS
|
||||
if [ -n "$COMPUTED_SKIP_IDS" ] && [ -n "$INPUT_SKIP_IDS" ]; then
|
||||
export SKIP_IDS="${COMPUTED_SKIP_IDS},${INPUT_SKIP_IDS}"
|
||||
elif [ -n "$COMPUTED_SKIP_IDS" ]; then
|
||||
export SKIP_IDS="$COMPUTED_SKIP_IDS"
|
||||
elif [ -n "$INPUT_SKIP_IDS" ]; then
|
||||
export SKIP_IDS="$INPUT_SKIP_IDS"
|
||||
else
|
||||
unset SKIP_IDS
|
||||
fi
|
||||
|
||||
echo "FINAL SKIP_IDS: $SKIP_IDS"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
echo "### Running inference... ###"
|
||||
|
||||
@@ -6,13 +6,13 @@ mkdir -p $EVAL_WORKSPACE
|
||||
|
||||
# 1. Prepare REPO
|
||||
echo "==== Prepare SWE-bench repo ===="
|
||||
OH_SWE_BENCH_REPO_PATH="https://github.com/All-Hands-AI/SWE-bench.git"
|
||||
OH_SWE_BENCH_REPO_PATH="https://github.com/OpenHands/SWE-bench.git"
|
||||
OH_SWE_BENCH_REPO_BRANCH="eval"
|
||||
git clone -b $OH_SWE_BENCH_REPO_BRANCH $OH_SWE_BENCH_REPO_PATH $EVAL_WORKSPACE/OH-SWE-bench
|
||||
|
||||
# 2. Prepare DATA
|
||||
echo "==== Prepare SWE-bench data ===="
|
||||
EVAL_IMAGE=ghcr.io/all-hands-ai/eval-swe-bench:builder_with_conda
|
||||
EVAL_IMAGE=ghcr.io/openhands/eval-swe-bench:builder_with_conda
|
||||
EVAL_WORKSPACE=$(realpath $EVAL_WORKSPACE)
|
||||
chmod +x $EVAL_WORKSPACE/OH-SWE-bench/swebench/harness/prepare_data.sh
|
||||
if [ -d $EVAL_WORKSPACE/eval_data ]; then
|
||||
|
||||
@@ -161,7 +161,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
|
||||
@@ -10,7 +10,7 @@ def verify_instance_costs(row: pd.Series) -> float:
|
||||
Verifies that the accumulated_cost matches the sum of individual costs in metrics.
|
||||
Also checks for duplicate consecutive costs which might indicate buggy counting.
|
||||
If the consecutive costs are identical, the file is affected by this bug:
|
||||
https://github.com/All-Hands-AI/OpenHands/issues/5383
|
||||
https://github.com/OpenHands/OpenHands/issues/5383
|
||||
|
||||
Args:
|
||||
row: DataFrame row containing instance data with metrics
|
||||
|
||||
@@ -34,8 +34,8 @@ We use it to train strong LM agents that achieve state-of-the-art open results o
|
||||
The process of running SWE-Gym is very similar to how you'd run SWE-Bench evaluation.
|
||||
|
||||
|
||||
1. First, clone OpenHands repo `git clone https://github.com/All-Hands-AI/OpenHands.git`
|
||||
2. Then setup the repo following [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md)
|
||||
1. First, clone OpenHands repo `git clone https://github.com/OpenHands/OpenHands.git`
|
||||
2. Then setup the repo following [Development.md](https://github.com/OpenHands/OpenHands/blob/main/Development.md)
|
||||
3. Then you can simply serve your own model as an OpenAI compatible endpoint, put those info in config.toml. You can do this by following instruction [here](../../README.md#setup).
|
||||
4. And then simply do the following to sample for 16x parallelism:
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = base_container_image
|
||||
|
||||
@@ -217,7 +217,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
@@ -259,6 +259,9 @@ def get_config(
|
||||
condenser=metadata.condenser_config,
|
||||
enable_prompt_extensions=False,
|
||||
model_routing=model_routing_config,
|
||||
system_prompt_filename=metadata.agent_config.system_prompt_filename
|
||||
if metadata.agent_config
|
||||
else 'system_prompt.j2',
|
||||
)
|
||||
config.set_agent_config(agent_config)
|
||||
|
||||
|
||||
@@ -180,7 +180,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
|
||||
@@ -9,7 +9,7 @@ def verify_instance_costs(row: pd.Series) -> float:
|
||||
"""Verifies that the accumulated_cost matches the sum of individual costs in metrics.
|
||||
Also checks for duplicate consecutive costs which might indicate buggy counting.
|
||||
If the consecutive costs are identical, the file is affected by this bug:
|
||||
https://github.com/All-Hands-AI/OpenHands/issues/5383
|
||||
https://github.com/OpenHands/OpenHands/issues/5383
|
||||
|
||||
Args:
|
||||
row: DataFrame row containing instance data with metrics
|
||||
|
||||
@@ -6,13 +6,13 @@ mkdir -p $EVAL_WORKSPACE
|
||||
|
||||
# 1. Prepare REPO
|
||||
echo "==== Prepare SWE-bench repo ===="
|
||||
OH_SWE_BENCH_REPO_PATH="https://github.com/All-Hands-AI/SWE-bench.git"
|
||||
OH_SWE_BENCH_REPO_PATH="https://github.com/OpenHands/SWE-bench.git"
|
||||
OH_SWE_BENCH_REPO_BRANCH="eval"
|
||||
git clone -b $OH_SWE_BENCH_REPO_BRANCH $OH_SWE_BENCH_REPO_PATH $EVAL_WORKSPACE/OH-SWE-bench
|
||||
|
||||
# 2. Prepare DATA
|
||||
echo "==== Prepare SWE-bench data ===="
|
||||
EVAL_IMAGE=ghcr.io/all-hands-ai/eval-swe-bench:builder_with_conda
|
||||
EVAL_IMAGE=ghcr.io/openhands/eval-swe-bench:builder_with_conda
|
||||
EVAL_WORKSPACE=$(realpath $EVAL_WORKSPACE)
|
||||
chmod +x $EVAL_WORKSPACE/OH-SWE-bench/swebench/harness/prepare_data.sh
|
||||
if [ -d $EVAL_WORKSPACE/eval_data ]; then
|
||||
|
||||
@@ -255,7 +255,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
|
||||
@@ -74,7 +74,7 @@ To contribute your evaluation results:
|
||||
## Additional Resources
|
||||
|
||||
- [TestGenEval Paper](https://arxiv.org/abs/2410.00752)
|
||||
- [OpenHands Documentation](https://github.com/All-Hands-AI/OpenHands)
|
||||
- [OpenHands Documentation](https://github.com/OpenHands/OpenHands)
|
||||
- [HuggingFace Datasets](https://huggingface.co/datasets)
|
||||
|
||||
For any questions or issues, please open an issue in the [OpenHands repository](https://github.com/All-Hands-AI/OpenHands/issues).
|
||||
For any questions or issues, please open an issue in the [OpenHands repository](https://github.com/OpenHands/OpenHands/issues).
|
||||
|
||||
@@ -124,7 +124,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = SandboxConfig(
|
||||
|
||||
@@ -6,13 +6,13 @@ mkdir -p $EVAL_WORKSPACE
|
||||
|
||||
# 1. Prepare REPO
|
||||
echo "==== Prepare SWE-bench repo ===="
|
||||
OH_SWE_BENCH_REPO_PATH="https://github.com/All-Hands-AI/SWE-bench.git"
|
||||
OH_SWE_BENCH_REPO_PATH="https://github.com/OpenHands/SWE-bench.git"
|
||||
OH_SWE_BENCH_REPO_BRANCH="eval"
|
||||
git clone -b $OH_SWE_BENCH_REPO_BRANCH $OH_SWE_BENCH_REPO_PATH $EVAL_WORKSPACE/OH-SWE-bench
|
||||
|
||||
# 2. Prepare DATA
|
||||
echo "==== Prepare SWE-bench data ===="
|
||||
EVAL_IMAGE=ghcr.io/all-hands-ai/eval-swe-bench:builder_with_conda
|
||||
EVAL_IMAGE=ghcr.io/openhands/eval-swe-bench:builder_with_conda
|
||||
EVAL_WORKSPACE=$(realpath $EVAL_WORKSPACE)
|
||||
chmod +x $EVAL_WORKSPACE/OH-SWE-bench/swebench/harness/prepare_data.sh
|
||||
if [ -d $EVAL_WORKSPACE/eval_data ]; then
|
||||
|
||||
@@ -162,7 +162,7 @@ while IFS= read -r task_image; do
|
||||
|
||||
# Prune unused images and volumes
|
||||
docker image rm "$task_image"
|
||||
docker images "ghcr.io/all-hands-ai/runtime" -q | xargs -r docker rmi -f
|
||||
docker images "ghcr.io/openhands/runtime" -q | xargs -r docker rmi -f
|
||||
docker volume prune -f
|
||||
docker system prune -f
|
||||
done < "$temp_file"
|
||||
|
||||
@@ -147,7 +147,7 @@ def get_config(
|
||||
logger.info(
|
||||
f'Using instance container image: {base_container_image}. '
|
||||
f'Please make sure this image exists. '
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
f'Submit an issue on https://github.com/OpenHands/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Integration tests
|
||||
|
||||
This directory implements integration tests that [was running in CI](https://github.com/All-Hands-AI/OpenHands/tree/23d3becf1d6f5d07e592f7345750c314a826b4e9/tests/integration).
|
||||
This directory implements integration tests that [was running in CI](https://github.com/OpenHands/OpenHands/tree/23d3becf1d6f5d07e592f7345750c314a826b4e9/tests/integration).
|
||||
|
||||
[PR 3985](https://github.com/All-Hands-AI/OpenHands/pull/3985) introduce LLM-based editing, which requires access to LLM to perform edit. Hence, we remove integration tests from CI and intend to run them as nightly evaluation to ensure the quality of OpenHands softwares.
|
||||
[PR 3985](https://github.com/OpenHands/OpenHands/pull/3985) introduce LLM-based editing, which requires access to LLM to perform edit. Hence, we remove integration tests from CI and intend to run them as nightly evaluation to ensure the quality of OpenHands softwares.
|
||||
|
||||
## To add new tests
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from openhands.runtime.base import Runtime
|
||||
|
||||
|
||||
class Test(BaseIntegrationTest):
|
||||
INSTRUCTION = 'Look at https://github.com/All-Hands-AI/OpenHands/pull/8, and tell me what is happening there and what did @asadm suggest.'
|
||||
INSTRUCTION = 'Look at https://github.com/OpenHands/OpenHands/pull/8, and tell me what is happening there and what did @asadm suggest.'
|
||||
|
||||
@classmethod
|
||||
def initialize_runtime(cls, runtime: Runtime) -> None:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user