mirror of
https://github.com/danielmiessler/Fabric.git
synced 2026-01-09 22:38:10 -05:00
Compare commits
89 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87df7dc383 | ||
|
|
1d69afa1c9 | ||
|
|
96c18b4c99 | ||
|
|
dd5173963b | ||
|
|
da1c8ec979 | ||
|
|
ac97f9984f | ||
|
|
181b812eaf | ||
|
|
fe94165d31 | ||
|
|
16e92690aa | ||
|
|
1c33799aa8 | ||
|
|
9559e618c3 | ||
|
|
ac32e8e64a | ||
|
|
82340e6126 | ||
|
|
5dec53726a | ||
|
|
b0eb136cbb | ||
|
|
63f4370ff1 | ||
|
|
b3cc2c737d | ||
|
|
e43b4191e4 | ||
|
|
744c565120 | ||
|
|
1473ac1465 | ||
|
|
c38c16f0db | ||
|
|
a4b1db4193 | ||
|
|
d44bc19a84 | ||
|
|
a2e618e11c | ||
|
|
cb90379b30 | ||
|
|
4868687746 | ||
|
|
85780fee76 | ||
|
|
497b1ed682 | ||
|
|
135433b749 | ||
|
|
f185dedb37 | ||
|
|
c74a157dcf | ||
|
|
91a336e870 | ||
|
|
5212fbcc37 | ||
|
|
6d8eb3d2b9 | ||
|
|
d3bba5d026 | ||
|
|
699762b694 | ||
|
|
f2a6f1bd98 | ||
|
|
3176adf59b | ||
|
|
7e29966622 | ||
|
|
0af0ab683d | ||
|
|
e72e67de71 | ||
|
|
414b6174e7 | ||
|
|
f63e0dfc05 | ||
|
|
4ef8578e47 | ||
|
|
12ee690ae4 | ||
|
|
cc378be485 | ||
|
|
06fc8d8732 | ||
|
|
9e4ed8ecb3 | ||
|
|
c369425708 | ||
|
|
cf074d3411 | ||
|
|
47f75237ff | ||
|
|
fad0a065d4 | ||
|
|
a59a3517d8 | ||
|
|
04c3c0c512 | ||
|
|
cb837bde2d | ||
|
|
2ad454b6dc | ||
|
|
c0ea25f816 | ||
|
|
87796d4fa9 | ||
|
|
e1945a0b62 | ||
|
|
ecac2b4c34 | ||
|
|
7ed4de269e | ||
|
|
6bd305906d | ||
|
|
6aeca6e4da | ||
|
|
b34f249e24 | ||
|
|
b187a80275 | ||
|
|
a6fc54a991 | ||
|
|
b9f4b9837a | ||
|
|
2bedf35957 | ||
|
|
b9df64a0d8 | ||
|
|
6b07b33ff2 | ||
|
|
ff245edd51 | ||
|
|
2e0a4da876 | ||
|
|
1f3befbbbc | ||
|
|
8988206fbe | ||
|
|
1bd5f9d7e4 | ||
|
|
832fd2f718 | ||
|
|
dd0935fb70 | ||
|
|
e64bdd849c | ||
|
|
be82b4b013 | ||
|
|
6e2f00090c | ||
|
|
7d6505fe98 | ||
|
|
23c1437794 | ||
|
|
dd5e57477f | ||
|
|
2c2b374664 | ||
|
|
b884c529bd | ||
|
|
137aff2268 | ||
|
|
42d3f45c57 | ||
|
|
f8ada0b148 | ||
|
|
cb5fa50f68 |
6
.github/workflows/patterns.yaml
vendored
6
.github/workflows/patterns.yaml
vendored
@@ -3,7 +3,7 @@ name: Patterns Artifact
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "patterns/**" # Trigger only on changes to files in the patterns folder
|
||||
- "data/patterns/**" # Trigger only on changes to files in the patterns folder
|
||||
|
||||
jobs:
|
||||
zip-and-upload:
|
||||
@@ -18,13 +18,13 @@ jobs:
|
||||
- name: Verify Changes in Patterns Folder
|
||||
run: |
|
||||
git fetch origin
|
||||
if git diff --quiet HEAD~1 -- patterns; then
|
||||
if git diff --quiet HEAD~1 -- data/patterns; then
|
||||
echo "No changes detected in patterns folder."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Zip the Patterns Folder
|
||||
run: zip -r patterns.zip patterns/
|
||||
run: zip -r patterns.zip data/patterns/
|
||||
|
||||
- name: Upload Patterns Artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -7,6 +7,10 @@ on:
|
||||
paths-ignore:
|
||||
- "data/patterns/**"
|
||||
- "**/*.md"
|
||||
- "data/strategies/**"
|
||||
- "cmd/generate_changelog/*.db"
|
||||
- "scripts/pattern_descriptions/*.json"
|
||||
- "web/static/data/pattern_descriptions.json"
|
||||
|
||||
permissions:
|
||||
contents: write # Ensure the workflow has write permissions
|
||||
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -131,9 +131,7 @@ celerybeat.pid
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
@@ -349,5 +347,6 @@ web/package-lock.json
|
||||
.gitignore_backup
|
||||
web/static/*.png
|
||||
|
||||
# Local VSCode project settings
|
||||
.vscode/
|
||||
# Local tmp directory
|
||||
.tmp/
|
||||
tmp/
|
||||
|
||||
3
.vscode/extensions.json
vendored
Normal file
3
.vscode/extensions.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"recommendations": ["davidanson.vscode-markdownlint"]
|
||||
}
|
||||
144
.vscode/settings.json
vendored
Normal file
144
.vscode/settings.json
vendored
Normal file
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"cSpell.words": [
|
||||
"addextension",
|
||||
"AIML",
|
||||
"anthropics",
|
||||
"badfile",
|
||||
"Behrens",
|
||||
"blindspots",
|
||||
"Bombal",
|
||||
"Cerebras",
|
||||
"compinit",
|
||||
"creatordate",
|
||||
"custompatterns",
|
||||
"danielmiessler",
|
||||
"davidanson",
|
||||
"Debugf",
|
||||
"dedup",
|
||||
"deepseek",
|
||||
"direnv",
|
||||
"dryrun",
|
||||
"dsrp",
|
||||
"editability",
|
||||
"Eisler",
|
||||
"elif",
|
||||
"envrc",
|
||||
"eugeis",
|
||||
"Eugen",
|
||||
"excalidraw",
|
||||
"exolab",
|
||||
"fabriclogo",
|
||||
"fpath",
|
||||
"frequencypenalty",
|
||||
"fsdb",
|
||||
"gantt",
|
||||
"genai",
|
||||
"githelper",
|
||||
"gjson",
|
||||
"GOARCH",
|
||||
"godotenv",
|
||||
"gofmt",
|
||||
"goimports",
|
||||
"gomod",
|
||||
"gonic",
|
||||
"goopenai",
|
||||
"GOPATH",
|
||||
"gopkg",
|
||||
"GOROOT",
|
||||
"Graphviz",
|
||||
"grokai",
|
||||
"Groq",
|
||||
"hackerone",
|
||||
"Haddix",
|
||||
"hasura",
|
||||
"hormozi",
|
||||
"Hormozi's",
|
||||
"HTMLURL",
|
||||
"jaredmontoya",
|
||||
"jessevdk",
|
||||
"Jina",
|
||||
"joho",
|
||||
"ksylvan",
|
||||
"Langdock",
|
||||
"ldflags",
|
||||
"libexec",
|
||||
"listcontexts",
|
||||
"listextensions",
|
||||
"listmodels",
|
||||
"listpatterns",
|
||||
"listsessions",
|
||||
"liststrategies",
|
||||
"listvendors",
|
||||
"lmstudio",
|
||||
"Makefiles",
|
||||
"markmap",
|
||||
"matplotlib",
|
||||
"mattn",
|
||||
"Miessler",
|
||||
"nometa",
|
||||
"numpy",
|
||||
"ollama",
|
||||
"opencode",
|
||||
"openrouter",
|
||||
"otiai",
|
||||
"pdflatex",
|
||||
"pipx",
|
||||
"PKCE",
|
||||
"pkgs",
|
||||
"presencepenalty",
|
||||
"printcontext",
|
||||
"printsession",
|
||||
"pycache",
|
||||
"pyperclip",
|
||||
"readystream",
|
||||
"restapi",
|
||||
"rmextension",
|
||||
"samber",
|
||||
"sashabaranov",
|
||||
"sdist",
|
||||
"seaborn",
|
||||
"semgrep",
|
||||
"sess",
|
||||
"storer",
|
||||
"Streamlit",
|
||||
"stretchr",
|
||||
"talkpanel",
|
||||
"Telos",
|
||||
"Thacker",
|
||||
"tidwall",
|
||||
"topp",
|
||||
"ttrc",
|
||||
"unalias",
|
||||
"unmarshalling",
|
||||
"updatepatterns",
|
||||
"videoid",
|
||||
"webp",
|
||||
"wipecontext",
|
||||
"wipesession",
|
||||
"writeups",
|
||||
"xclip",
|
||||
"yourpatternname"
|
||||
],
|
||||
"cSpell.ignorePaths": ["go.mod", ".gitignore", "CHANGELOG.md"],
|
||||
"markdownlint.config": {
|
||||
"MD004": false,
|
||||
"MD011": false,
|
||||
"MD024": false,
|
||||
"MD025": false,
|
||||
"M032": false,
|
||||
"MD033": {
|
||||
"allowed_elements": [
|
||||
"a",
|
||||
"br",
|
||||
"code",
|
||||
"div",
|
||||
"em",
|
||||
"h4",
|
||||
"img",
|
||||
"module",
|
||||
"p"
|
||||
]
|
||||
},
|
||||
"MD041": false
|
||||
}
|
||||
}
|
||||
2317
CHANGELOG.md
Normal file
2317
CHANGELOG.md
Normal file
File diff suppressed because it is too large
Load Diff
37
README.md
37
README.md
@@ -3,7 +3,7 @@ Fabric is graciously supported by…
|
||||
|
||||
[](https://warp.dev/fabric)
|
||||
|
||||
<img src="./images/fabric-logo-gif.gif" alt="fabriclogo" width="400" height="400"/>
|
||||
<img src="./docs/images/fabric-logo-gif.gif" alt="fabriclogo" width="400" height="400"/>
|
||||
|
||||
# `fabric`
|
||||
|
||||
@@ -29,7 +29,7 @@ Fabric is graciously supported by…
|
||||
[Helper Apps](#helper-apps) •
|
||||
[Meta](#meta)
|
||||
|
||||

|
||||

|
||||
|
||||
</div>
|
||||
|
||||
@@ -113,30 +113,9 @@ Keep in mind that many of these were recorded when Fabric was Python-based, so r
|
||||
|
||||
## Updates
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> July 4, 2025
|
||||
>
|
||||
> - **Web Search**: Fabric now supports web search for Anthropic and OpenAI models using the `--search` and `--search-location` flags. This replaces the previous plugin-based search, so you may want to remove the old `ANTHROPIC_WEB_SEARCH_TOOL_*` variables from your `~/.config/fabric/.env` file.
|
||||
> - **Image Generation**: Fabric now has powerful image generation capabilities with OpenAI.
|
||||
> - Generate images from text prompts and save them using `--image-file`.
|
||||
> - Edit existing images by providing an input image with `--attachment`.
|
||||
> - Control image `size`, `quality`, `compression`, and `background` with the new `--image-*` flags.
|
||||
>
|
||||
>June 17, 2025
|
||||
>
|
||||
>- Fabric now supports Perplexity AI. Configure it by using `fabric -S` to add your Perplexity AI API Key,
|
||||
> and then try:
|
||||
>
|
||||
> ```bash
|
||||
> fabric -m sonar-pro "What is the latest world news?"
|
||||
> ```
|
||||
>
|
||||
>June 11, 2025
|
||||
>
|
||||
>- Fabric's YouTube transcription now needs `yt-dlp` to be installed. Make sure to install the latest
|
||||
> version (2025.06.09 as of this note). The YouTube API key is only needed for comments (the `--comments` flag)
|
||||
> and metadata extraction (the `--metadata` flag).
|
||||
Fabric is evolving rapidly.
|
||||
|
||||
Stay current with the latest features by reviewing the [CHANGELOG](./CHANGELOG.md) for all recent changes.
|
||||
|
||||
## Philosophy
|
||||
|
||||
@@ -628,7 +607,7 @@ Now let's look at some things you can do with Fabric.
|
||||
<br />
|
||||
<br />
|
||||
|
||||
If you're not looking to do anything fancy, and you just want a lot of great prompts, you can navigate to the [`/patterns`](https://github.com/danielmiessler/fabric/tree/main/patterns) directory and start exploring!
|
||||
If you're not looking to do anything fancy, and you just want a lot of great prompts, you can navigate to the [`/patterns`](https://github.com/danielmiessler/fabric/tree/main/data/patterns) directory and start exploring!
|
||||
|
||||
We hope that if you used nothing else from Fabric, the Patterns by themselves will make the project useful.
|
||||
|
||||
@@ -644,7 +623,7 @@ be used in addition to the basic patterns.
|
||||
See the [Thinking Faster by Writing Less](https://arxiv.org/pdf/2502.18600) paper and
|
||||
the [Thought Generation section of Learn Prompting](https://learnprompting.org/docs/advanced/thought_generation/introduction) for examples of prompt strategies.
|
||||
|
||||
Each strategy is available as a small `json` file in the [`/strategies`](https://github.com/danielmiessler/fabric/tree/main/strategies) directory.
|
||||
Each strategy is available as a small `json` file in the [`/strategies`](https://github.com/danielmiessler/fabric/tree/main/data/strategies) directory.
|
||||
|
||||
The prompt modification of the strategy is applied to the system prompt and passed on to the
|
||||
LLM in the chat session.
|
||||
@@ -736,7 +715,7 @@ Make sure you have a LaTeX distribution (like TeX Live or MiKTeX) installed on y
|
||||
It generates a `json` representation of a directory of code that can be fed into an AI model
|
||||
with instructions to create a new feature or edit the code in a specified way.
|
||||
|
||||
See [the Create Coding Feature Pattern README](./patterns/create_coding_feature/README.md) for details.
|
||||
See [the Create Coding Feature Pattern README](./data/patterns/create_coding_feature/README.md) for details.
|
||||
|
||||
Install it first using:
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
package main
|
||||
|
||||
var version = "v1.4.241"
|
||||
var version = "v1.4.256"
|
||||
|
||||
151
cmd/generate_changelog/PRD.md
Normal file
151
cmd/generate_changelog/PRD.md
Normal file
@@ -0,0 +1,151 @@
|
||||
# Product Requirements Document: Changelog Generator
|
||||
|
||||
## Overview
|
||||
|
||||
The Changelog Generator is a high-performance Go tool that automatically generates comprehensive changelogs from git history and GitHub pull requests.
|
||||
|
||||
## Goals
|
||||
|
||||
1. **Performance**: Very fast. Efficient enough to be used in CI/CD as part of release process.
|
||||
2. **Completeness**: Capture ALL commits including unreleased changes
|
||||
3. **Efficiency**: Minimize API calls through caching and batch operations
|
||||
4. **Reliability**: Handle errors gracefully with proper Go error handling
|
||||
5. **Simplicity**: Single binary with no runtime dependencies
|
||||
|
||||
## Key Features
|
||||
|
||||
### 1. One-Pass Git History Algorithm
|
||||
|
||||
- Walk git history once from newest to oldest
|
||||
- Start with "Unreleased" bucket for all new commits
|
||||
- Switch buckets when encountering version commits
|
||||
- No need to calculate ranges between versions
|
||||
|
||||
### 2. Native Library Integration
|
||||
|
||||
- **go-git**: Pure Go git implementation (no git binary required)
|
||||
- **go-github**: Official GitHub Go client library
|
||||
- Benefits: Type safety, better error handling, no subprocess overhead
|
||||
|
||||
### 3. Smart Caching System
|
||||
|
||||
- SQLite-based persistent cache
|
||||
- Stores: versions, commits, PR details, last processed commit
|
||||
- Enables incremental updates on subsequent runs
|
||||
- Instant changelog regeneration from cache
|
||||
|
||||
### 4. Concurrent Processing
|
||||
|
||||
- Parallel GitHub API calls (up to 10 concurrent)
|
||||
- Batch PR fetching with deduplication
|
||||
- Rate limiting awareness
|
||||
|
||||
### 5. Enhanced Output
|
||||
|
||||
- "Unreleased" section for commits since last version
|
||||
- Clean markdown formatting
|
||||
- Configurable version limiting
|
||||
- Direct commit tracking (non-PR commits)
|
||||
|
||||
## Technical Architecture
|
||||
|
||||
### Module Structure
|
||||
|
||||
```text
|
||||
cmd/generate_changelog/
|
||||
├── main.go # CLI entry point with cobra
|
||||
├── internal/
|
||||
│ ├── git/ # Git operations (go-git)
|
||||
│ ├── github/ # GitHub API client (go-github)
|
||||
│ ├── cache/ # SQLite caching layer
|
||||
│ ├── changelog/ # Core generation logic
|
||||
│ └── config/ # Configuration management
|
||||
└── changelog.db # SQLite cache (generated)
|
||||
```
|
||||
|
||||
### Data Flow
|
||||
|
||||
1. Git walker collects all commits in one pass
|
||||
2. Commits bucketed by version (starting with "Unreleased")
|
||||
3. PR numbers extracted from merge commits
|
||||
4. GitHub API batch-fetches PR details
|
||||
5. Cache stores everything for future runs
|
||||
6. Formatter generates markdown output
|
||||
|
||||
### Cache Schema
|
||||
|
||||
- **metadata**: Last processed commit SHA
|
||||
- **versions**: Version names, dates, commit SHAs
|
||||
- **commits**: Full commit details with version associations
|
||||
- **pull_requests**: PR details including commits
|
||||
- Indexes on version and PR number for fast lookups
|
||||
|
||||
### Features
|
||||
|
||||
- **Unreleased section**: Shows all new commits
|
||||
- **Better caching**: SQLite vs JSON, incremental updates
|
||||
- **Smarter deduplication**: Removes consecutive duplicate commits
|
||||
- **Direct commit tracking**: Shows non-PR commits
|
||||
|
||||
### Reliability
|
||||
|
||||
- **No subprocess errors**: Direct library usage
|
||||
- **Type safety**: Compile-time checking
|
||||
- **Better error handling**: Go's explicit error returns
|
||||
|
||||
### Deployment
|
||||
|
||||
- **Single binary**: No Python/pip/dependencies
|
||||
- **Cross-platform**: Compile for any OS/architecture
|
||||
- **No git CLI required**: Uses go-git library
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
- `GITHUB_TOKEN`: GitHub API authentication token
|
||||
|
||||
### Command Line Flags
|
||||
|
||||
- `--repo, -r`: Repository path (default: current directory)
|
||||
- `--output, -o`: Output file (default: stdout)
|
||||
- `--limit, -l`: Version limit (default: all)
|
||||
- `--version, -v`: Target specific version
|
||||
- `--save-data`: Export debug JSON
|
||||
- `--cache`: Cache file location
|
||||
- `--no-cache`: Disable caching
|
||||
- `--rebuild-cache`: Force cache rebuild
|
||||
- `--token`: GitHub token override
|
||||
|
||||
## Success Metrics
|
||||
|
||||
1. **Performance**: Generate full changelog in <5 seconds for fabric repo
|
||||
2. **Completeness**: 100% commit coverage including unreleased
|
||||
3. **Accuracy**: Correct PR associations and change extraction
|
||||
4. **Reliability**: Handle network failures gracefully
|
||||
5. **Usability**: Simple CLI with sensible defaults
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
1. **Multiple output formats**: JSON, HTML, etc.
|
||||
2. **Custom version patterns**: Configurable regex
|
||||
3. **Change categorization**: feat/fix/docs auto-grouping
|
||||
4. **Conventional commits**: Full support for semantic versioning
|
||||
5. **GitLab/Bitbucket**: Support other platforms
|
||||
6. **Web UI**: Interactive changelog browser
|
||||
7. **Incremental updates**: Update existing CHANGELOG.md file
|
||||
8. **Breaking change detection**: Highlight breaking changes
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- ✅ Core architecture and modules
|
||||
- ✅ One-pass git walking algorithm
|
||||
- ✅ GitHub API integration with concurrency
|
||||
- ✅ SQLite caching system
|
||||
- ✅ Changelog formatting and generation
|
||||
- ✅ CLI with all planned flags
|
||||
- ✅ Documentation (README and PRD)
|
||||
|
||||
## Conclusion
|
||||
|
||||
This Go implementation provides a modern, efficient, and feature-rich changelog generator.
|
||||
263
cmd/generate_changelog/README.md
Normal file
263
cmd/generate_changelog/README.md
Normal file
@@ -0,0 +1,263 @@
|
||||
# Changelog Generator
|
||||
|
||||
A high-performance changelog generator for Git repositories that automatically creates comprehensive, well-formatted changelogs from your git history and GitHub pull requests.
|
||||
|
||||
## Features
|
||||
|
||||
- **One-pass git history walking**: Efficiently processes entire repository history in a single pass
|
||||
- **Automatic PR detection**: Extracts pull request information from merge commits
|
||||
- **GitHub API integration**: Fetches detailed PR information including commits, authors, and descriptions
|
||||
- **Smart caching**: SQLite-based caching for instant incremental updates
|
||||
- **Unreleased changes**: Tracks all commits since the last release
|
||||
- **Concurrent processing**: Parallel GitHub API calls for improved performance
|
||||
- **Flexible output**: Generate complete changelogs or target specific versions
|
||||
- **GraphQL optimization**: Ultra-fast PR fetching using GitHub GraphQL API (~5-10 calls vs 1000s)
|
||||
- **Intelligent sync**: Automatically syncs new PRs every 24 hours or when missing PRs are detected
|
||||
- **AI-powered summaries**: Optional Fabric integration for enhanced changelog summaries
|
||||
- **Advanced caching**: Content-based change detection for AI summaries with hash comparison
|
||||
- **Author type detection**: Distinguishes between users, bots, and organizations
|
||||
- **Lightning-fast incremental updates**: SHA→PR mapping for instant git operations
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
go install github.com/danielmiessler/fabric/cmd/generate_changelog@latest
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic usage (generate complete changelog)
|
||||
|
||||
```bash
|
||||
generate_changelog
|
||||
```
|
||||
|
||||
### Save to file
|
||||
|
||||
```bash
|
||||
generate_changelog -o CHANGELOG.md
|
||||
```
|
||||
|
||||
### Generate for specific version
|
||||
|
||||
```bash
|
||||
generate_changelog -v v1.4.244
|
||||
```
|
||||
|
||||
### Limit to recent versions
|
||||
|
||||
```bash
|
||||
generate_changelog -l 10
|
||||
```
|
||||
|
||||
### Using GitHub token for private repos or higher rate limits
|
||||
|
||||
```bash
|
||||
export GITHUB_TOKEN=your_token_here
|
||||
generate_changelog
|
||||
|
||||
# Or pass directly
|
||||
generate_changelog --token your_token_here
|
||||
```
|
||||
|
||||
### AI-enhanced summaries
|
||||
|
||||
```bash
|
||||
# Enable AI summaries using Fabric
|
||||
generate_changelog --ai-summarize
|
||||
|
||||
# Use custom model for AI summaries
|
||||
FABRIC_CHANGELOG_SUMMARIZE_MODEL=claude-opus-4 generate_changelog --ai-summarize
|
||||
```
|
||||
|
||||
### Cache management
|
||||
|
||||
```bash
|
||||
# Rebuild cache from scratch
|
||||
generate_changelog --rebuild-cache
|
||||
|
||||
# Force a full PR sync from GitHub
|
||||
generate_changelog --force-pr-sync
|
||||
|
||||
# Disable cache usage
|
||||
generate_changelog --no-cache
|
||||
|
||||
# Use custom cache location
|
||||
generate_changelog --cache /path/to/cache.db
|
||||
```
|
||||
|
||||
## Command Line Options
|
||||
|
||||
| Flag | Short | Description | Default |
|
||||
|------|-------|-------------|---------|
|
||||
| `--repo` | `-r` | Repository path | `.` (current directory) |
|
||||
| `--output` | `-o` | Output file | stdout |
|
||||
| `--limit` | `-l` | Limit number of versions | 0 (all) |
|
||||
| `--version` | `-v` | Generate for specific version | |
|
||||
| `--save-data` | | Save version data to JSON | false |
|
||||
| `--cache` | | Cache database file | `./cmd/generate_changelog/changelog.db` |
|
||||
| `--no-cache` | | Disable cache usage | false |
|
||||
| `--rebuild-cache` | | Rebuild cache from scratch | false |
|
||||
| `--force-pr-sync` | | Force a full PR sync from GitHub | false |
|
||||
| `--token` | | GitHub API token | `$GITHUB_TOKEN` |
|
||||
| `--ai-summarize` | | Generate AI-enhanced summaries using Fabric | false |
|
||||
|
||||
## Output Format
|
||||
|
||||
The generated changelog follows this structure:
|
||||
|
||||
```markdown
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
### PR [#1601](url) by [author](profile): PR Title
|
||||
- Change description 1
|
||||
- Change description 2
|
||||
|
||||
### Direct commits
|
||||
- Direct commit message 1
|
||||
- Direct commit message 2
|
||||
|
||||
## v1.4.244 (2025-07-09)
|
||||
|
||||
### PR [#1598](url) by [author](profile): PR Title
|
||||
- Change description
|
||||
...
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Git History Walking**: The tool walks through your git history from newest to oldest commits
|
||||
2. **Version Detection**: Identifies version bump commits (pattern: "Update version to vX.Y.Z")
|
||||
3. **PR Extraction**: Detects merge commits and extracts PR numbers
|
||||
4. **GitHub API Calls**: Fetches detailed PR information in parallel batches
|
||||
5. **Change Extraction**: Extracts changes from PR commit messages or PR body
|
||||
6. **Formatting**: Generates clean, organized markdown output
|
||||
|
||||
## Performance
|
||||
|
||||
- **Native Go libraries**: Uses go-git and go-github for maximum performance
|
||||
- **Concurrent API calls**: Processes up to 10 GitHub API requests in parallel
|
||||
- **Smart caching**: SQLite cache eliminates redundant API calls
|
||||
- **Incremental updates**: Only processes new commits on subsequent runs
|
||||
- **GraphQL optimization**: Uses GitHub GraphQL API to fetch all PR data in ~5-10 calls
|
||||
- **AI-powered summaries**: Optional Fabric integration with intelligent caching
|
||||
- **Content-based change detection**: AI summaries only regenerated when content changes
|
||||
- **Lightning-fast git operations**: SHA→PR mapping stored in database for instant lookups
|
||||
|
||||
### Major Optimization: GraphQL + Advanced Caching
|
||||
|
||||
The tool has been optimized to drastically reduce GitHub API calls and improve performance:
|
||||
|
||||
**Previous approach**: Individual API calls for each PR (2 API calls per PR)
|
||||
|
||||
- For a repo with 500 PRs: 1,000 API calls
|
||||
|
||||
**Current approach**: GraphQL batch fetching with intelligent caching
|
||||
|
||||
- For a repo with 500 PRs: ~5-10 GraphQL calls (initial fetch) + 0 calls (subsequent runs with cache)
|
||||
- **99%+ reduction in API calls after initial run!**
|
||||
|
||||
The optimization includes:
|
||||
|
||||
1. **GraphQL Batch Fetch**: Uses GitHub's GraphQL API to fetch all merged PRs with commits in minimal calls
|
||||
2. **Smart Caching**: Stores complete PR data, commits, and SHA mappings in SQLite
|
||||
3. **Incremental Sync**: Only fetches PRs merged after the last sync timestamp
|
||||
4. **Automatic Refresh**: PRs are synced every 24 hours or when missing PRs are detected
|
||||
5. **AI Summary Caching**: Content-based change detection prevents unnecessary AI regeneration
|
||||
6. **Fallback Support**: If GraphQL fails, falls back to REST API batch fetching
|
||||
7. **Lightning Git Operations**: Pre-computed SHA→PR mappings for instant commit association
|
||||
|
||||
## Requirements
|
||||
|
||||
- Go 1.24+ (for installation from source)
|
||||
- Git repository
|
||||
- GitHub token (optional, for private repos or higher rate limits)
|
||||
- Fabric CLI (optional, for AI-enhanced summaries)
|
||||
|
||||
## Authentication
|
||||
|
||||
The tool supports GitHub authentication via:
|
||||
|
||||
1. Environment variable: `export GITHUB_TOKEN=your_token`
|
||||
2. Command line flag: `--token your_token`
|
||||
3. `.env` file in the same directory as the binary
|
||||
|
||||
### Environment File Support
|
||||
|
||||
Create a `.env` file next to the `generate_changelog` binary:
|
||||
|
||||
```bash
|
||||
GITHUB_TOKEN=your_github_token_here
|
||||
FABRIC_CHANGELOG_SUMMARIZE_MODEL=claude-sonnet-4-20250514
|
||||
```
|
||||
|
||||
The tool automatically loads `.env` files for convenient configuration management.
|
||||
|
||||
Without authentication, the tool is limited to 60 GitHub API requests per hour.
|
||||
|
||||
## Caching
|
||||
|
||||
The SQLite cache stores:
|
||||
|
||||
- Version information and commit associations
|
||||
- Pull request details (title, body, commits, authors)
|
||||
- Last processed commit SHA for incremental updates
|
||||
- Last PR sync timestamp for intelligent refresh
|
||||
- AI summaries with content-based change detection
|
||||
- SHA→PR mappings for lightning-fast git operations
|
||||
|
||||
Cache benefits:
|
||||
|
||||
- Instant changelog regeneration
|
||||
- Drastically reduced GitHub API usage (99%+ reduction after initial run)
|
||||
- Offline changelog generation (after initial cache build)
|
||||
- Automatic PR data refresh every 24 hours
|
||||
- Batch database transactions for better performance
|
||||
- Content-aware AI summary regeneration
|
||||
|
||||
## AI-Enhanced Summaries
|
||||
|
||||
The tool can generate AI-powered summaries using Fabric for more polished, professional changelogs:
|
||||
|
||||
```bash
|
||||
# Enable AI summarization
|
||||
generate_changelog --ai-summarize
|
||||
|
||||
# Custom model (default: claude-sonnet-4-20250514)
|
||||
FABRIC_CHANGELOG_SUMMARIZE_MODEL=claude-opus-4 generate_changelog --ai-summarize
|
||||
```
|
||||
|
||||
### AI Summary Features
|
||||
|
||||
- **Content-based change detection**: AI summaries are only regenerated when version content changes
|
||||
- **Intelligent caching**: Preserves existing summaries and only processes changed versions
|
||||
- **Content hash comparison**: Uses SHA256 hashing to detect when "Unreleased" content changes
|
||||
- **Automatic fallback**: Falls back to raw content if AI processing fails
|
||||
- **Error detection**: Identifies and handles AI processing errors gracefully
|
||||
- **Minimum content filtering**: Skips AI processing for very brief content (< 256 characters)
|
||||
|
||||
### AI Model Configuration
|
||||
|
||||
Set the model via environment variable:
|
||||
|
||||
```bash
|
||||
export FABRIC_CHANGELOG_SUMMARIZE_MODEL=claude-opus-4
|
||||
# or
|
||||
export FABRIC_CHANGELOG_SUMMARIZE_MODEL=gpt-4
|
||||
```
|
||||
|
||||
AI summaries are cached and only regenerated when:
|
||||
|
||||
- Version content changes (detected via hash comparison)
|
||||
- No existing AI summary exists for the version
|
||||
- Force rebuild is requested
|
||||
|
||||
## Contributing
|
||||
|
||||
This tool is part of the Fabric project. Contributions are welcome!
|
||||
|
||||
## License
|
||||
|
||||
The MIT License. Same as the Fabric project.
|
||||
BIN
cmd/generate_changelog/changelog.db
Normal file
BIN
cmd/generate_changelog/changelog.db
Normal file
Binary file not shown.
448
cmd/generate_changelog/internal/cache/cache.go
vendored
Normal file
448
cmd/generate_changelog/internal/cache/cache.go
vendored
Normal file
@@ -0,0 +1,448 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/git"
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/github"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
type Cache struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func New(dbPath string) (*Cache, error) {
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open database: %w", err)
|
||||
}
|
||||
|
||||
cache := &Cache{db: db}
|
||||
if err := cache.createTables(); err != nil {
|
||||
return nil, fmt.Errorf("failed to create tables: %w", err)
|
||||
}
|
||||
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func (c *Cache) Close() error {
|
||||
return c.db.Close()
|
||||
}
|
||||
|
||||
func (c *Cache) createTables() error {
|
||||
queries := []string{
|
||||
`CREATE TABLE IF NOT EXISTS metadata (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS versions (
|
||||
name TEXT PRIMARY KEY,
|
||||
date DATETIME,
|
||||
commit_sha TEXT,
|
||||
pr_numbers TEXT,
|
||||
ai_summary TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS commits (
|
||||
sha TEXT PRIMARY KEY,
|
||||
version TEXT NOT NULL,
|
||||
message TEXT,
|
||||
author TEXT,
|
||||
email TEXT,
|
||||
date DATETIME,
|
||||
is_merge BOOLEAN,
|
||||
pr_number INTEGER,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (version) REFERENCES versions(name)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS pull_requests (
|
||||
number INTEGER PRIMARY KEY,
|
||||
title TEXT,
|
||||
body TEXT,
|
||||
author TEXT,
|
||||
author_url TEXT,
|
||||
author_type TEXT DEFAULT 'user',
|
||||
url TEXT,
|
||||
merged_at DATETIME,
|
||||
merge_commit TEXT,
|
||||
commits TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_commits_version ON commits(version)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_commits_pr_number ON commits(pr_number)`,
|
||||
`CREATE TABLE IF NOT EXISTS commit_pr_mapping (
|
||||
commit_sha TEXT PRIMARY KEY,
|
||||
pr_number INTEGER NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (pr_number) REFERENCES pull_requests(number)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_commit_pr_mapping_sha ON commit_pr_mapping(commit_sha)`,
|
||||
}
|
||||
|
||||
for _, query := range queries {
|
||||
if _, err := c.db.Exec(query); err != nil {
|
||||
return fmt.Errorf("failed to execute query: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cache) GetLastProcessedTag() (string, error) {
|
||||
var tag string
|
||||
err := c.db.QueryRow("SELECT value FROM metadata WHERE key = 'last_processed_tag'").Scan(&tag)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", nil
|
||||
}
|
||||
return tag, err
|
||||
}
|
||||
|
||||
func (c *Cache) SetLastProcessedTag(tag string) error {
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO metadata (key, value, updated_at)
|
||||
VALUES ('last_processed_tag', ?, CURRENT_TIMESTAMP)
|
||||
`, tag)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) SaveVersion(v *git.Version) error {
|
||||
prNumbers, _ := json.Marshal(v.PRNumbers)
|
||||
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO versions (name, date, commit_sha, pr_numbers, ai_summary)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, v.Name, v.Date, v.CommitSHA, string(prNumbers), v.AISummary)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateVersionAISummary updates only the AI summary for a specific version
|
||||
func (c *Cache) UpdateVersionAISummary(versionName, aiSummary string) error {
|
||||
_, err := c.db.Exec(`
|
||||
UPDATE versions SET ai_summary = ? WHERE name = ?
|
||||
`, aiSummary, versionName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) SaveCommit(commit *git.Commit, version string) error {
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO commits
|
||||
(sha, version, message, author, email, date, is_merge, pr_number)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, commit.SHA, version, commit.Message, commit.Author, commit.Email,
|
||||
commit.Date, commit.IsMerge, commit.PRNumber)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) SavePR(pr *github.PR) error {
|
||||
commits, _ := json.Marshal(pr.Commits)
|
||||
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO pull_requests
|
||||
(number, title, body, author, author_url, author_type, url, merged_at, merge_commit, commits)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, pr.Number, pr.Title, pr.Body, pr.Author, pr.AuthorURL, pr.AuthorType,
|
||||
pr.URL, pr.MergedAt, pr.MergeCommit, string(commits))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) GetPR(number int) (*github.PR, error) {
|
||||
var pr github.PR
|
||||
var commitsJSON string
|
||||
|
||||
err := c.db.QueryRow(`
|
||||
SELECT number, title, body, author, author_url, COALESCE(author_type, 'user'), url, merged_at, merge_commit, commits
|
||||
FROM pull_requests WHERE number = ?
|
||||
`, number).Scan(
|
||||
&pr.Number, &pr.Title, &pr.Body, &pr.Author, &pr.AuthorURL, &pr.AuthorType,
|
||||
&pr.URL, &pr.MergedAt, &pr.MergeCommit, &commitsJSON,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(commitsJSON), &pr.Commits); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal commits: %w", err)
|
||||
}
|
||||
|
||||
return &pr, nil
|
||||
}
|
||||
|
||||
func (c *Cache) GetVersions() (map[string]*git.Version, error) {
|
||||
rows, err := c.db.Query(`
|
||||
SELECT name, date, commit_sha, pr_numbers, ai_summary FROM versions
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
versions := make(map[string]*git.Version)
|
||||
|
||||
for rows.Next() {
|
||||
var v git.Version
|
||||
var dateStr sql.NullString
|
||||
var prNumbersJSON string
|
||||
var aiSummary sql.NullString
|
||||
|
||||
if err := rows.Scan(&v.Name, &dateStr, &v.CommitSHA, &prNumbersJSON, &aiSummary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if dateStr.Valid {
|
||||
v.Date, _ = time.Parse(time.RFC3339, dateStr.String)
|
||||
}
|
||||
|
||||
if prNumbersJSON != "" {
|
||||
json.Unmarshal([]byte(prNumbersJSON), &v.PRNumbers)
|
||||
}
|
||||
|
||||
if aiSummary.Valid {
|
||||
v.AISummary = aiSummary.String
|
||||
}
|
||||
|
||||
v.Commits, err = c.getCommitsForVersion(v.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
versions[v.Name] = &v
|
||||
}
|
||||
|
||||
return versions, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Cache) getCommitsForVersion(version string) ([]*git.Commit, error) {
|
||||
rows, err := c.db.Query(`
|
||||
SELECT sha, message, author, email, date, is_merge, pr_number
|
||||
FROM commits WHERE version = ?
|
||||
ORDER BY date DESC
|
||||
`, version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var commits []*git.Commit
|
||||
|
||||
for rows.Next() {
|
||||
var commit git.Commit
|
||||
if err := rows.Scan(
|
||||
&commit.SHA, &commit.Message, &commit.Author, &commit.Email,
|
||||
&commit.Date, &commit.IsMerge, &commit.PRNumber,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
commits = append(commits, &commit)
|
||||
}
|
||||
|
||||
return commits, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Cache) Clear() error {
|
||||
tables := []string{"metadata", "versions", "commits", "pull_requests"}
|
||||
for _, table := range tables {
|
||||
if _, err := c.db.Exec("DELETE FROM " + table); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetLastPRSync returns the timestamp of the last PR sync
|
||||
func (c *Cache) GetLastPRSync() (time.Time, error) {
|
||||
var timestamp string
|
||||
err := c.db.QueryRow("SELECT value FROM metadata WHERE key = 'last_pr_sync'").Scan(×tamp)
|
||||
if err == sql.ErrNoRows {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
return time.Parse(time.RFC3339, timestamp)
|
||||
}
|
||||
|
||||
// SetLastPRSync updates the timestamp of the last PR sync
|
||||
func (c *Cache) SetLastPRSync(timestamp time.Time) error {
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO metadata (key, value, updated_at)
|
||||
VALUES ('last_pr_sync', ?, CURRENT_TIMESTAMP)
|
||||
`, timestamp.Format(time.RFC3339))
|
||||
return err
|
||||
}
|
||||
|
||||
// SavePRBatch saves multiple PRs in a single transaction for better performance
|
||||
func (c *Cache) SavePRBatch(prs []*github.PR) error {
|
||||
tx, err := c.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
INSERT OR REPLACE INTO pull_requests
|
||||
(number, title, body, author, author_url, author_type, url, merged_at, merge_commit, commits)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare statement: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, pr := range prs {
|
||||
commits, _ := json.Marshal(pr.Commits)
|
||||
_, err := stmt.Exec(
|
||||
pr.Number, pr.Title, pr.Body, pr.Author, pr.AuthorURL, pr.AuthorType,
|
||||
pr.URL, pr.MergedAt, pr.MergeCommit, string(commits),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save PR #%d: %w", pr.Number, err)
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetAllPRs returns all cached PRs
|
||||
func (c *Cache) GetAllPRs() (map[int]*github.PR, error) {
|
||||
rows, err := c.db.Query(`
|
||||
SELECT number, title, body, author, author_url, COALESCE(author_type, 'user'), url, merged_at, merge_commit, commits
|
||||
FROM pull_requests
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
prs := make(map[int]*github.PR)
|
||||
|
||||
for rows.Next() {
|
||||
var pr github.PR
|
||||
var commitsJSON string
|
||||
|
||||
if err := rows.Scan(
|
||||
&pr.Number, &pr.Title, &pr.Body, &pr.Author, &pr.AuthorURL, &pr.AuthorType,
|
||||
&pr.URL, &pr.MergedAt, &pr.MergeCommit, &commitsJSON,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(commitsJSON), &pr.Commits); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal commits for PR #%d: %w", pr.Number, err)
|
||||
}
|
||||
|
||||
prs[pr.Number] = &pr
|
||||
}
|
||||
|
||||
return prs, rows.Err()
|
||||
}
|
||||
|
||||
// MarkPRAsNonExistent marks a PR number as non-existent to avoid future fetches
|
||||
func (c *Cache) MarkPRAsNonExistent(prNumber int) error {
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO metadata (key, value, updated_at)
|
||||
VALUES (?, 'non_existent', CURRENT_TIMESTAMP)
|
||||
`, fmt.Sprintf("pr_non_existent_%d", prNumber))
|
||||
return err
|
||||
}
|
||||
|
||||
// IsPRMarkedAsNonExistent checks if a PR is marked as non-existent
|
||||
func (c *Cache) IsPRMarkedAsNonExistent(prNumber int) bool {
|
||||
var value string
|
||||
err := c.db.QueryRow("SELECT value FROM metadata WHERE key = ?",
|
||||
fmt.Sprintf("pr_non_existent_%d", prNumber)).Scan(&value)
|
||||
return err == nil && value == "non_existent"
|
||||
}
|
||||
|
||||
// SaveCommitPRMappings saves SHA→PR mappings for all commits in PRs
|
||||
func (c *Cache) SaveCommitPRMappings(prs []*github.PR) error {
|
||||
tx, err := c.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
INSERT OR REPLACE INTO commit_pr_mapping (commit_sha, pr_number)
|
||||
VALUES (?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare statement: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, pr := range prs {
|
||||
for _, commit := range pr.Commits {
|
||||
_, err := stmt.Exec(commit.SHA, pr.Number)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save commit mapping %s→%d: %w", commit.SHA, pr.Number, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetPRNumberBySHA returns the PR number for a given commit SHA
|
||||
func (c *Cache) GetPRNumberBySHA(sha string) (int, bool) {
|
||||
var prNumber int
|
||||
err := c.db.QueryRow("SELECT pr_number FROM commit_pr_mapping WHERE commit_sha = ?", sha).Scan(&prNumber)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, false
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return prNumber, true
|
||||
}
|
||||
|
||||
// GetCommitSHAsForPR returns all commit SHAs for a given PR number
|
||||
func (c *Cache) GetCommitSHAsForPR(prNumber int) ([]string, error) {
|
||||
rows, err := c.db.Query("SELECT commit_sha FROM commit_pr_mapping WHERE pr_number = ?", prNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var shas []string
|
||||
for rows.Next() {
|
||||
var sha string
|
||||
if err := rows.Scan(&sha); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shas = append(shas, sha)
|
||||
}
|
||||
|
||||
return shas, rows.Err()
|
||||
}
|
||||
|
||||
// GetUnreleasedContentHash returns the cached content hash for Unreleased
|
||||
func (c *Cache) GetUnreleasedContentHash() (string, error) {
|
||||
var hash string
|
||||
err := c.db.QueryRow("SELECT value FROM metadata WHERE key = 'unreleased_content_hash'").Scan(&hash)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", fmt.Errorf("no content hash found")
|
||||
}
|
||||
return hash, err
|
||||
}
|
||||
|
||||
// SetUnreleasedContentHash stores the content hash for Unreleased
|
||||
func (c *Cache) SetUnreleasedContentHash(hash string) error {
|
||||
_, err := c.db.Exec(`
|
||||
INSERT OR REPLACE INTO metadata (key, value, updated_at)
|
||||
VALUES ('unreleased_content_hash', ?, CURRENT_TIMESTAMP)
|
||||
`, hash)
|
||||
return err
|
||||
}
|
||||
699
cmd/generate_changelog/internal/changelog/generator.go
Normal file
699
cmd/generate_changelog/internal/changelog/generator.go
Normal file
@@ -0,0 +1,699 @@
|
||||
package changelog
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/cache"
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/config"
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/git"
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/github"
|
||||
)
|
||||
|
||||
type Generator struct {
|
||||
cfg *config.Config
|
||||
gitWalker *git.Walker
|
||||
ghClient *github.Client
|
||||
cache *cache.Cache
|
||||
versions map[string]*git.Version
|
||||
prs map[int]*github.PR
|
||||
}
|
||||
|
||||
func New(cfg *config.Config) (*Generator, error) {
|
||||
gitWalker, err := git.NewWalker(cfg.RepoPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create git walker: %w", err)
|
||||
}
|
||||
|
||||
owner, repo, err := gitWalker.GetRepoInfo()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get repo info: %w", err)
|
||||
}
|
||||
|
||||
ghClient := github.NewClient(cfg.GitHubToken, owner, repo)
|
||||
|
||||
var c *cache.Cache
|
||||
if !cfg.NoCache {
|
||||
c, err = cache.New(cfg.CacheFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create cache: %w", err)
|
||||
}
|
||||
|
||||
if cfg.RebuildCache {
|
||||
if err := c.Clear(); err != nil {
|
||||
return nil, fmt.Errorf("failed to clear cache: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &Generator{
|
||||
cfg: cfg,
|
||||
gitWalker: gitWalker,
|
||||
ghClient: ghClient,
|
||||
cache: c,
|
||||
prs: make(map[int]*github.PR),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *Generator) Generate() (string, error) {
|
||||
if err := g.collectData(); err != nil {
|
||||
return "", fmt.Errorf("failed to collect data: %w", err)
|
||||
}
|
||||
|
||||
if err := g.fetchPRs(); err != nil {
|
||||
return "", fmt.Errorf("failed to fetch PRs: %w", err)
|
||||
}
|
||||
|
||||
return g.formatChangelog(), nil
|
||||
}
|
||||
|
||||
func (g *Generator) collectData() error {
|
||||
if g.cache != nil && !g.cfg.RebuildCache {
|
||||
cachedTag, err := g.cache.GetLastProcessedTag()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get last processed tag: %w", err)
|
||||
}
|
||||
|
||||
if cachedTag != "" {
|
||||
// Get the current latest tag from git
|
||||
currentTag, err := g.gitWalker.GetLatestTag()
|
||||
if err == nil {
|
||||
// Load cached data - we can use it even if there are new tags
|
||||
cachedVersions, err := g.cache.GetVersions()
|
||||
if err == nil && len(cachedVersions) > 0 {
|
||||
g.versions = cachedVersions
|
||||
|
||||
// Load cached PRs
|
||||
for _, version := range g.versions {
|
||||
for _, prNum := range version.PRNumbers {
|
||||
if pr, err := g.cache.GetPR(prNum); err == nil && pr != nil {
|
||||
g.prs[prNum] = pr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have new tags since cache, process the new versions only
|
||||
if currentTag != cachedTag {
|
||||
fmt.Fprintf(os.Stderr, "Processing new versions since %s...\n", cachedTag)
|
||||
newVersions, err := g.gitWalker.WalkHistorySinceTag(cachedTag)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to walk history since tag %s: %v\n", cachedTag, err)
|
||||
} else {
|
||||
// Merge new versions into cached versions (only add if not already cached)
|
||||
for name, version := range newVersions {
|
||||
if name != "Unreleased" { // Handle Unreleased separately
|
||||
if existingVersion, exists := g.versions[name]; !exists {
|
||||
g.versions[name] = version
|
||||
} else {
|
||||
// Update existing version with new PR numbers if they're missing
|
||||
if len(existingVersion.PRNumbers) == 0 && len(version.PRNumbers) > 0 {
|
||||
existingVersion.PRNumbers = version.PRNumbers
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Always update Unreleased section with latest commits
|
||||
unreleasedVersion, err := g.gitWalker.WalkCommitsSinceTag(currentTag)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to walk commits since tag %s: %v\n", currentTag, err)
|
||||
} else if unreleasedVersion != nil {
|
||||
// Preserve existing AI summary if available
|
||||
if existingUnreleased, exists := g.versions["Unreleased"]; exists {
|
||||
unreleasedVersion.AISummary = existingUnreleased.AISummary
|
||||
}
|
||||
// Replace or add the unreleased version
|
||||
g.versions["Unreleased"] = unreleasedVersion
|
||||
}
|
||||
|
||||
// Save any new versions to cache (after potential AI processing)
|
||||
if currentTag != cachedTag {
|
||||
for _, version := range g.versions {
|
||||
// Skip versions that were already cached and Unreleased
|
||||
if version.Name != "Unreleased" {
|
||||
if err := g.cache.SaveVersion(version); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to save version to cache: %v\n", err)
|
||||
}
|
||||
|
||||
for _, commit := range version.Commits {
|
||||
if err := g.cache.SaveCommit(commit, version.Name); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to save commit to cache: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update the last processed tag
|
||||
if err := g.cache.SetLastProcessedTag(currentTag); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to update last processed tag: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
versions, err := g.gitWalker.WalkHistory()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to walk history: %w", err)
|
||||
}
|
||||
|
||||
g.versions = versions
|
||||
|
||||
if g.cache != nil {
|
||||
for _, version := range versions {
|
||||
if err := g.cache.SaveVersion(version); err != nil {
|
||||
return fmt.Errorf("failed to save version to cache: %w", err)
|
||||
}
|
||||
|
||||
for _, commit := range version.Commits {
|
||||
if err := g.cache.SaveCommit(commit, version.Name); err != nil {
|
||||
return fmt.Errorf("failed to save commit to cache: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save the latest tag as our cache anchor point
|
||||
if latestTag, err := g.gitWalker.GetLatestTag(); err == nil && latestTag != "" {
|
||||
if err := g.cache.SetLastProcessedTag(latestTag); err != nil {
|
||||
return fmt.Errorf("failed to save last processed tag: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Generator) fetchPRs() error {
|
||||
// First, load all cached PRs
|
||||
if g.cache != nil {
|
||||
cachedPRs, err := g.cache.GetAllPRs()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to load cached PRs: %v\n", err)
|
||||
} else {
|
||||
g.prs = cachedPRs
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we need to fetch new PRs
|
||||
var lastSync time.Time
|
||||
if g.cache != nil {
|
||||
lastSync, _ = g.cache.GetLastPRSync()
|
||||
}
|
||||
|
||||
// Check if we need to sync for missing PRs
|
||||
missingPRs := false
|
||||
for _, version := range g.versions {
|
||||
for _, prNum := range version.PRNumbers {
|
||||
if _, exists := g.prs[prNum]; !exists {
|
||||
missingPRs = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if missingPRs {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if missingPRs {
|
||||
fmt.Fprintf(os.Stderr, "Full sync triggered due to missing PRs in cache.\n")
|
||||
}
|
||||
// If we have never synced or it's been more than 24 hours, do a full sync
|
||||
// Also sync if we have versions with PR numbers that aren't cached
|
||||
needsSync := lastSync.IsZero() || time.Since(lastSync) > 24*time.Hour || g.cfg.ForcePRSync || missingPRs
|
||||
|
||||
if !needsSync {
|
||||
fmt.Fprintf(os.Stderr, "Using cached PR data (last sync: %s)\n", lastSync.Format("2006-01-02 15:04:05"))
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "Fetching merged PRs from GitHub using GraphQL...\n")
|
||||
|
||||
// Use GraphQL for ultimate performance - gets everything in ~5-10 calls
|
||||
prs, err := g.ghClient.FetchAllMergedPRsGraphQL(lastSync)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "GraphQL fetch failed, falling back to REST API: %v\n", err)
|
||||
// Fall back to REST API
|
||||
prs, err = g.ghClient.FetchAllMergedPRs(lastSync)
|
||||
if err != nil {
|
||||
return fmt.Errorf("both GraphQL and REST API failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update our PR map with new data
|
||||
for _, pr := range prs {
|
||||
g.prs[pr.Number] = pr
|
||||
}
|
||||
|
||||
// Save all PRs to cache in a batch transaction
|
||||
if g.cache != nil && len(prs) > 0 {
|
||||
// Save PRs
|
||||
if err := g.cache.SavePRBatch(prs); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to cache PRs: %v\n", err)
|
||||
}
|
||||
|
||||
// Save SHA→PR mappings for lightning-fast git operations
|
||||
if err := g.cache.SaveCommitPRMappings(prs); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to cache commit mappings: %v\n", err)
|
||||
}
|
||||
|
||||
// Update last sync timestamp
|
||||
if err := g.cache.SetLastPRSync(time.Now()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to update last sync timestamp: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(prs) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "Fetched %d PRs with commits (total cached: %d)\n", len(prs), len(g.prs))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Generator) formatChangelog() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("# Changelog\n")
|
||||
|
||||
versionList := g.getSortedVersions()
|
||||
|
||||
for _, version := range versionList {
|
||||
if g.cfg.Version != "" && version.Name != g.cfg.Version {
|
||||
continue
|
||||
}
|
||||
|
||||
versionText := g.formatVersion(version)
|
||||
if versionText != "" {
|
||||
sb.WriteString("\n")
|
||||
sb.WriteString(versionText)
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (g *Generator) getSortedVersions() []*git.Version {
|
||||
var versions []*git.Version
|
||||
var releasedVersions []*git.Version
|
||||
|
||||
// Collect all released versions (non-"Unreleased")
|
||||
for name, version := range g.versions {
|
||||
if name != "Unreleased" {
|
||||
releasedVersions = append(releasedVersions, version)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort released versions by date (newest first)
|
||||
sort.Slice(releasedVersions, func(i, j int) bool {
|
||||
return releasedVersions[i].Date.After(releasedVersions[j].Date)
|
||||
})
|
||||
|
||||
// Add "Unreleased" first if it exists and has commits
|
||||
if unreleased, exists := g.versions["Unreleased"]; exists && len(unreleased.Commits) > 0 {
|
||||
versions = append(versions, unreleased)
|
||||
}
|
||||
|
||||
// Add sorted released versions
|
||||
versions = append(versions, releasedVersions...)
|
||||
|
||||
if g.cfg.Limit > 0 && len(versions) > g.cfg.Limit {
|
||||
versions = versions[:g.cfg.Limit]
|
||||
}
|
||||
|
||||
return versions
|
||||
}
|
||||
|
||||
func (g *Generator) formatVersion(version *git.Version) string {
|
||||
var sb strings.Builder
|
||||
|
||||
// Generate raw content
|
||||
rawContent := g.generateRawVersionContent(version)
|
||||
if rawContent == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
header := g.formatVersionHeader(version)
|
||||
sb.WriteString(("\n"))
|
||||
sb.WriteString(header)
|
||||
|
||||
// If AI summarization is enabled, enhance with AI
|
||||
if g.cfg.EnableAISummary {
|
||||
// For "Unreleased", check if content has changed since last AI summary
|
||||
if version.Name == "Unreleased" && version.AISummary != "" && g.cache != nil {
|
||||
// Get cached content hash
|
||||
cachedHash, err := g.cache.GetUnreleasedContentHash()
|
||||
if err == nil {
|
||||
// Calculate current content hash
|
||||
currentHash := hashContent(rawContent)
|
||||
if cachedHash == currentHash {
|
||||
// Content unchanged, use cached summary
|
||||
fmt.Fprintf(os.Stderr, "✅ %s content unchanged (skipping AI)\n", version.Name)
|
||||
sb.WriteString(version.AISummary)
|
||||
return fixMarkdown(sb.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For released versions, if we have cached AI summary, use it!
|
||||
if version.Name != "Unreleased" && version.AISummary != "" {
|
||||
fmt.Fprintf(os.Stderr, "✅ %s already summarized (skipping)\n", version.Name)
|
||||
sb.WriteString(version.AISummary)
|
||||
return fixMarkdown(sb.String())
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "🤖 AI summarizing %s...", version.Name)
|
||||
|
||||
aiSummary, err := SummarizeVersionContent(rawContent)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, " Failed: %v\n", err)
|
||||
sb.WriteString((rawContent))
|
||||
return fixMarkdown(sb.String())
|
||||
}
|
||||
if checkForAIError(aiSummary) {
|
||||
fmt.Fprintf(os.Stderr, " AI error detected, using raw content instead\n")
|
||||
sb.WriteString(rawContent)
|
||||
fmt.Fprintf(os.Stderr, "Raw Content was: (%d bytes) %s \n", len(rawContent), rawContent)
|
||||
fmt.Fprintf(os.Stderr, "AI Summary was: (%d bytes) %s\n", len(aiSummary), aiSummary)
|
||||
return fixMarkdown(sb.String())
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, " Done!\n")
|
||||
aiSummary = strings.TrimSpace(aiSummary)
|
||||
|
||||
// Cache the AI summary and content hash
|
||||
version.AISummary = aiSummary
|
||||
if g.cache != nil {
|
||||
if err := g.cache.UpdateVersionAISummary(version.Name, aiSummary); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to cache AI summary: %v\n", err)
|
||||
}
|
||||
// Cache content hash for "Unreleased" to detect changes
|
||||
if version.Name == "Unreleased" {
|
||||
if err := g.cache.SetUnreleasedContentHash(hashContent(rawContent)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to cache content hash: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sb.WriteString(aiSummary)
|
||||
return fixMarkdown(sb.String())
|
||||
}
|
||||
|
||||
sb.WriteString(rawContent)
|
||||
return fixMarkdown(sb.String())
|
||||
}
|
||||
|
||||
func checkForAIError(summary string) bool {
|
||||
// Check for common AI error patterns
|
||||
errorPatterns := []string{
|
||||
"I don't see any", "please provide",
|
||||
"content you've provided appears to be incomplete",
|
||||
}
|
||||
|
||||
for _, pattern := range errorPatterns {
|
||||
if strings.Contains(summary, pattern) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// formatVersionHeader formats just the version header (## ...)
|
||||
func (g *Generator) formatVersionHeader(version *git.Version) string {
|
||||
if version.Name == "Unreleased" {
|
||||
return "## Unreleased\n\n"
|
||||
}
|
||||
return fmt.Sprintf("\n## %s (%s)\n\n", version.Name, version.Date.Format("2006-01-02"))
|
||||
}
|
||||
|
||||
// generateRawVersionContent generates the raw content (PRs + commits) for a version
|
||||
func (g *Generator) generateRawVersionContent(version *git.Version) string {
|
||||
var sb strings.Builder
|
||||
|
||||
// Build a set of commit SHAs that are part of fetched PRs
|
||||
prCommitSHAs := make(map[string]bool)
|
||||
for _, prNum := range version.PRNumbers {
|
||||
if pr, exists := g.prs[prNum]; exists {
|
||||
for _, prCommit := range pr.Commits {
|
||||
prCommitSHAs[prCommit.SHA] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prCommits := make(map[int][]*git.Commit)
|
||||
directCommits := []*git.Commit{}
|
||||
|
||||
for _, commit := range version.Commits {
|
||||
// Skip version bump commits from output
|
||||
if commit.IsVersion {
|
||||
continue
|
||||
}
|
||||
|
||||
// If this commit is part of a fetched PR, don't include it in direct commits
|
||||
if prCommitSHAs[commit.SHA] {
|
||||
continue
|
||||
}
|
||||
|
||||
if commit.PRNumber > 0 {
|
||||
prCommits[commit.PRNumber] = append(prCommits[commit.PRNumber], commit)
|
||||
} else {
|
||||
directCommits = append(directCommits, commit)
|
||||
}
|
||||
}
|
||||
|
||||
// There are occasionally no PRs or direct commits other than version bumps, so we handle that gracefully
|
||||
if len(prCommits) == 0 && len(directCommits) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
prependNewline := ""
|
||||
for _, prNum := range version.PRNumbers {
|
||||
if pr, exists := g.prs[prNum]; exists {
|
||||
sb.WriteString(prependNewline)
|
||||
sb.WriteString(g.formatPR(pr))
|
||||
prependNewline = "\n"
|
||||
}
|
||||
}
|
||||
|
||||
if len(directCommits) > 0 {
|
||||
// Sort direct commits by date (newest first) for consistent ordering
|
||||
sort.Slice(directCommits, func(i, j int) bool {
|
||||
return directCommits[i].Date.After(directCommits[j].Date)
|
||||
})
|
||||
|
||||
sb.WriteString(prependNewline + "### Direct commits\n\n")
|
||||
for _, commit := range directCommits {
|
||||
message := g.formatCommitMessage(strings.TrimSpace(commit.Message))
|
||||
if message != "" && !g.isDuplicateMessage(message, directCommits) {
|
||||
sb.WriteString(fmt.Sprintf("- %s\n", message))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fixMarkdown(
|
||||
strings.ReplaceAll(sb.String(), "\n-\n", "\n"), // Remove empty list items
|
||||
)
|
||||
}
|
||||
|
||||
func fixMarkdown(content string) string {
|
||||
|
||||
// Fix MD032/blank-around-lists: Lists should be surrounded by blank lines
|
||||
lines := strings.Split(content, "\n")
|
||||
inList := false
|
||||
preListNewline := false
|
||||
for i := range lines {
|
||||
line := strings.TrimSpace(lines[i])
|
||||
if strings.HasPrefix(line, "- ") || strings.HasPrefix(line, "* ") {
|
||||
if !inList {
|
||||
inList = true
|
||||
// Ensure there's a blank line before the list starts
|
||||
if !preListNewline && i > 0 && lines[i-1] != "" {
|
||||
line = "\n" + line
|
||||
preListNewline = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if inList {
|
||||
inList = false
|
||||
preListNewline = false
|
||||
}
|
||||
}
|
||||
lines[i] = strings.TrimRight(line, " \t")
|
||||
}
|
||||
|
||||
fixedContent := strings.TrimSpace(strings.Join(lines, "\n"))
|
||||
|
||||
return fixedContent + "\n"
|
||||
}
|
||||
|
||||
func (g *Generator) formatPR(pr *github.PR) string {
|
||||
var sb strings.Builder
|
||||
|
||||
pr.Title = strings.TrimRight(strings.TrimSpace(pr.Title), ".")
|
||||
|
||||
// Add type indicator for non-users
|
||||
authorName := pr.Author
|
||||
switch pr.AuthorType {
|
||||
case "bot":
|
||||
authorName += "[bot]"
|
||||
case "organization":
|
||||
authorName += "[org]"
|
||||
}
|
||||
|
||||
sb.WriteString(fmt.Sprintf("### PR [#%d](%s) by [%s](%s): %s\n\n",
|
||||
pr.Number, pr.URL, authorName, pr.AuthorURL, strings.TrimSpace(pr.Title)))
|
||||
|
||||
changes := g.extractChanges(pr)
|
||||
for _, change := range changes {
|
||||
if change != "" {
|
||||
sb.WriteString(fmt.Sprintf("- %s\n", change))
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (g *Generator) extractChanges(pr *github.PR) []string {
|
||||
var changes []string
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, commit := range pr.Commits {
|
||||
message := g.formatCommitMessage(commit.Message)
|
||||
if message != "" && !seen[message] {
|
||||
seen[message] = true
|
||||
changes = append(changes, message)
|
||||
}
|
||||
}
|
||||
|
||||
if len(changes) == 0 && pr.Body != "" {
|
||||
lines := strings.Split(pr.Body, "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "- ") || strings.HasPrefix(line, "* ") {
|
||||
change := strings.TrimPrefix(strings.TrimPrefix(line, "- "), "* ")
|
||||
if change != "" {
|
||||
changes = append(changes, change)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
func normalizeLineEndings(content string) string {
|
||||
return strings.ReplaceAll(content, "\r\n", "\n")
|
||||
}
|
||||
|
||||
func (g *Generator) formatCommitMessage(message string) string {
|
||||
strings_to_remove := []string{
|
||||
"### CHANGES\n", "## CHANGES\n", "# CHANGES\n",
|
||||
"...\n", "---\n", "## Changes\n", "## Change",
|
||||
"Update version to v..1 and commit\n",
|
||||
"# What this Pull Request (PR) does\n",
|
||||
"# Conflicts:",
|
||||
}
|
||||
|
||||
message = normalizeLineEndings(message)
|
||||
// No hard tabs
|
||||
message = strings.ReplaceAll(message, "\t", " ")
|
||||
|
||||
if len(message) > 0 {
|
||||
message = strings.ToUpper(message[:1]) + message[1:]
|
||||
}
|
||||
|
||||
for _, str := range strings_to_remove {
|
||||
if strings.Contains(message, str) {
|
||||
message = strings.ReplaceAll(message, str, "")
|
||||
}
|
||||
}
|
||||
|
||||
message = fixFormatting(message)
|
||||
|
||||
return message
|
||||
}
|
||||
|
||||
func fixFormatting(message string) string {
|
||||
// Turn "*"" lists into "-" lists"
|
||||
message = strings.ReplaceAll(message, "* ", "- ")
|
||||
// Remove extra spaces around dashes
|
||||
message = strings.ReplaceAll(message, "- ", "- ")
|
||||
message = strings.ReplaceAll(message, "- ", "- ")
|
||||
// turn bare URL into <URL>
|
||||
if strings.Contains(message, "http://") || strings.Contains(message, "https://") {
|
||||
// Use regex to wrap bare URLs with angle brackets
|
||||
urlRegex := regexp.MustCompile(`\b(https?://[^\s<>]+)`)
|
||||
message = urlRegex.ReplaceAllString(message, "<$1>")
|
||||
}
|
||||
|
||||
// Replace "## LINKS\n" with "- "
|
||||
message = strings.ReplaceAll(message, "## LINKS\n", "- ")
|
||||
// Dependabot messages: "- [Commits]" should become "\n- [Commits]"
|
||||
message = strings.TrimSpace(message)
|
||||
// Turn multiple newlines into a single newline
|
||||
message = strings.TrimSpace(strings.ReplaceAll(message, "\n\n", "\n"))
|
||||
// Fix inline trailing spaces
|
||||
message = strings.ReplaceAll(message, " \n", "\n")
|
||||
// Fix weird indent before list,
|
||||
message = strings.ReplaceAll(message, "\n - ", "\n- ")
|
||||
|
||||
// blanks-around-lists MD032 fix
|
||||
// Use regex to ensure blank line before list items that don't already have one
|
||||
listRegex := regexp.MustCompile(`(?m)([^\n-].*[^:\n])\n([-*] .*)`)
|
||||
message = listRegex.ReplaceAllString(message, "$1\n\n$2")
|
||||
|
||||
// Change random first-level "#" to 4th level "####"
|
||||
// This is a hack to fix spurious first-level headings that are not actual headings
|
||||
// but rather just comments or notes in the commit message.
|
||||
message = strings.ReplaceAll(message, "# ", "\n#### ")
|
||||
message = strings.ReplaceAll(message, "\n\n\n", "\n\n")
|
||||
|
||||
// Wrap any non-wrapped Emails with angle brackets
|
||||
emailRegex := regexp.MustCompile(`([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})`)
|
||||
message = emailRegex.ReplaceAllString(message, "<$1>")
|
||||
|
||||
// Wrap any non-wrapped URLs with angle brackets
|
||||
urlRegex := regexp.MustCompile(`(https?://[^\s<]+)`)
|
||||
message = urlRegex.ReplaceAllString(message, "<$1>")
|
||||
|
||||
message = strings.ReplaceAll(message, "<<", "<")
|
||||
message = strings.ReplaceAll(message, ">>", ">")
|
||||
|
||||
// Fix some spurious Issue/PR links at the beginning of a commit message line
|
||||
prOrIssueLinkRegex := regexp.MustCompile("\n" + `(#\d+)`)
|
||||
message = prOrIssueLinkRegex.ReplaceAllString(message, " $1")
|
||||
|
||||
// Remove leading/trailing whitespace
|
||||
message = strings.TrimSpace(message)
|
||||
return message
|
||||
}
|
||||
|
||||
func (g *Generator) isDuplicateMessage(message string, commits []*git.Commit) bool {
|
||||
if message == "." || strings.ToLower(message) == "fix" {
|
||||
count := 0
|
||||
for _, commit := range commits {
|
||||
formatted := g.formatCommitMessage(commit.Message)
|
||||
if formatted == message {
|
||||
count++
|
||||
if count > 1 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hashContent generates a SHA256 hash of the content for change detection
|
||||
func hashContent(content string) string {
|
||||
hash := sha256.Sum256([]byte(content))
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
79
cmd/generate_changelog/internal/changelog/summarize.go
Normal file
79
cmd/generate_changelog/internal/changelog/summarize.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package changelog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const DefaultSummarizeModel = "claude-sonnet-4-20250514"
|
||||
const MinContentLength = 256 // Minimum content length to consider for summarization
|
||||
|
||||
const prompt = `# ROLE
|
||||
You are an expert Technical Writer specializing in creating clear, concise,
|
||||
and professional release notes from raw Git commit logs.
|
||||
|
||||
# TASK
|
||||
Your goal is to transform a provided block of Git commit logs into a clean,
|
||||
human-readable changelog summary. You will identify the most important changes,
|
||||
format them as a bulleted list, and preserve the associated Pull Request (PR)
|
||||
information.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
Follow these steps in order:
|
||||
1. Deeply analyze the input. You will be given a block of text containing PR
|
||||
information and commit log messages. Carefully read through the logs
|
||||
to identify individual commits and their descriptions.
|
||||
2. Identify Key Changes: Focus on commits that represent significant changes,
|
||||
such as new features ("feat"), bug fixes ("fix"), performance improvements ("perf"),
|
||||
or breaking changes ("BREAKING CHANGE").
|
||||
3. Select the Top 5: From the identified key changes, select a maximum of the five
|
||||
(5) most impactful ones to include in the summary.
|
||||
If there are five or fewer total changes, include all of them.
|
||||
4. Format the Output:
|
||||
- Where you see a PR header, include the PR header verbatim. NO CHANGES.
|
||||
**This is a critical rule: Do not modify the PR header, as it contains
|
||||
important links.** What follow the PR header are the related changes.
|
||||
- Do not add any additional text or preamble. Begin directly with the output.
|
||||
- Use bullet points for each key change. Starting each point with a hyphen ("-").
|
||||
- Ensure that the summary is concise and focused on the main changes.
|
||||
- The summary should be in American English (en-US), using proper grammar and punctuation.
|
||||
5. If the content is too brief or you do not see any PR headers, return the content as is.
|
||||
`
|
||||
|
||||
// getSummarizeModel returns the model to use for AI summarization
|
||||
func getSummarizeModel() string {
|
||||
if model := os.Getenv("FABRIC_CHANGELOG_SUMMARIZE_MODEL"); model != "" {
|
||||
return model
|
||||
}
|
||||
return DefaultSummarizeModel
|
||||
}
|
||||
|
||||
// SummarizeVersionContent takes raw version content and returns AI-enhanced summary
|
||||
func SummarizeVersionContent(content string) (string, error) {
|
||||
if strings.TrimSpace(content) == "" {
|
||||
return "", fmt.Errorf("no content to summarize")
|
||||
}
|
||||
if len(content) < MinContentLength {
|
||||
// If content is too brief, return it as is
|
||||
return content, nil
|
||||
}
|
||||
|
||||
model := getSummarizeModel()
|
||||
|
||||
cmd := exec.Command("fabric", "-m", model, prompt)
|
||||
cmd.Stdin = strings.NewReader(content)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fabric command failed: %w", err)
|
||||
}
|
||||
|
||||
summary := strings.TrimSpace(string(output))
|
||||
if summary == "" {
|
||||
return "", fmt.Errorf("fabric returned empty summary")
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
15
cmd/generate_changelog/internal/config/config.go
Normal file
15
cmd/generate_changelog/internal/config/config.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package config
|
||||
|
||||
type Config struct {
|
||||
RepoPath string
|
||||
OutputFile string
|
||||
Limit int
|
||||
Version string
|
||||
SaveData bool
|
||||
CacheFile string
|
||||
NoCache bool
|
||||
RebuildCache bool
|
||||
GitHubToken string
|
||||
ForcePRSync bool
|
||||
EnableAISummary bool
|
||||
}
|
||||
26
cmd/generate_changelog/internal/git/types.go
Normal file
26
cmd/generate_changelog/internal/git/types.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type Commit struct {
|
||||
SHA string
|
||||
Message string
|
||||
Author string
|
||||
Email string
|
||||
Date time.Time
|
||||
IsMerge bool
|
||||
PRNumber int
|
||||
IsVersion bool
|
||||
Version string
|
||||
}
|
||||
|
||||
type Version struct {
|
||||
Name string
|
||||
Date time.Time
|
||||
CommitSHA string
|
||||
Commits []*Commit
|
||||
PRNumbers []int
|
||||
AISummary string
|
||||
}
|
||||
402
cmd/generate_changelog/internal/git/walker.go
Normal file
402
cmd/generate_changelog/internal/git/walker.go
Normal file
@@ -0,0 +1,402 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/go-git/go-git/v5/plumbing/storer"
|
||||
)
|
||||
|
||||
var (
|
||||
versionPattern = regexp.MustCompile(`Update version to (v\d+\.\d+\.\d+)`)
|
||||
prPattern = regexp.MustCompile(`Merge pull request #(\d+)`)
|
||||
)
|
||||
|
||||
type Walker struct {
|
||||
repo *git.Repository
|
||||
}
|
||||
|
||||
func NewWalker(repoPath string) (*Walker, error) {
|
||||
repo, err := git.PlainOpen(repoPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open repository: %w", err)
|
||||
}
|
||||
|
||||
return &Walker{repo: repo}, nil
|
||||
}
|
||||
|
||||
// GetLatestTag returns the name of the most recent tag by committer date
|
||||
func (w *Walker) GetLatestTag() (string, error) {
|
||||
tagRefs, err := w.repo.Tags()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var latestTagCommit *object.Commit
|
||||
var latestTagName string
|
||||
|
||||
err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error {
|
||||
revision := plumbing.Revision(tagRef.Name().String())
|
||||
tagCommitHash, err := w.repo.ResolveRevision(revision)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
commit, err := w.repo.CommitObject(*tagCommitHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if latestTagCommit == nil {
|
||||
latestTagCommit = commit
|
||||
latestTagName = tagRef.Name().Short() // Get short name like "v1.4.245"
|
||||
}
|
||||
|
||||
if commit.Committer.When.After(latestTagCommit.Committer.When) {
|
||||
latestTagCommit = commit
|
||||
latestTagName = tagRef.Name().Short()
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return latestTagName, nil
|
||||
}
|
||||
|
||||
// WalkCommitsSinceTag walks commits from the specified tag to HEAD and returns only "Unreleased" version
|
||||
func (w *Walker) WalkCommitsSinceTag(tagName string) (*Version, error) {
|
||||
// Get the tag reference
|
||||
tagRef, err := w.repo.Tag(tagName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find tag %s: %w", tagName, err)
|
||||
}
|
||||
|
||||
// Get the commit that the tag points to
|
||||
tagCommit, err := w.repo.CommitObject(tagRef.Hash())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get tag commit: %w", err)
|
||||
}
|
||||
|
||||
// Get HEAD
|
||||
headRef, err := w.repo.Head()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get HEAD: %w", err)
|
||||
}
|
||||
|
||||
// Walk from HEAD back to the tag commit (exclusive)
|
||||
commitIter, err := w.repo.Log(&git.LogOptions{
|
||||
From: headRef.Hash(),
|
||||
Order: git.LogOrderCommitterTime,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get commit log: %w", err)
|
||||
}
|
||||
|
||||
version := &Version{
|
||||
Name: "Unreleased",
|
||||
Commits: []*Commit{},
|
||||
}
|
||||
|
||||
prNumbers := []int{}
|
||||
|
||||
err = commitIter.ForEach(func(c *object.Commit) error {
|
||||
// Stop when we reach the tag commit (don't include it)
|
||||
if c.Hash == tagCommit.Hash {
|
||||
return fmt.Errorf("reached tag commit") // Use error to break out of iteration
|
||||
}
|
||||
|
||||
commit := &Commit{
|
||||
SHA: c.Hash.String(),
|
||||
Message: strings.TrimSpace(c.Message),
|
||||
Date: c.Committer.When,
|
||||
}
|
||||
|
||||
// Check for version patterns
|
||||
if versionMatch := versionPattern.FindStringSubmatch(commit.Message); versionMatch != nil {
|
||||
commit.IsVersion = true
|
||||
}
|
||||
|
||||
// Check for PR merge patterns
|
||||
if prMatch := prPattern.FindStringSubmatch(commit.Message); prMatch != nil {
|
||||
if prNumber, err := strconv.Atoi(prMatch[1]); err == nil {
|
||||
commit.PRNumber = prNumber
|
||||
prNumbers = append(prNumbers, prNumber)
|
||||
}
|
||||
}
|
||||
|
||||
version.Commits = append(version.Commits, commit)
|
||||
return nil
|
||||
})
|
||||
|
||||
// Ignore the "reached tag commit" error - it's expected
|
||||
if err != nil && !strings.Contains(err.Error(), "reached tag commit") {
|
||||
return nil, fmt.Errorf("failed to walk commits: %w", err)
|
||||
}
|
||||
|
||||
// Remove duplicates from prNumbers and set them
|
||||
prNumbersMap := make(map[int]bool)
|
||||
for _, prNum := range prNumbers {
|
||||
prNumbersMap[prNum] = true
|
||||
}
|
||||
|
||||
version.PRNumbers = make([]int, 0, len(prNumbersMap))
|
||||
for prNum := range prNumbersMap {
|
||||
version.PRNumbers = append(version.PRNumbers, prNum)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
func (w *Walker) WalkHistory() (map[string]*Version, error) {
|
||||
ref, err := w.repo.Head()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get HEAD: %w", err)
|
||||
}
|
||||
|
||||
commitIter, err := w.repo.Log(&git.LogOptions{
|
||||
From: ref.Hash(),
|
||||
Order: git.LogOrderCommitterTime,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get commit log: %w", err)
|
||||
}
|
||||
|
||||
versions := make(map[string]*Version)
|
||||
currentVersion := "Unreleased"
|
||||
versions[currentVersion] = &Version{
|
||||
Name: currentVersion,
|
||||
Commits: []*Commit{},
|
||||
}
|
||||
|
||||
prNumbers := make(map[string][]int)
|
||||
|
||||
err = commitIter.ForEach(func(c *object.Commit) error {
|
||||
// c.Message = Summarize(c.Message)
|
||||
commit := &Commit{
|
||||
SHA: c.Hash.String(),
|
||||
Message: strings.TrimSpace(c.Message),
|
||||
Author: c.Author.Name,
|
||||
Email: c.Author.Email,
|
||||
Date: c.Author.When,
|
||||
IsMerge: len(c.ParentHashes) > 1,
|
||||
}
|
||||
|
||||
if matches := versionPattern.FindStringSubmatch(commit.Message); len(matches) > 1 {
|
||||
commit.IsVersion = true
|
||||
commit.Version = matches[1]
|
||||
currentVersion = commit.Version
|
||||
|
||||
if _, exists := versions[currentVersion]; !exists {
|
||||
versions[currentVersion] = &Version{
|
||||
Name: currentVersion,
|
||||
Date: commit.Date,
|
||||
CommitSHA: commit.SHA,
|
||||
Commits: []*Commit{},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if matches := prPattern.FindStringSubmatch(commit.Message); len(matches) > 1 {
|
||||
prNumber := 0
|
||||
fmt.Sscanf(matches[1], "%d", &prNumber)
|
||||
commit.PRNumber = prNumber
|
||||
|
||||
prNumbers[currentVersion] = append(prNumbers[currentVersion], prNumber)
|
||||
}
|
||||
|
||||
versions[currentVersion].Commits = append(versions[currentVersion].Commits, commit)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to walk commits: %w", err)
|
||||
}
|
||||
|
||||
for version, prs := range prNumbers {
|
||||
versions[version].PRNumbers = dedupInts(prs)
|
||||
}
|
||||
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
func (w *Walker) GetRepoInfo() (owner string, name string, err error) {
|
||||
remotes, err := w.repo.Remotes()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get remotes: %w", err)
|
||||
}
|
||||
|
||||
// First try upstream (preferred for forks)
|
||||
for _, remote := range remotes {
|
||||
if remote.Config().Name == "upstream" {
|
||||
urls := remote.Config().URLs
|
||||
if len(urls) > 0 {
|
||||
owner, name = parseGitHubURL(urls[0])
|
||||
if owner != "" && name != "" {
|
||||
return owner, name, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Then try origin
|
||||
for _, remote := range remotes {
|
||||
if remote.Config().Name == "origin" {
|
||||
urls := remote.Config().URLs
|
||||
if len(urls) > 0 {
|
||||
owner, name = parseGitHubURL(urls[0])
|
||||
if owner != "" && name != "" {
|
||||
return owner, name, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "danielmiessler", "fabric", nil
|
||||
}
|
||||
|
||||
func parseGitHubURL(url string) (owner, repo string) {
|
||||
patterns := []string{
|
||||
`github\.com[:/]([^/]+)/([^/.]+)`,
|
||||
`github\.com[:/]([^/]+)/([^/]+)\.git$`,
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
re := regexp.MustCompile(pattern)
|
||||
matches := re.FindStringSubmatch(url)
|
||||
if len(matches) > 2 {
|
||||
return matches[1], matches[2]
|
||||
}
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// WalkHistorySinceTag walks git history from HEAD down to (but not including) the specified tag
|
||||
// and returns any version commits found along the way
|
||||
func (w *Walker) WalkHistorySinceTag(sinceTag string) (map[string]*Version, error) {
|
||||
// Get the commit SHA for the sinceTag
|
||||
tagRef, err := w.repo.Tag(sinceTag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get tag %s: %w", sinceTag, err)
|
||||
}
|
||||
|
||||
tagCommit, err := w.repo.CommitObject(tagRef.Hash())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get commit for tag %s: %w", sinceTag, err)
|
||||
}
|
||||
|
||||
// Get HEAD reference
|
||||
ref, err := w.repo.Head()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get HEAD: %w", err)
|
||||
}
|
||||
|
||||
// Walk from HEAD down to the tag commit (excluding it)
|
||||
commitIter, err := w.repo.Log(&git.LogOptions{
|
||||
From: ref.Hash(),
|
||||
Order: git.LogOrderCommitterTime,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create commit iterator: %w", err)
|
||||
}
|
||||
defer commitIter.Close()
|
||||
|
||||
versions := make(map[string]*Version)
|
||||
currentVersion := "Unreleased"
|
||||
prNumbers := make(map[string][]int)
|
||||
|
||||
err = commitIter.ForEach(func(c *object.Commit) error {
|
||||
// Stop iteration when the hash of the current commit matches the hash of the specified sinceTag commit
|
||||
if c.Hash == tagCommit.Hash {
|
||||
return storer.ErrStop
|
||||
}
|
||||
|
||||
commit := &Commit{
|
||||
SHA: c.Hash.String(),
|
||||
Message: strings.TrimSpace(c.Message),
|
||||
Author: c.Author.Name,
|
||||
Email: c.Author.Email,
|
||||
Date: c.Author.When,
|
||||
IsMerge: len(c.ParentHashes) > 1,
|
||||
}
|
||||
|
||||
// Check for version pattern
|
||||
if matches := versionPattern.FindStringSubmatch(commit.Message); len(matches) > 1 {
|
||||
commit.IsVersion = true
|
||||
commit.Version = matches[1]
|
||||
currentVersion = commit.Version
|
||||
|
||||
if _, exists := versions[currentVersion]; !exists {
|
||||
versions[currentVersion] = &Version{
|
||||
Name: currentVersion,
|
||||
Date: commit.Date,
|
||||
CommitSHA: commit.SHA,
|
||||
Commits: []*Commit{},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check for PR merge pattern
|
||||
if matches := prPattern.FindStringSubmatch(commit.Message); len(matches) > 1 {
|
||||
prNumber, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
// Handle parsing error (e.g., log it or skip processing)
|
||||
return fmt.Errorf("failed to parse PR number: %v", err)
|
||||
}
|
||||
commit.PRNumber = prNumber
|
||||
|
||||
prNumbers[currentVersion] = append(prNumbers[currentVersion], prNumber)
|
||||
}
|
||||
|
||||
// Add commit to current version
|
||||
if _, exists := versions[currentVersion]; !exists {
|
||||
versions[currentVersion] = &Version{
|
||||
Name: currentVersion,
|
||||
Date: time.Time{}, // Zero value, will be set by version commit
|
||||
CommitSHA: "",
|
||||
Commits: []*Commit{},
|
||||
}
|
||||
}
|
||||
|
||||
versions[currentVersion].Commits = append(versions[currentVersion].Commits, commit)
|
||||
return nil
|
||||
})
|
||||
|
||||
// Handle the stop condition - storer.ErrStop is expected
|
||||
if err == storer.ErrStop {
|
||||
err = nil
|
||||
}
|
||||
|
||||
// Assign collected PR numbers to each version
|
||||
for version, prs := range prNumbers {
|
||||
versions[version].PRNumbers = dedupInts(prs)
|
||||
}
|
||||
|
||||
return versions, err
|
||||
}
|
||||
|
||||
func dedupInts(ints []int) []int {
|
||||
seen := make(map[int]bool)
|
||||
result := []int{}
|
||||
|
||||
for _, i := range ints {
|
||||
if !seen[i] {
|
||||
seen[i] = true
|
||||
result = append(result, i)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
354
cmd/generate_changelog/internal/github/client.go
Normal file
354
cmd/generate_changelog/internal/github/client.go
Normal file
@@ -0,0 +1,354 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-github/v66/github"
|
||||
"github.com/hasura/go-graphql-client"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
client *github.Client
|
||||
graphqlClient *graphql.Client
|
||||
owner string
|
||||
repo string
|
||||
token string
|
||||
}
|
||||
|
||||
func NewClient(token, owner, repo string) *Client {
|
||||
var githubClient *github.Client
|
||||
var httpClient *http.Client
|
||||
var gqlClient *graphql.Client
|
||||
|
||||
if token != "" {
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: token},
|
||||
)
|
||||
httpClient = oauth2.NewClient(context.Background(), ts)
|
||||
githubClient = github.NewClient(httpClient)
|
||||
gqlClient = graphql.NewClient("https://api.github.com/graphql", httpClient)
|
||||
} else {
|
||||
httpClient = http.DefaultClient
|
||||
githubClient = github.NewClient(nil)
|
||||
gqlClient = graphql.NewClient("https://api.github.com/graphql", httpClient)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
client: githubClient,
|
||||
graphqlClient: gqlClient,
|
||||
owner: owner,
|
||||
repo: repo,
|
||||
token: token,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) FetchPRs(prNumbers []int) ([]*PR, error) {
|
||||
if len(prNumbers) == 0 {
|
||||
return []*PR{}, nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
prs := make([]*PR, 0, len(prNumbers))
|
||||
prsChan := make(chan *PR, len(prNumbers))
|
||||
errChan := make(chan error, len(prNumbers))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
semaphore := make(chan struct{}, 10)
|
||||
|
||||
for _, prNumber := range prNumbers {
|
||||
wg.Add(1)
|
||||
go func(num int) {
|
||||
defer wg.Done()
|
||||
|
||||
semaphore <- struct{}{}
|
||||
defer func() { <-semaphore }()
|
||||
|
||||
pr, err := c.fetchSinglePR(ctx, num)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("failed to fetch PR #%d: %w", num, err)
|
||||
return
|
||||
}
|
||||
prsChan <- pr
|
||||
}(prNumber)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(prsChan)
|
||||
close(errChan)
|
||||
}()
|
||||
|
||||
var errors []error
|
||||
for pr := range prsChan {
|
||||
prs = append(prs, pr)
|
||||
}
|
||||
for err := range errChan {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
return prs, fmt.Errorf("some PRs failed to fetch: %v", errors)
|
||||
}
|
||||
|
||||
return prs, nil
|
||||
}
|
||||
|
||||
func (c *Client) fetchSinglePR(ctx context.Context, prNumber int) (*PR, error) {
|
||||
pr, _, err := c.client.PullRequests.Get(ctx, c.owner, c.repo, prNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
commits, _, err := c.client.PullRequests.ListCommits(ctx, c.owner, c.repo, prNumber, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch commits: %w", err)
|
||||
}
|
||||
|
||||
result := &PR{
|
||||
Number: prNumber,
|
||||
Title: getString(pr.Title),
|
||||
Body: getString(pr.Body),
|
||||
URL: getString(pr.HTMLURL),
|
||||
Commits: make([]PRCommit, 0, len(commits)),
|
||||
}
|
||||
|
||||
if pr.MergedAt != nil {
|
||||
result.MergedAt = pr.MergedAt.Time
|
||||
}
|
||||
|
||||
if pr.User != nil {
|
||||
result.Author = getString(pr.User.Login)
|
||||
result.AuthorURL = getString(pr.User.HTMLURL)
|
||||
userType := getString(pr.User.Type) // GitHub API returns "User", "Organization", or "Bot"
|
||||
|
||||
// Convert GitHub API type to lowercase
|
||||
switch userType {
|
||||
case "User":
|
||||
result.AuthorType = "user"
|
||||
case "Organization":
|
||||
result.AuthorType = "organization"
|
||||
case "Bot":
|
||||
result.AuthorType = "bot"
|
||||
default:
|
||||
result.AuthorType = "user" // Default fallback
|
||||
}
|
||||
}
|
||||
|
||||
if pr.MergeCommitSHA != nil {
|
||||
result.MergeCommit = *pr.MergeCommitSHA
|
||||
}
|
||||
|
||||
for _, commit := range commits {
|
||||
if commit.Commit != nil {
|
||||
prCommit := PRCommit{
|
||||
SHA: getString(commit.SHA),
|
||||
Message: strings.TrimSpace(getString(commit.Commit.Message)),
|
||||
}
|
||||
if commit.Commit.Author != nil {
|
||||
prCommit.Author = getString(commit.Commit.Author.Name)
|
||||
}
|
||||
result.Commits = append(result.Commits, prCommit)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func getString(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
// FetchAllMergedPRs fetches all merged PRs using GitHub's search API
|
||||
// This is much more efficient than fetching PRs individually
|
||||
func (c *Client) FetchAllMergedPRs(since time.Time) ([]*PR, error) {
|
||||
ctx := context.Background()
|
||||
var allPRs []*PR
|
||||
|
||||
// Build search query for merged PRs
|
||||
query := fmt.Sprintf("repo:%s/%s is:pr is:merged", c.owner, c.repo)
|
||||
if !since.IsZero() {
|
||||
query += fmt.Sprintf(" merged:>=%s", since.Format("2006-01-02"))
|
||||
}
|
||||
|
||||
opts := &github.SearchOptions{
|
||||
Sort: "created",
|
||||
Order: "desc",
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: 100, // Maximum allowed
|
||||
},
|
||||
}
|
||||
|
||||
for {
|
||||
result, resp, err := c.client.Search.Issues(ctx, query, opts)
|
||||
if err != nil {
|
||||
return allPRs, fmt.Errorf("failed to search PRs: %w", err)
|
||||
}
|
||||
|
||||
// Process PRs in parallel
|
||||
prsChan := make(chan *PR, len(result.Issues))
|
||||
errChan := make(chan error, len(result.Issues))
|
||||
var wg sync.WaitGroup
|
||||
semaphore := make(chan struct{}, 10) // Limit concurrent requests
|
||||
|
||||
for _, issue := range result.Issues {
|
||||
if issue.PullRequestLinks == nil {
|
||||
continue // Not a PR
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(prNumber int) {
|
||||
defer wg.Done()
|
||||
|
||||
semaphore <- struct{}{}
|
||||
defer func() { <-semaphore }()
|
||||
|
||||
pr, err := c.fetchSinglePR(ctx, prNumber)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("failed to fetch PR #%d: %w", prNumber, err)
|
||||
return
|
||||
}
|
||||
prsChan <- pr
|
||||
}(*issue.Number)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(prsChan)
|
||||
close(errChan)
|
||||
}()
|
||||
|
||||
// Collect results
|
||||
for pr := range prsChan {
|
||||
allPRs = append(allPRs, pr)
|
||||
}
|
||||
|
||||
// Check for errors
|
||||
for err := range errChan {
|
||||
// Log error but continue processing
|
||||
fmt.Fprintf(os.Stderr, "Warning: %v\n", err)
|
||||
}
|
||||
|
||||
if resp.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opts.Page = resp.NextPage
|
||||
}
|
||||
|
||||
return allPRs, nil
|
||||
}
|
||||
|
||||
// FetchAllMergedPRsGraphQL fetches all merged PRs with their commits using GraphQL
|
||||
// This is the ultimate optimization - gets everything in ~5-10 API calls
|
||||
func (c *Client) FetchAllMergedPRsGraphQL(since time.Time) ([]*PR, error) {
|
||||
ctx := context.Background()
|
||||
var allPRs []*PR
|
||||
var after *string
|
||||
totalFetched := 0
|
||||
|
||||
for {
|
||||
// Prepare variables
|
||||
variables := map[string]interface{}{
|
||||
"owner": graphql.String(c.owner),
|
||||
"repo": graphql.String(c.repo),
|
||||
"after": (*graphql.String)(after),
|
||||
}
|
||||
|
||||
// Execute GraphQL query
|
||||
var query PullRequestsQuery
|
||||
err := c.graphqlClient.Query(ctx, &query, variables)
|
||||
if err != nil {
|
||||
return allPRs, fmt.Errorf("GraphQL query failed: %w", err)
|
||||
}
|
||||
|
||||
prs := query.Repository.PullRequests.Nodes
|
||||
fmt.Fprintf(os.Stderr, "Fetched %d PRs via GraphQL (page %d)\n", len(prs), (totalFetched/100)+1)
|
||||
|
||||
// Convert GraphQL PRs to our PR struct
|
||||
for _, gqlPR := range prs {
|
||||
// If we have a since filter, stop when we reach older PRs
|
||||
if !since.IsZero() && gqlPR.MergedAt.Before(since) {
|
||||
fmt.Fprintf(os.Stderr, "Reached PRs older than %s, stopping\n", since.Format("2006-01-02"))
|
||||
return allPRs, nil
|
||||
}
|
||||
|
||||
pr := &PR{
|
||||
Number: gqlPR.Number,
|
||||
Title: gqlPR.Title,
|
||||
Body: gqlPR.Body,
|
||||
URL: gqlPR.URL,
|
||||
MergedAt: gqlPR.MergedAt,
|
||||
Commits: make([]PRCommit, 0, len(gqlPR.Commits.Nodes)),
|
||||
}
|
||||
|
||||
// Handle author - check if it's nil first
|
||||
if gqlPR.Author != nil {
|
||||
pr.Author = gqlPR.Author.Login
|
||||
pr.AuthorURL = gqlPR.Author.URL
|
||||
|
||||
switch gqlPR.Author.Typename {
|
||||
case "Bot":
|
||||
pr.AuthorType = "bot"
|
||||
case "Organization":
|
||||
pr.AuthorType = "organization"
|
||||
case "User":
|
||||
pr.AuthorType = "user"
|
||||
default:
|
||||
pr.AuthorType = "user" // fallback
|
||||
if gqlPR.Author.Typename != "" {
|
||||
fmt.Fprintf(os.Stderr, "PR #%d: Unknown author typename '%s'\n", gqlPR.Number, gqlPR.Author.Typename)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Author is nil - try to fetch from REST API as fallback
|
||||
fmt.Fprintf(os.Stderr, "PR #%d: Author is nil in GraphQL response, fetching from REST API\n", gqlPR.Number)
|
||||
|
||||
// Fetch this specific PR from REST API
|
||||
restPR, err := c.fetchSinglePR(ctx, gqlPR.Number)
|
||||
if err == nil && restPR != nil && restPR.Author != "" {
|
||||
pr.Author = restPR.Author
|
||||
pr.AuthorURL = restPR.AuthorURL
|
||||
pr.AuthorType = restPR.AuthorType
|
||||
} else {
|
||||
// Fallback if REST API also fails
|
||||
pr.Author = "[unknown]"
|
||||
pr.AuthorURL = ""
|
||||
pr.AuthorType = "user"
|
||||
}
|
||||
}
|
||||
|
||||
// Convert commits
|
||||
for _, commitNode := range gqlPR.Commits.Nodes {
|
||||
commit := PRCommit{
|
||||
SHA: commitNode.Commit.OID,
|
||||
Message: strings.TrimSpace(commitNode.Commit.Message),
|
||||
Author: commitNode.Commit.Author.Name,
|
||||
}
|
||||
pr.Commits = append(pr.Commits, commit)
|
||||
}
|
||||
|
||||
allPRs = append(allPRs, pr)
|
||||
}
|
||||
|
||||
totalFetched += len(prs)
|
||||
|
||||
// Check if we need to fetch more pages
|
||||
if !query.Repository.PullRequests.PageInfo.HasNextPage {
|
||||
break
|
||||
}
|
||||
|
||||
after = &query.Repository.PullRequests.PageInfo.EndCursor
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "Total PRs fetched via GraphQL: %d\n", len(allPRs))
|
||||
return allPRs, nil
|
||||
}
|
||||
57
cmd/generate_changelog/internal/github/types.go
Normal file
57
cmd/generate_changelog/internal/github/types.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package github
|
||||
|
||||
import "time"
|
||||
|
||||
type PR struct {
|
||||
Number int
|
||||
Title string
|
||||
Body string
|
||||
Author string
|
||||
AuthorURL string
|
||||
AuthorType string // "user", "organization", or "bot"
|
||||
URL string
|
||||
MergedAt time.Time
|
||||
Commits []PRCommit
|
||||
MergeCommit string
|
||||
}
|
||||
|
||||
type PRCommit struct {
|
||||
SHA string
|
||||
Message string
|
||||
Author string
|
||||
}
|
||||
|
||||
// GraphQL query structures for hasura client
|
||||
type PullRequestsQuery struct {
|
||||
Repository struct {
|
||||
PullRequests struct {
|
||||
PageInfo struct {
|
||||
HasNextPage bool
|
||||
EndCursor string
|
||||
}
|
||||
Nodes []struct {
|
||||
Number int
|
||||
Title string
|
||||
Body string
|
||||
URL string
|
||||
MergedAt time.Time
|
||||
Author *struct {
|
||||
Typename string `graphql:"__typename"`
|
||||
Login string `graphql:"login"`
|
||||
URL string `graphql:"url"`
|
||||
}
|
||||
Commits struct {
|
||||
Nodes []struct {
|
||||
Commit struct {
|
||||
OID string `graphql:"oid"`
|
||||
Message string
|
||||
Author struct {
|
||||
Name string
|
||||
}
|
||||
}
|
||||
}
|
||||
} `graphql:"commits(first: 250)"`
|
||||
}
|
||||
} `graphql:"pullRequests(first: 100, after: $after, states: MERGED, orderBy: {field: UPDATED_AT, direction: DESC})"`
|
||||
} `graphql:"repository(owner: $owner, name: $repo)"`
|
||||
}
|
||||
84
cmd/generate_changelog/main.go
Normal file
84
cmd/generate_changelog/main.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/changelog"
|
||||
"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/config"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
cfg = &config.Config{}
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "generate_changelog",
|
||||
Short: "Generate changelog from git history and GitHub PRs",
|
||||
Long: `A high-performance changelog generator that walks git history,
|
||||
collects version information and pull requests, and generates a
|
||||
comprehensive changelog in markdown format.`,
|
||||
RunE: run,
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.Flags().StringVarP(&cfg.RepoPath, "repo", "r", ".", "Repository path")
|
||||
rootCmd.Flags().StringVarP(&cfg.OutputFile, "output", "o", "", "Output file (default: stdout)")
|
||||
rootCmd.Flags().IntVarP(&cfg.Limit, "limit", "l", 0, "Limit number of versions (0 = all)")
|
||||
rootCmd.Flags().StringVarP(&cfg.Version, "version", "v", "", "Generate changelog for specific version")
|
||||
rootCmd.Flags().BoolVar(&cfg.SaveData, "save-data", false, "Save version data to JSON for debugging")
|
||||
rootCmd.Flags().StringVar(&cfg.CacheFile, "cache", "./cmd/generate_changelog/changelog.db", "Cache database file")
|
||||
rootCmd.Flags().BoolVar(&cfg.NoCache, "no-cache", false, "Disable cache usage")
|
||||
rootCmd.Flags().BoolVar(&cfg.RebuildCache, "rebuild-cache", false, "Rebuild cache from scratch")
|
||||
rootCmd.Flags().StringVar(&cfg.GitHubToken, "token", "", "GitHub API token (or set GITHUB_TOKEN env var)")
|
||||
rootCmd.Flags().BoolVar(&cfg.ForcePRSync, "force-pr-sync", false, "Force a full PR sync from GitHub (ignores cache age)")
|
||||
rootCmd.Flags().BoolVar(&cfg.EnableAISummary, "ai-summarize", false, "Generate AI-enhanced summaries using Fabric")
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, args []string) error {
|
||||
if cfg.GitHubToken == "" {
|
||||
cfg.GitHubToken = os.Getenv("GITHUB_TOKEN")
|
||||
}
|
||||
|
||||
generator, err := changelog.New(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create changelog generator: %w", err)
|
||||
}
|
||||
|
||||
output, err := generator.Generate()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate changelog: %w", err)
|
||||
}
|
||||
|
||||
if cfg.OutputFile != "" {
|
||||
if err := os.WriteFile(cfg.OutputFile, []byte(output), 0644); err != nil {
|
||||
return fmt.Errorf("failed to write output file: %w", err)
|
||||
}
|
||||
fmt.Printf("Changelog written to %s\n", cfg.OutputFile)
|
||||
} else {
|
||||
fmt.Print(output)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Load .env file from the same directory as the binary
|
||||
if exePath, err := os.Executable(); err == nil {
|
||||
envPath := filepath.Join(filepath.Dir(exePath), ".env")
|
||||
if _, err := os.Stat(envPath); err == nil {
|
||||
// .env file exists, load it
|
||||
if err := godotenv.Load(envPath); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Failed to load .env file: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,9 @@ _fabric() {
|
||||
'(--liststrategies)--liststrategies[List all strategies]' \
|
||||
'(--listvendors)--listvendors[List all vendors]' \
|
||||
'(--shell-complete-list)--shell-complete-list[Output raw list without headers/formatting (for shell completion)]' \
|
||||
'(--suppress-think)--suppress-think[Suppress text enclosed in thinking tags]' \
|
||||
'(--think-start-tag)--think-start-tag[Start tag for thinking sections (default: <think>)]:start tag:' \
|
||||
'(--think-end-tag)--think-end-tag[End tag for thinking sections (default: </think>)]:end tag:' \
|
||||
'(-h --help)'{-h,--help}'[Show this help message]' \
|
||||
'*:arguments:'
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ _fabric() {
|
||||
_get_comp_words_by_ref -n : cur prev words cword
|
||||
|
||||
# Define all possible options/flags
|
||||
local opts="--pattern -p --variable -v --context -C --session --attachment -a --setup -S --temperature -t --topp -T --stream -s --presencepenalty -P --raw -r --frequencypenalty -F --listpatterns -l --listmodels -L --listcontexts -x --listsessions -X --updatepatterns -U --copy -c --model -m --modelContextLength --output -o --output-session --latest -n --changeDefaultModel -d --youtube -y --playlist --transcript --transcript-with-timestamps --comments --metadata --language -g --scrape_url -u --scrape_question -q --seed -e --wipecontext -w --wipesession -W --printcontext --printsession --readability --input-has-vars --dry-run --serve --serveOllama --address --api-key --config --search --search-location --image-file --image-size --image-quality --image-compression --image-background --version --listextensions --addextension --rmextension --strategy --liststrategies --listvendors --shell-complete-list --help -h"
|
||||
local opts="--pattern -p --variable -v --context -C --session --attachment -a --setup -S --temperature -t --topp -T --stream -s --presencepenalty -P --raw -r --frequencypenalty -F --listpatterns -l --listmodels -L --listcontexts -x --listsessions -X --updatepatterns -U --copy -c --model -m --modelContextLength --output -o --output-session --latest -n --changeDefaultModel -d --youtube -y --playlist --transcript --transcript-with-timestamps --comments --metadata --language -g --scrape_url -u --scrape_question -q --seed -e --wipecontext -w --wipesession -W --printcontext --printsession --readability --input-has-vars --dry-run --serve --serveOllama --address --api-key --config --search --search-location --image-file --image-size --image-quality --image-compression --image-background --suppress-think --think-start-tag --think-end-tag --version --listextensions --addextension --rmextension --strategy --liststrategies --listvendors --shell-complete-list --help -h"
|
||||
|
||||
# Helper function for dynamic completions
|
||||
_fabric_get_list() {
|
||||
@@ -81,7 +81,7 @@ _fabric() {
|
||||
return 0
|
||||
;;
|
||||
# Options requiring simple arguments (no specific completion logic here)
|
||||
-v | --variable | -t | --temperature | -T | --topp | -P | --presencepenalty | -F | --frequencypenalty | --modelContextLength | -n | --latest | -y | --youtube | -g | --language | -u | --scrape_url | -q | --scrape_question | -e | --seed | --address | --api-key | --search-location | --image-compression)
|
||||
-v | --variable | -t | --temperature | -T | --topp | -P | --presencepenalty | -F | --frequencypenalty | --modelContextLength | -n | --latest | -y | --youtube | -g | --language | -u | --scrape_url | -q | --scrape_question | -e | --seed | --address | --api-key | --search-location | --image-compression | --think-start-tag | --think-end-tag)
|
||||
# No specific completion suggestions, user types the value
|
||||
return 0
|
||||
;;
|
||||
|
||||
@@ -69,6 +69,8 @@ complete -c fabric -l image-background -d "Background type: opaque, transparent
|
||||
complete -c fabric -l addextension -d "Register a new extension from config file path" -r -a "*.yaml *.yml"
|
||||
complete -c fabric -l rmextension -d "Remove a registered extension by name" -a "(__fabric_get_extensions)"
|
||||
complete -c fabric -l strategy -d "Choose a strategy from the available strategies" -a "(__fabric_get_strategies)"
|
||||
complete -c fabric -l think-start-tag -d "Start tag for thinking sections (default: <think>)"
|
||||
complete -c fabric -l think-end-tag -d "End tag for thinking sections (default: </think>)"
|
||||
|
||||
# Boolean flags (no arguments)
|
||||
complete -c fabric -s S -l setup -d "Run setup for all reconfigurable parts of fabric"
|
||||
@@ -98,4 +100,5 @@ complete -c fabric -l listextensions -d "List all registered extensions"
|
||||
complete -c fabric -l liststrategies -d "List all strategies"
|
||||
complete -c fabric -l listvendors -d "List all vendors"
|
||||
complete -c fabric -l shell-complete-list -d "Output raw list without headers/formatting (for shell completion)"
|
||||
complete -c fabric -l suppress-think -d "Suppress text enclosed in thinking tags"
|
||||
complete -c fabric -s h -l help -d "Show this help message"
|
||||
|
||||
@@ -7,7 +7,7 @@ Generate code changes to an existing coding project using AI.
|
||||
After installing the `code_helper` binary:
|
||||
|
||||
```bash
|
||||
go install github.com/danielmiessler/fabric/plugins/tools/code_helper@latest
|
||||
go install github.com/danielmiessler/fabric/cmd/code_helper@latest
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
8
data/patterns/generate_code_rules/system.md
Normal file
8
data/patterns/generate_code_rules/system.md
Normal file
@@ -0,0 +1,8 @@
|
||||
# IDENTITY AND PURPOSE
|
||||
|
||||
You are a senior developer and expert prompt engineer. Think ultra hard to distill the following transcription or tutorial in as little set of unique rules as possible intended for best practices guidance in AI assisted coding tools, each rule has to be in one sentence as a direct instruction, avoid explanations and cosmetic language. Output in Markdown, I prefer bullet dash (-).
|
||||
|
||||
---
|
||||
|
||||
# TRANSCRIPT
|
||||
|
||||
@@ -71,6 +71,7 @@
|
||||
default = self.packages.${system}.fabric;
|
||||
fabric = pkgs.callPackage ./nix/pkgs/fabric {
|
||||
go = goVersion;
|
||||
inherit self;
|
||||
inherit (gomod2nix.legacyPackages.${system}) buildGoApplication;
|
||||
};
|
||||
inherit (gomod2nix.legacyPackages.${system}) gomod2nix;
|
||||
|
||||
17
go.mod
17
go.mod
@@ -16,17 +16,21 @@ require (
|
||||
github.com/go-git/go-git/v5 v5.16.2
|
||||
github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612
|
||||
github.com/google/generative-ai-go v0.20.1
|
||||
github.com/google/go-github/v66 v66.0.0
|
||||
github.com/hasura/go-graphql-client v0.14.4
|
||||
github.com/jessevdk/go-flags v1.6.1
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/mattn/go-sqlite3 v1.14.28
|
||||
github.com/ollama/ollama v0.9.0
|
||||
github.com/openai/openai-go v1.8.2
|
||||
github.com/otiai10/copy v1.14.1
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/samber/lo v1.50.0
|
||||
github.com/sgaunet/perplexity-go/v2 v2.8.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.10.0
|
||||
golang.org/x/oauth2 v0.30.0
|
||||
golang.org/x/text v0.26.0
|
||||
golang.org/x/text v0.27.0
|
||||
google.golang.org/api v0.236.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
@@ -59,6 +63,7 @@ require (
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/coder/websocket v1.8.13 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
@@ -75,10 +80,12 @@ require (
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.14.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
@@ -89,10 +96,11 @@ require (
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/otiai10/mint v1.6.3 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.2 // indirect
|
||||
github.com/pjbgf/sha1cd v0.4.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/sergi/go-diff v1.4.0 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
@@ -108,9 +116,10 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.36.0 // indirect
|
||||
golang.org/x/arch v0.18.0 // indirect
|
||||
golang.org/x/crypto v0.39.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b // indirect
|
||||
golang.org/x/net v0.41.0 // indirect
|
||||
golang.org/x/sync v0.15.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/sync v0.16.0 // indirect
|
||||
golang.org/x/sys v0.34.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
|
||||
|
||||
40
go.sum
40
go.sum
@@ -71,6 +71,9 @@ github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZ
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE=
|
||||
github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
|
||||
github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -125,9 +128,14 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/generative-ai-go v0.20.1 h1:6dEIujpgN2V0PgLhr6c/M1ynRdc7ARtiIDPFzj45uNQ=
|
||||
github.com/google/generative-ai-go v0.20.1/go.mod h1:TjOnZJmZKzarWbjUJgy+r3Ee7HGBRVLhOIgupnwR4Bg=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd/D/9+M=
|
||||
github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
@@ -137,6 +145,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0=
|
||||
github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w=
|
||||
github.com/hasura/go-graphql-client v0.14.4 h1:bYU7/+V50T2YBGdNQXt6l4f2cMZPECPUd8cyCR+ixtw=
|
||||
github.com/hasura/go-graphql-client v0.14.4/go.mod h1:jfSZtBER3or+88Q9vFhWHiFMPppfYILRyl+0zsgPIIw=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
|
||||
@@ -163,6 +175,8 @@ github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjS
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
|
||||
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -180,8 +194,8 @@ github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs=
|
||||
github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4=
|
||||
github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A=
|
||||
github.com/pjbgf/sha1cd v0.4.0 h1:NXzbL1RvjTUi6kgYZCX3fPwwl27Q1LJndxtUDVfJGRY=
|
||||
github.com/pjbgf/sha1cd v0.4.0/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -189,6 +203,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/samber/lo v1.50.0 h1:XrG0xOeHs+4FQ8gJR97zDz5uOFMW7OwFWiFVzqopKgY=
|
||||
github.com/samber/lo v1.50.0/go.mod h1:RjZyNk6WSnUFRKK6EyOhsRJMqft3G+pg7dCWHQCWvsc=
|
||||
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
|
||||
@@ -199,6 +214,10 @@ github.com/sgaunet/perplexity-go/v2 v2.8.0/go.mod h1:MSks4RNuivCi0GqJyylhFdgSJFV
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
|
||||
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
|
||||
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
|
||||
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -255,8 +274,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
|
||||
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
|
||||
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
|
||||
golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b h1:QoALfVG9rhQ/M7vYDScfPdWjGL9dlsVVM5VGh7aKoAA=
|
||||
golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
@@ -283,8 +302,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
|
||||
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
|
||||
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -301,8 +320,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
||||
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -324,8 +343,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
|
||||
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
|
||||
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
|
||||
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
|
||||
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -335,6 +354,7 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0=
|
||||
google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4=
|
||||
google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78=
|
||||
|
||||
66
internal/cli/chat.go
Normal file
66
internal/cli/chat.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
"github.com/danielmiessler/fabric/internal/domain"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
)
|
||||
|
||||
// handleChatProcessing handles the main chat processing logic
|
||||
func handleChatProcessing(currentFlags *Flags, registry *core.PluginRegistry, messageTools string) (err error) {
|
||||
if messageTools != "" {
|
||||
currentFlags.AppendMessage(messageTools)
|
||||
}
|
||||
|
||||
var chatter *core.Chatter
|
||||
if chatter, err = registry.GetChatter(currentFlags.Model, currentFlags.ModelContextLength,
|
||||
currentFlags.Strategy, currentFlags.Stream, currentFlags.DryRun); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var session *fsdb.Session
|
||||
var chatReq *domain.ChatRequest
|
||||
if chatReq, err = currentFlags.BuildChatRequest(strings.Join(os.Args[1:], " ")); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if chatReq.Language == "" {
|
||||
chatReq.Language = registry.Language.DefaultLanguage.Value
|
||||
}
|
||||
var chatOptions *domain.ChatOptions
|
||||
if chatOptions, err = currentFlags.BuildChatOptions(); err != nil {
|
||||
return
|
||||
}
|
||||
if session, err = chatter.Send(chatReq, chatOptions); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
result := session.GetLastMessage().Content
|
||||
|
||||
if !currentFlags.Stream || currentFlags.SuppressThink {
|
||||
// print the result if it was not streamed already or suppress-think disabled streaming output
|
||||
fmt.Println(result)
|
||||
}
|
||||
|
||||
// if the copy flag is set, copy the message to the clipboard
|
||||
if currentFlags.Copy {
|
||||
if err = CopyToClipboard(result); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// if the output flag is set, create an output file
|
||||
if currentFlags.Output != "" {
|
||||
if currentFlags.OutputSession {
|
||||
sessionAsString := session.String()
|
||||
err = CreateOutputFile(sessionAsString, currentFlags.Output)
|
||||
} else {
|
||||
err = CreateOutputFile(result, currentFlags.Output)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -4,18 +4,11 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/tools/youtube"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
"github.com/danielmiessler/fabric/internal/domain"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/ai"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
restapi "github.com/danielmiessler/fabric/internal/server"
|
||||
"github.com/danielmiessler/fabric/internal/tools/converter"
|
||||
"github.com/danielmiessler/fabric/internal/tools/youtube"
|
||||
)
|
||||
|
||||
// Cli Controls the cli. It takes in the flags and runs the appropriate functions
|
||||
@@ -30,277 +23,67 @@ func Cli(version string) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
var homedir string
|
||||
if homedir, err = os.UserHomeDir(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fabricDb := fsdb.NewDb(filepath.Join(homedir, ".config/fabric"))
|
||||
|
||||
if err = fabricDb.Configure(); err != nil {
|
||||
// Initialize database and registry
|
||||
var registry, err2 = initializeFabric()
|
||||
if err2 != nil {
|
||||
if !currentFlags.Setup {
|
||||
println(err.Error())
|
||||
fmt.Fprintln(os.Stderr, err2.Error())
|
||||
currentFlags.Setup = true
|
||||
}
|
||||
}
|
||||
|
||||
var registry *core.PluginRegistry
|
||||
if registry, err = core.NewPluginRegistry(fabricDb); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// if the setup flag is set, run the setup function
|
||||
if currentFlags.Setup {
|
||||
err = registry.Setup()
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.Serve {
|
||||
registry.ConfigureVendors()
|
||||
err = restapi.Serve(registry, currentFlags.ServeAddress, currentFlags.ServeAPIKey)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ServeOllama {
|
||||
registry.ConfigureVendors()
|
||||
err = restapi.ServeOllama(registry, currentFlags.ServeAddress, version)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.UpdatePatterns {
|
||||
err = registry.PatternsLoader.PopulateDB()
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ChangeDefaultModel {
|
||||
if err = registry.Defaults.Setup(); err != nil {
|
||||
return
|
||||
// Return early if registry is nil to prevent panics in subsequent handlers
|
||||
if registry == nil {
|
||||
return err2
|
||||
}
|
||||
err = registry.SaveEnvFile()
|
||||
}
|
||||
|
||||
// Handle setup and server commands
|
||||
var handled bool
|
||||
if handled, err = handleSetupAndServerCommands(currentFlags, registry, version); err != nil || handled {
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.LatestPatterns != "0" {
|
||||
var parsedToInt int
|
||||
if parsedToInt, err = strconv.Atoi(currentFlags.LatestPatterns); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = fabricDb.Patterns.PrintLatestPatterns(parsedToInt); err != nil {
|
||||
return
|
||||
}
|
||||
// Handle configuration commands
|
||||
if handled, err = handleConfigurationCommands(currentFlags, registry); err != nil || handled {
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ListPatterns {
|
||||
err = fabricDb.Patterns.ListNames(currentFlags.ShellCompleteOutput)
|
||||
// Handle listing commands
|
||||
if handled, err = handleListingCommands(currentFlags, registry.Db, registry); err != nil || handled {
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ListAllModels {
|
||||
var models *ai.VendorsModels
|
||||
if models, err = registry.VendorManager.GetModels(); err != nil {
|
||||
return
|
||||
}
|
||||
models.Print(currentFlags.ShellCompleteOutput)
|
||||
// Handle management commands
|
||||
if handled, err = handleManagementCommands(currentFlags, registry.Db); err != nil || handled {
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ListAllContexts {
|
||||
err = fabricDb.Contexts.ListNames(currentFlags.ShellCompleteOutput)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ListAllSessions {
|
||||
err = fabricDb.Sessions.ListNames(currentFlags.ShellCompleteOutput)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.WipeContext != "" {
|
||||
err = fabricDb.Contexts.Delete(currentFlags.WipeContext)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.WipeSession != "" {
|
||||
err = fabricDb.Sessions.Delete(currentFlags.WipeSession)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.PrintSession != "" {
|
||||
err = fabricDb.Sessions.PrintSession(currentFlags.PrintSession)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.PrintContext != "" {
|
||||
err = fabricDb.Contexts.PrintContext(currentFlags.PrintContext)
|
||||
// Handle extension commands
|
||||
if handled, err = handleExtensionCommands(currentFlags, registry); err != nil || handled {
|
||||
return
|
||||
}
|
||||
|
||||
// Process HTML readability if needed
|
||||
if currentFlags.HtmlReadability {
|
||||
if msg, cleanErr := converter.HtmlReadability(currentFlags.Message); cleanErr != nil {
|
||||
fmt.Println("use original input, because can't apply html readability", err)
|
||||
fmt.Println("use original input, because can't apply html readability", cleanErr)
|
||||
} else {
|
||||
currentFlags.Message = msg
|
||||
}
|
||||
}
|
||||
|
||||
if currentFlags.ListExtensions {
|
||||
err = registry.TemplateExtensions.ListExtensions()
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.AddExtension != "" {
|
||||
err = registry.TemplateExtensions.RegisterExtension(currentFlags.AddExtension)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.RemoveExtension != "" {
|
||||
err = registry.TemplateExtensions.RemoveExtension(currentFlags.RemoveExtension)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ListStrategies {
|
||||
err = registry.Strategies.ListStrategies(currentFlags.ShellCompleteOutput)
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.ListVendors {
|
||||
err = registry.ListVendors(os.Stdout)
|
||||
return
|
||||
}
|
||||
|
||||
// if the interactive flag is set, run the interactive function
|
||||
// if currentFlags.Interactive {
|
||||
// interactive.Interactive()
|
||||
// }
|
||||
|
||||
// if none of the above currentFlags are set, run the initiate chat function
|
||||
|
||||
// Handle tool-based message processing
|
||||
var messageTools string
|
||||
|
||||
if currentFlags.YouTube != "" {
|
||||
if !registry.YouTube.IsConfigured() {
|
||||
err = fmt.Errorf("YouTube is not configured, please run the setup procedure")
|
||||
return
|
||||
}
|
||||
|
||||
var videoId string
|
||||
var playlistId string
|
||||
if videoId, playlistId, err = registry.YouTube.GetVideoOrPlaylistId(currentFlags.YouTube); err != nil {
|
||||
return
|
||||
} else if (videoId == "" || currentFlags.YouTubePlaylist) && playlistId != "" {
|
||||
if currentFlags.Output != "" {
|
||||
err = registry.YouTube.FetchAndSavePlaylist(playlistId, currentFlags.Output)
|
||||
} else {
|
||||
var videos []*youtube.VideoMeta
|
||||
if videos, err = registry.YouTube.FetchPlaylistVideos(playlistId); err != nil {
|
||||
err = fmt.Errorf("error fetching playlist videos: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, video := range videos {
|
||||
var message string
|
||||
if message, err = processYoutubeVideo(currentFlags, registry, video.Id); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !currentFlags.IsChatRequest() {
|
||||
if err = WriteOutput(message, fmt.Sprintf("%v.md", video.TitleNormalized)); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
messageTools = AppendMessage(messageTools, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if messageTools, err = processYoutubeVideo(currentFlags, registry, videoId); err != nil {
|
||||
return
|
||||
}
|
||||
if !currentFlags.IsChatRequest() {
|
||||
err = currentFlags.WriteOutput(messageTools)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (currentFlags.ScrapeURL != "" || currentFlags.ScrapeQuestion != "") && registry.Jina.IsConfigured() {
|
||||
// Check if the scrape_url flag is set and call ScrapeURL
|
||||
if currentFlags.ScrapeURL != "" {
|
||||
var website string
|
||||
if website, err = registry.Jina.ScrapeURL(currentFlags.ScrapeURL); err != nil {
|
||||
return
|
||||
}
|
||||
messageTools = AppendMessage(messageTools, website)
|
||||
}
|
||||
|
||||
// Check if the scrape_question flag is set and call ScrapeQuestion
|
||||
if currentFlags.ScrapeQuestion != "" {
|
||||
var website string
|
||||
if website, err = registry.Jina.ScrapeQuestion(currentFlags.ScrapeQuestion); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
messageTools = AppendMessage(messageTools, website)
|
||||
}
|
||||
|
||||
if !currentFlags.IsChatRequest() {
|
||||
err = currentFlags.WriteOutput(messageTools)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if messageTools != "" {
|
||||
currentFlags.AppendMessage(messageTools)
|
||||
}
|
||||
|
||||
var chatter *core.Chatter
|
||||
if chatter, err = registry.GetChatter(currentFlags.Model, currentFlags.ModelContextLength,
|
||||
currentFlags.Strategy, currentFlags.Stream, currentFlags.DryRun); err != nil {
|
||||
if messageTools, err = handleToolProcessing(currentFlags, registry); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var session *fsdb.Session
|
||||
var chatReq *domain.ChatRequest
|
||||
if chatReq, err = currentFlags.BuildChatRequest(strings.Join(os.Args[1:], " ")); err != nil {
|
||||
return
|
||||
// Return early for non-chat tool operations
|
||||
if messageTools != "" && !currentFlags.IsChatRequest() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if chatReq.Language == "" {
|
||||
chatReq.Language = registry.Language.DefaultLanguage.Value
|
||||
}
|
||||
var chatOptions *domain.ChatOptions
|
||||
if chatOptions, err = currentFlags.BuildChatOptions(); err != nil {
|
||||
return
|
||||
}
|
||||
if session, err = chatter.Send(chatReq, chatOptions); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
result := session.GetLastMessage().Content
|
||||
|
||||
if !currentFlags.Stream {
|
||||
// print the result if it was not streamed already
|
||||
fmt.Println(result)
|
||||
}
|
||||
|
||||
// if the copy flag is set, copy the message to the clipboard
|
||||
if currentFlags.Copy {
|
||||
if err = CopyToClipboard(result); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// if the output flag is set, create an output file
|
||||
if currentFlags.Output != "" {
|
||||
if currentFlags.OutputSession {
|
||||
sessionAsString := session.String()
|
||||
err = CreateOutputFile(sessionAsString, currentFlags.Output)
|
||||
} else {
|
||||
err = CreateOutputFile(result, currentFlags.Output)
|
||||
}
|
||||
}
|
||||
// Handle chat processing
|
||||
err = handleChatProcessing(currentFlags, registry, messageTools)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
28
internal/cli/configuration.go
Normal file
28
internal/cli/configuration.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
)
|
||||
|
||||
// handleConfigurationCommands handles configuration-related commands
|
||||
// Returns (handled, error) where handled indicates if a command was processed and should exit
|
||||
func handleConfigurationCommands(currentFlags *Flags, registry *core.PluginRegistry) (handled bool, err error) {
|
||||
if currentFlags.UpdatePatterns {
|
||||
if err = registry.PatternsLoader.PopulateDB(); err != nil {
|
||||
return true, err
|
||||
}
|
||||
// Save configuration in case any paths were migrated during pattern loading
|
||||
err = registry.SaveEnvFile()
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.ChangeDefaultModel {
|
||||
if err = registry.Defaults.Setup(); err != nil {
|
||||
return true, err
|
||||
}
|
||||
err = registry.SaveEnvFile()
|
||||
return true, err
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
@@ -18,4 +18,9 @@ temperature: 0.88
|
||||
seed: 42
|
||||
|
||||
stream: true
|
||||
raw: false
|
||||
raw: false
|
||||
|
||||
# suppress vendor thinking output
|
||||
suppressThink: false
|
||||
thinkStartTag: "<think>"
|
||||
thinkEndTag: "</think>"
|
||||
|
||||
26
internal/cli/extensions.go
Normal file
26
internal/cli/extensions.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
)
|
||||
|
||||
// handleExtensionCommands handles extension-related commands
|
||||
// Returns (handled, error) where handled indicates if a command was processed and should exit
|
||||
func handleExtensionCommands(currentFlags *Flags, registry *core.PluginRegistry) (handled bool, err error) {
|
||||
if currentFlags.ListExtensions {
|
||||
err = registry.TemplateExtensions.ListExtensions()
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.AddExtension != "" {
|
||||
err = registry.TemplateExtensions.RegisterExtension(currentFlags.AddExtension)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.RemoveExtension != "" {
|
||||
err = registry.TemplateExtensions.RemoveExtension(currentFlags.RemoveExtension)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
@@ -83,6 +83,9 @@ type Flags struct {
|
||||
ImageQuality string `long:"image-quality" description:"Image quality: low, medium, high, auto (default: auto)"`
|
||||
ImageCompression int `long:"image-compression" description:"Compression level 0-100 for JPEG/WebP formats (default: not set)"`
|
||||
ImageBackground string `long:"image-background" description:"Background type: opaque, transparent (default: opaque, only for PNG/WebP)"`
|
||||
SuppressThink bool `long:"suppress-think" yaml:"suppressThink" description:"Suppress text enclosed in thinking tags"`
|
||||
ThinkStartTag string `long:"think-start-tag" yaml:"thinkStartTag" description:"Start tag for thinking sections" default:"<think>"`
|
||||
ThinkEndTag string `long:"think-end-tag" yaml:"thinkEndTag" description:"End tag for thinking sections" default:"</think>"`
|
||||
}
|
||||
|
||||
var debug = false
|
||||
@@ -99,26 +102,34 @@ func Init() (ret *Flags, err error) {
|
||||
usedFlags := make(map[string]bool)
|
||||
yamlArgsScan := os.Args[1:]
|
||||
|
||||
// Get list of fields that have yaml tags, could be in yaml config
|
||||
yamlFields := make(map[string]bool)
|
||||
// Create mapping from flag names (both short and long) to yaml tag names
|
||||
flagToYamlTag := make(map[string]string)
|
||||
t := reflect.TypeOf(Flags{})
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
if yamlTag := t.Field(i).Tag.Get("yaml"); yamlTag != "" {
|
||||
yamlFields[yamlTag] = true
|
||||
//Debugf("Found yaml-configured field: %s\n", yamlTag)
|
||||
field := t.Field(i)
|
||||
yamlTag := field.Tag.Get("yaml")
|
||||
if yamlTag != "" {
|
||||
longTag := field.Tag.Get("long")
|
||||
shortTag := field.Tag.Get("short")
|
||||
if longTag != "" {
|
||||
flagToYamlTag[longTag] = yamlTag
|
||||
Debugf("Mapped long flag %s to yaml tag %s\n", longTag, yamlTag)
|
||||
}
|
||||
if shortTag != "" {
|
||||
flagToYamlTag[shortTag] = yamlTag
|
||||
Debugf("Mapped short flag %s to yaml tag %s\n", shortTag, yamlTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scan args for that are provided by cli and might be in yaml
|
||||
for _, arg := range yamlArgsScan {
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
flag := strings.TrimPrefix(arg, "--")
|
||||
if i := strings.Index(flag, "="); i > 0 {
|
||||
flag = flag[:i]
|
||||
}
|
||||
if yamlFields[flag] {
|
||||
usedFlags[flag] = true
|
||||
Debugf("CLI flag used: %s\n", flag)
|
||||
flag := extractFlag(arg)
|
||||
|
||||
if flag != "" {
|
||||
if yamlTag, exists := flagToYamlTag[flag]; exists {
|
||||
usedFlags[yamlTag] = true
|
||||
Debugf("CLI flag used: %s (yaml: %s)\n", flag, yamlTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -131,6 +142,16 @@ func Init() (ret *Flags, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check to see if a ~/.fabric.yaml config file exists (only when user didn't specify a config)
|
||||
if ret.Config == "" {
|
||||
// Default to ~/.fabric.yaml if no config specified
|
||||
if defaultConfigPath, err := util.GetDefaultConfigPath(); err == nil && defaultConfigPath != "" {
|
||||
ret.Config = defaultConfigPath
|
||||
} else if err != nil {
|
||||
Debugf("Could not determine default config path: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// If config specified, load and apply YAML for unused flags
|
||||
if ret.Config != "" {
|
||||
var yamlFlags *Flags
|
||||
@@ -165,7 +186,6 @@ func Init() (ret *Flags, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Handle stdin and messages
|
||||
// Handle stdin and messages
|
||||
info, _ := os.Stdin.Stat()
|
||||
pipedToStdin := (info.Mode() & os.ModeCharDevice) == 0
|
||||
@@ -185,6 +205,22 @@ func Init() (ret *Flags, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func extractFlag(arg string) string {
|
||||
var flag string
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
flag = strings.TrimPrefix(arg, "--")
|
||||
if i := strings.Index(flag, "="); i > 0 {
|
||||
flag = flag[:i]
|
||||
}
|
||||
} else if strings.HasPrefix(arg, "-") && len(arg) > 1 {
|
||||
flag = strings.TrimPrefix(arg, "-")
|
||||
if i := strings.Index(flag, "="); i > 0 {
|
||||
flag = flag[:i]
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
func assignWithConversion(targetField, sourceField reflect.Value) error {
|
||||
// Handle string source values
|
||||
if sourceField.Kind() == reflect.String {
|
||||
@@ -376,6 +412,15 @@ func (o *Flags) BuildChatOptions() (ret *domain.ChatOptions, err error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
startTag := o.ThinkStartTag
|
||||
if startTag == "" {
|
||||
startTag = "<think>"
|
||||
}
|
||||
endTag := o.ThinkEndTag
|
||||
if endTag == "" {
|
||||
endTag = "</think>"
|
||||
}
|
||||
|
||||
ret = &domain.ChatOptions{
|
||||
Model: o.Model,
|
||||
Temperature: o.Temperature,
|
||||
@@ -392,6 +437,9 @@ func (o *Flags) BuildChatOptions() (ret *domain.ChatOptions, err error) {
|
||||
ImageQuality: o.ImageQuality,
|
||||
ImageCompression: o.ImageCompression,
|
||||
ImageBackground: o.ImageBackground,
|
||||
SuppressThink: o.SuppressThink,
|
||||
ThinkStartTag: startTag,
|
||||
ThinkEndTag: endTag,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -64,6 +64,9 @@ func TestBuildChatOptions(t *testing.T) {
|
||||
FrequencyPenalty: 0.2,
|
||||
Raw: false,
|
||||
Seed: 1,
|
||||
SuppressThink: false,
|
||||
ThinkStartTag: "<think>",
|
||||
ThinkEndTag: "</think>",
|
||||
}
|
||||
options, err := flags.BuildChatOptions()
|
||||
assert.NoError(t, err)
|
||||
@@ -85,12 +88,29 @@ func TestBuildChatOptionsDefaultSeed(t *testing.T) {
|
||||
FrequencyPenalty: 0.2,
|
||||
Raw: false,
|
||||
Seed: 0,
|
||||
SuppressThink: false,
|
||||
ThinkStartTag: "<think>",
|
||||
ThinkEndTag: "</think>",
|
||||
}
|
||||
options, err := flags.BuildChatOptions()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expectedOptions, options)
|
||||
}
|
||||
|
||||
func TestBuildChatOptionsSuppressThink(t *testing.T) {
|
||||
flags := &Flags{
|
||||
SuppressThink: true,
|
||||
ThinkStartTag: "[[t]]",
|
||||
ThinkEndTag: "[[/t]]",
|
||||
}
|
||||
|
||||
options, err := flags.BuildChatOptions()
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, options.SuppressThink)
|
||||
assert.Equal(t, "[[t]]", options.ThinkStartTag)
|
||||
assert.Equal(t, "[[/t]]", options.ThinkEndTag)
|
||||
}
|
||||
|
||||
func TestInitWithYAMLConfig(t *testing.T) {
|
||||
// Create a temporary YAML config file
|
||||
configContent := `
|
||||
|
||||
28
internal/cli/initialization.go
Normal file
28
internal/cli/initialization.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
)
|
||||
|
||||
// initializeFabric initializes the fabric database and plugin registry
|
||||
func initializeFabric() (registry *core.PluginRegistry, err error) {
|
||||
var homedir string
|
||||
if homedir, err = os.UserHomeDir(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fabricDb := fsdb.NewDb(filepath.Join(homedir, ".config/fabric"))
|
||||
if err = fabricDb.Configure(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if registry, err = core.NewPluginRegistry(fabricDb); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
62
internal/cli/listing.go
Normal file
62
internal/cli/listing.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/ai"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
)
|
||||
|
||||
// handleListingCommands handles listing-related commands
|
||||
// Returns (handled, error) where handled indicates if a command was processed and should exit
|
||||
func handleListingCommands(currentFlags *Flags, fabricDb *fsdb.Db, registry *core.PluginRegistry) (handled bool, err error) {
|
||||
if currentFlags.LatestPatterns != "0" {
|
||||
var parsedToInt int
|
||||
if parsedToInt, err = strconv.Atoi(currentFlags.LatestPatterns); err != nil {
|
||||
return true, err
|
||||
}
|
||||
|
||||
if err = fabricDb.Patterns.PrintLatestPatterns(parsedToInt); err != nil {
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if currentFlags.ListPatterns {
|
||||
err = fabricDb.Patterns.ListNames(currentFlags.ShellCompleteOutput)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.ListAllModels {
|
||||
var models *ai.VendorsModels
|
||||
if models, err = registry.VendorManager.GetModels(); err != nil {
|
||||
return true, err
|
||||
}
|
||||
models.Print(currentFlags.ShellCompleteOutput)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if currentFlags.ListAllContexts {
|
||||
err = fabricDb.Contexts.ListNames(currentFlags.ShellCompleteOutput)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.ListAllSessions {
|
||||
err = fabricDb.Sessions.ListNames(currentFlags.ShellCompleteOutput)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.ListStrategies {
|
||||
err = registry.Strategies.ListStrategies(currentFlags.ShellCompleteOutput)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.ListVendors {
|
||||
err = registry.ListVendors(os.Stdout)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
31
internal/cli/management.go
Normal file
31
internal/cli/management.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
)
|
||||
|
||||
// handleManagementCommands handles management-related commands (delete, print, etc.)
|
||||
// Returns (handled, error) where handled indicates if a command was processed and should exit
|
||||
func handleManagementCommands(currentFlags *Flags, fabricDb *fsdb.Db) (handled bool, err error) {
|
||||
if currentFlags.WipeContext != "" {
|
||||
err = fabricDb.Contexts.Delete(currentFlags.WipeContext)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.WipeSession != "" {
|
||||
err = fabricDb.Sessions.Delete(currentFlags.WipeSession)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.PrintSession != "" {
|
||||
err = fabricDb.Sessions.PrintSession(currentFlags.PrintSession)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.PrintContext != "" {
|
||||
err = fabricDb.Contexts.PrintContext(currentFlags.PrintContext)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
30
internal/cli/setup_server.go
Normal file
30
internal/cli/setup_server.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
restapi "github.com/danielmiessler/fabric/internal/server"
|
||||
)
|
||||
|
||||
// handleSetupAndServerCommands handles setup and server-related commands
|
||||
// Returns (handled, error) where handled indicates if a command was processed and should exit
|
||||
func handleSetupAndServerCommands(currentFlags *Flags, registry *core.PluginRegistry, version string) (handled bool, err error) {
|
||||
// if the setup flag is set, run the setup function
|
||||
if currentFlags.Setup {
|
||||
err = registry.Setup()
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.Serve {
|
||||
registry.ConfigureVendors()
|
||||
err = restapi.Serve(registry, currentFlags.ServeAddress, currentFlags.ServeAPIKey)
|
||||
return true, err
|
||||
}
|
||||
|
||||
if currentFlags.ServeOllama {
|
||||
registry.ConfigureVendors()
|
||||
err = restapi.ServeOllama(registry, currentFlags.ServeAddress, version)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
90
internal/cli/tools.go
Normal file
90
internal/cli/tools.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
"github.com/danielmiessler/fabric/internal/tools/youtube"
|
||||
)
|
||||
|
||||
// handleToolProcessing handles YouTube and web scraping tool processing
|
||||
func handleToolProcessing(currentFlags *Flags, registry *core.PluginRegistry) (messageTools string, err error) {
|
||||
if currentFlags.YouTube != "" {
|
||||
if !registry.YouTube.IsConfigured() {
|
||||
err = fmt.Errorf("YouTube is not configured, please run the setup procedure")
|
||||
return
|
||||
}
|
||||
|
||||
var videoId string
|
||||
var playlistId string
|
||||
if videoId, playlistId, err = registry.YouTube.GetVideoOrPlaylistId(currentFlags.YouTube); err != nil {
|
||||
return
|
||||
} else if (videoId == "" || currentFlags.YouTubePlaylist) && playlistId != "" {
|
||||
if currentFlags.Output != "" {
|
||||
err = registry.YouTube.FetchAndSavePlaylist(playlistId, currentFlags.Output)
|
||||
} else {
|
||||
var videos []*youtube.VideoMeta
|
||||
if videos, err = registry.YouTube.FetchPlaylistVideos(playlistId); err != nil {
|
||||
err = fmt.Errorf("error fetching playlist videos: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, video := range videos {
|
||||
var message string
|
||||
if message, err = processYoutubeVideo(currentFlags, registry, video.Id); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !currentFlags.IsChatRequest() {
|
||||
if err = WriteOutput(message, fmt.Sprintf("%v.md", video.TitleNormalized)); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
messageTools = AppendMessage(messageTools, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if messageTools, err = processYoutubeVideo(currentFlags, registry, videoId); err != nil {
|
||||
return
|
||||
}
|
||||
if !currentFlags.IsChatRequest() {
|
||||
err = currentFlags.WriteOutput(messageTools)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if currentFlags.ScrapeURL != "" || currentFlags.ScrapeQuestion != "" {
|
||||
if !registry.Jina.IsConfigured() {
|
||||
err = fmt.Errorf("scraping functionality is not configured. Please set up Jina to enable scraping")
|
||||
return
|
||||
}
|
||||
// Check if the scrape_url flag is set and call ScrapeURL
|
||||
if currentFlags.ScrapeURL != "" {
|
||||
var website string
|
||||
if website, err = registry.Jina.ScrapeURL(currentFlags.ScrapeURL); err != nil {
|
||||
return
|
||||
}
|
||||
messageTools = AppendMessage(messageTools, website)
|
||||
}
|
||||
|
||||
// Check if the scrape_question flag is set and call ScrapeQuestion
|
||||
if currentFlags.ScrapeQuestion != "" {
|
||||
var website string
|
||||
if website, err = registry.Jina.ScrapeQuestion(currentFlags.ScrapeQuestion); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
messageTools = AppendMessage(messageTools, website)
|
||||
}
|
||||
|
||||
if !currentFlags.IsChatRequest() {
|
||||
err = currentFlags.WriteOutput(messageTools)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -79,7 +79,9 @@ func (o *Chatter) Send(request *domain.ChatRequest, opts *domain.ChatOptions) (s
|
||||
|
||||
for response := range responseChan {
|
||||
message += response
|
||||
fmt.Print(response)
|
||||
if !opts.SuppressThink {
|
||||
fmt.Print(response)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for goroutine to finish
|
||||
@@ -101,6 +103,10 @@ func (o *Chatter) Send(request *domain.ChatRequest, opts *domain.ChatOptions) (s
|
||||
}
|
||||
}
|
||||
|
||||
if opts.SuppressThink && !o.DryRun {
|
||||
message = domain.StripThinkBlocks(message, opts.ThinkStartTag, opts.ThinkEndTag)
|
||||
}
|
||||
|
||||
if message == "" {
|
||||
session = nil
|
||||
err = fmt.Errorf("empty response")
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
type mockVendor struct {
|
||||
sendStreamError error
|
||||
streamChunks []string
|
||||
sendFunc func(context.Context, []*chat.ChatCompletionMessage, *domain.ChatOptions) (string, error)
|
||||
}
|
||||
|
||||
func (m *mockVendor) GetName() string {
|
||||
@@ -57,6 +58,9 @@ func (m *mockVendor) SendStream(messages []*chat.ChatCompletionMessage, opts *do
|
||||
}
|
||||
|
||||
func (m *mockVendor) Send(ctx context.Context, messages []*chat.ChatCompletionMessage, opts *domain.ChatOptions) (string, error) {
|
||||
if m.sendFunc != nil {
|
||||
return m.sendFunc(ctx, messages, opts)
|
||||
}
|
||||
return "test response", nil
|
||||
}
|
||||
|
||||
@@ -64,6 +68,51 @@ func (m *mockVendor) NeedsRawMode(modelName string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func TestChatter_Send_SuppressThink(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
db := fsdb.NewDb(tempDir)
|
||||
|
||||
mockVendor := &mockVendor{}
|
||||
|
||||
chatter := &Chatter{
|
||||
db: db,
|
||||
Stream: false,
|
||||
vendor: mockVendor,
|
||||
model: "test-model",
|
||||
}
|
||||
|
||||
request := &domain.ChatRequest{
|
||||
Message: &chat.ChatCompletionMessage{
|
||||
Role: chat.ChatMessageRoleUser,
|
||||
Content: "test",
|
||||
},
|
||||
}
|
||||
|
||||
opts := &domain.ChatOptions{
|
||||
Model: "test-model",
|
||||
SuppressThink: true,
|
||||
ThinkStartTag: "<think>",
|
||||
ThinkEndTag: "</think>",
|
||||
}
|
||||
|
||||
// custom send function returning a message with think tags
|
||||
mockVendor.sendFunc = func(ctx context.Context, msgs []*chat.ChatCompletionMessage, o *domain.ChatOptions) (string, error) {
|
||||
return "<think>hidden</think> visible", nil
|
||||
}
|
||||
|
||||
session, err := chatter.Send(request, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Send returned error: %v", err)
|
||||
}
|
||||
if session == nil {
|
||||
t.Fatal("expected session")
|
||||
}
|
||||
last := session.GetLastMessage()
|
||||
if last.Content != "visible" {
|
||||
t.Errorf("expected filtered content 'visible', got %q", last.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatter_Send_StreamingErrorPropagation(t *testing.T) {
|
||||
// Create a temporary database for testing
|
||||
tempDir := t.TempDir()
|
||||
|
||||
@@ -259,8 +259,24 @@ func (o *PluginRegistry) GetModels() (ret *ai.VendorsModels, err error) {
|
||||
func (o *PluginRegistry) Configure() (err error) {
|
||||
o.ConfigureVendors()
|
||||
_ = o.Defaults.Configure()
|
||||
if err := o.CustomPatterns.Configure(); err != nil {
|
||||
return fmt.Errorf("error configuring CustomPatterns: %w", err)
|
||||
}
|
||||
_ = o.PatternsLoader.Configure()
|
||||
|
||||
// Refresh the database custom patterns directory after custom patterns plugin is configured
|
||||
customPatternsDir := os.Getenv("CUSTOM_PATTERNS_DIRECTORY")
|
||||
if customPatternsDir != "" {
|
||||
// Expand home directory if needed
|
||||
if strings.HasPrefix(customPatternsDir, "~/") {
|
||||
if homeDir, err := os.UserHomeDir(); err == nil {
|
||||
customPatternsDir = filepath.Join(homeDir, customPatternsDir[2:])
|
||||
}
|
||||
}
|
||||
o.Db.Patterns.CustomPatternsDir = customPatternsDir
|
||||
o.PatternsLoader.Patterns.CustomPatternsDir = customPatternsDir
|
||||
}
|
||||
|
||||
//YouTube and Jina are not mandatory, so ignore not configured error
|
||||
_ = o.YouTube.Configure()
|
||||
_ = o.Jina.Configure()
|
||||
|
||||
@@ -33,6 +33,9 @@ type ChatOptions struct {
|
||||
ImageQuality string
|
||||
ImageCompression int
|
||||
ImageBackground string
|
||||
SuppressThink bool
|
||||
ThinkStartTag string
|
||||
ThinkEndTag string
|
||||
}
|
||||
|
||||
// NormalizeMessages remove empty messages and ensure messages order user-assist-user
|
||||
|
||||
32
internal/domain/think.go
Normal file
32
internal/domain/think.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// StripThinkBlocks removes any content between the provided start and end tags
|
||||
// from the input string. Whitespace following the end tag is also removed so
|
||||
// output resumes at the next non-empty line.
|
||||
var (
|
||||
regexCache = make(map[string]*regexp.Regexp)
|
||||
cacheMutex sync.Mutex
|
||||
)
|
||||
|
||||
func StripThinkBlocks(input, startTag, endTag string) string {
|
||||
if startTag == "" || endTag == "" {
|
||||
return input
|
||||
}
|
||||
|
||||
cacheKey := startTag + "|" + endTag
|
||||
cacheMutex.Lock()
|
||||
re, exists := regexCache[cacheKey]
|
||||
if !exists {
|
||||
pattern := "(?s)" + regexp.QuoteMeta(startTag) + ".*?" + regexp.QuoteMeta(endTag) + "\\s*"
|
||||
re = regexp.MustCompile(pattern)
|
||||
regexCache[cacheKey] = re
|
||||
}
|
||||
cacheMutex.Unlock()
|
||||
|
||||
return re.ReplaceAllString(input, "")
|
||||
}
|
||||
19
internal/domain/think_test.go
Normal file
19
internal/domain/think_test.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package domain
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestStripThinkBlocks(t *testing.T) {
|
||||
input := "<think>internal</think>\n\nresult"
|
||||
got := StripThinkBlocks(input, "<think>", "</think>")
|
||||
if got != "result" {
|
||||
t.Errorf("expected %q, got %q", "result", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripThinkBlocksCustomTags(t *testing.T) {
|
||||
input := "[[t]]hidden[[/t]] visible"
|
||||
got := StripThinkBlocks(input, "[[t]]", "[[/t]]")
|
||||
if got != "visible" {
|
||||
t.Errorf("expected %q, got %q", "visible", got)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -71,7 +72,7 @@ func (t *OAuthTransport) getValidToken(tokenIdentifier string) (string, error) {
|
||||
}
|
||||
// If no token exists, run OAuth flow
|
||||
if token == nil {
|
||||
fmt.Println("No OAuth token found, initiating authentication...")
|
||||
fmt.Fprintln(os.Stderr, "No OAuth token found, initiating authentication...")
|
||||
newAccessToken, err := RunOAuthFlow(tokenIdentifier)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to authenticate: %w", err)
|
||||
@@ -81,11 +82,11 @@ func (t *OAuthTransport) getValidToken(tokenIdentifier string) (string, error) {
|
||||
|
||||
// Check if token needs refresh (5 minute buffer)
|
||||
if token.IsExpired(5) {
|
||||
fmt.Println("OAuth token expired, refreshing...")
|
||||
fmt.Fprintln(os.Stderr, "OAuth token expired, refreshing...")
|
||||
newAccessToken, err := RefreshToken(tokenIdentifier)
|
||||
if err != nil {
|
||||
// If refresh fails, try re-authentication
|
||||
fmt.Println("Token refresh failed, re-authenticating...")
|
||||
fmt.Fprintln(os.Stderr, "Token refresh failed, re-authenticating...")
|
||||
newAccessToken, err = RunOAuthFlow(tokenIdentifier)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to refresh or re-authenticate: %w", err)
|
||||
@@ -137,13 +138,13 @@ func RunOAuthFlow(tokenIdentifier string) (token string, err error) {
|
||||
if err == nil && existingToken != nil {
|
||||
// If token exists but is expired, try refreshing first
|
||||
if existingToken.IsExpired(5) {
|
||||
fmt.Println("Found expired OAuth token, attempting refresh...")
|
||||
fmt.Fprintln(os.Stderr, "Found expired OAuth token, attempting refresh...")
|
||||
refreshedToken, refreshErr := RefreshToken(tokenIdentifier)
|
||||
if refreshErr == nil {
|
||||
fmt.Println("Token refresh successful")
|
||||
fmt.Fprintln(os.Stderr, "Token refresh successful")
|
||||
return refreshedToken, nil
|
||||
}
|
||||
fmt.Printf("Token refresh failed (%v), proceeding with full OAuth flow...\n", refreshErr)
|
||||
fmt.Fprintf(os.Stderr, "Token refresh failed (%v), proceeding with full OAuth flow...\n", refreshErr)
|
||||
} else {
|
||||
// Token exists and is still valid
|
||||
return existingToken.AccessToken, nil
|
||||
@@ -170,10 +171,10 @@ func RunOAuthFlow(tokenIdentifier string) (token string, err error) {
|
||||
oauth2.SetAuthURLParam("state", verifier),
|
||||
)
|
||||
|
||||
fmt.Println("Open the following URL in your browser. Fabric would like to authorize:")
|
||||
fmt.Println(authURL)
|
||||
fmt.Fprintln(os.Stderr, "Open the following URL in your browser. Fabric would like to authorize:")
|
||||
fmt.Fprintln(os.Stderr, authURL)
|
||||
openBrowser(authURL)
|
||||
fmt.Print("Paste the authorization code here: ")
|
||||
fmt.Fprint(os.Stderr, "Paste the authorization code here: ")
|
||||
var code string
|
||||
fmt.Scanln(&code)
|
||||
parts := strings.SplitN(code, "#", 2)
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"github.com/danielmiessler/fabric/internal/plugins"
|
||||
)
|
||||
|
||||
const DryRunResponse = "Dry run: Fake response sent by DryRun plugin\n"
|
||||
|
||||
type Client struct {
|
||||
*plugins.PluginBase
|
||||
}
|
||||
@@ -85,27 +87,37 @@ func (c *Client) formatOptions(opts *domain.ChatOptions) string {
|
||||
if opts.ImageFile != "" {
|
||||
builder.WriteString(fmt.Sprintf("ImageFile: %s\n", opts.ImageFile))
|
||||
}
|
||||
if opts.SuppressThink {
|
||||
builder.WriteString("SuppressThink: enabled\n")
|
||||
builder.WriteString(fmt.Sprintf("Thinking Start Tag: %s\n", opts.ThinkStartTag))
|
||||
builder.WriteString(fmt.Sprintf("Thinking End Tag: %s\n", opts.ThinkEndTag))
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func (c *Client) SendStream(msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions, channel chan string) error {
|
||||
func (c *Client) constructRequest(msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString("Dry run: Would send the following request:\n\n")
|
||||
builder.WriteString(c.formatMessages(msgs))
|
||||
builder.WriteString(c.formatOptions(opts))
|
||||
|
||||
channel <- builder.String()
|
||||
close(channel)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func (c *Client) SendStream(msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions, channel chan string) error {
|
||||
defer close(channel)
|
||||
request := c.constructRequest(msgs, opts)
|
||||
channel <- request
|
||||
channel <- "\n"
|
||||
channel <- DryRunResponse
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) Send(_ context.Context, msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions) (string, error) {
|
||||
fmt.Println("Dry run: Would send the following request:")
|
||||
fmt.Print(c.formatMessages(msgs))
|
||||
fmt.Print(c.formatOptions(opts))
|
||||
request := c.constructRequest(msgs, opts)
|
||||
|
||||
return "", nil
|
||||
return request + "\n" + DryRunResponse, nil
|
||||
}
|
||||
|
||||
func (c *Client) Setup() error {
|
||||
|
||||
105
internal/plugins/ai/openai_compatible/direct_models_call.go
Normal file
105
internal/plugins/ai/openai_compatible/direct_models_call.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package openai_compatible
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Model represents a model returned by the API
|
||||
type Model struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
// ErrorResponseLimit defines the maximum length of error response bodies for truncation.
|
||||
const errorResponseLimit = 1024 // Limit for error response body size
|
||||
|
||||
// DirectlyGetModels is used to fetch models directly from the API
|
||||
// when the standard OpenAI SDK method fails due to a nonstandard format.
|
||||
// This is useful for providers like Together that return a direct array of models.
|
||||
func (c *Client) DirectlyGetModels(ctx context.Context) ([]string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
baseURL := c.ApiBaseURL.Value
|
||||
if baseURL == "" {
|
||||
return nil, fmt.Errorf("API base URL not configured for provider %s", c.GetName())
|
||||
}
|
||||
|
||||
// Build the /models endpoint URL
|
||||
fullURL, err := url.JoinPath(baseURL, "models")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create models URL: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fullURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.ApiKey.Value))
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TODO: Consider reusing a single http.Client instance (e.g., as a field on Client) instead of allocating a new one for each request.
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
// Read the response body for debugging
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
bodyString := string(bodyBytes)
|
||||
if len(bodyString) > errorResponseLimit { // Truncate if too large
|
||||
bodyString = bodyString[:errorResponseLimit] + "..."
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected status code: %d from provider %s, response body: %s",
|
||||
resp.StatusCode, c.GetName(), bodyString)
|
||||
}
|
||||
|
||||
// Read the response body once
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Try to parse as an object with data field (OpenAI format)
|
||||
var openAIFormat struct {
|
||||
Data []Model `json:"data"`
|
||||
}
|
||||
// Try to parse as a direct array (Together format)
|
||||
var directArray []Model
|
||||
|
||||
if err := json.Unmarshal(bodyBytes, &openAIFormat); err == nil && len(openAIFormat.Data) > 0 {
|
||||
return extractModelIDs(openAIFormat.Data), nil
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(bodyBytes, &directArray); err == nil && len(directArray) > 0 {
|
||||
return extractModelIDs(directArray), nil
|
||||
}
|
||||
|
||||
var truncatedBody string
|
||||
if len(bodyBytes) > errorResponseLimit {
|
||||
truncatedBody = string(bodyBytes[:errorResponseLimit]) + "..."
|
||||
} else {
|
||||
truncatedBody = string(bodyBytes)
|
||||
}
|
||||
return nil, fmt.Errorf("unable to parse models response; raw response: %s", truncatedBody)
|
||||
}
|
||||
|
||||
func extractModelIDs(models []Model) []string {
|
||||
modelIDs := make([]string, 0, len(models))
|
||||
for _, model := range models {
|
||||
modelIDs = append(modelIDs, model.ID)
|
||||
}
|
||||
return modelIDs
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package openai_compatible
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
@@ -31,6 +32,19 @@ func NewClient(providerConfig ProviderConfig) *Client {
|
||||
return client
|
||||
}
|
||||
|
||||
// ListModels overrides the default ListModels to handle different response formats
|
||||
func (c *Client) ListModels() ([]string, error) {
|
||||
// First try the standard OpenAI SDK approach
|
||||
models, err := c.Client.ListModels()
|
||||
if err == nil && len(models) > 0 { // only return if OpenAI SDK returns models
|
||||
return models, nil
|
||||
}
|
||||
|
||||
// TODO: Handle context properly in Fabric by accepting and propagating a context.Context
|
||||
// instead of creating a new one here.
|
||||
return c.DirectlyGetModels(context.Background())
|
||||
}
|
||||
|
||||
// ProviderMap is a map of provider name to ProviderConfig for O(1) lookup
|
||||
var ProviderMap = map[string]ProviderConfig{
|
||||
"AIML": {
|
||||
@@ -83,6 +97,11 @@ var ProviderMap = map[string]ProviderConfig{
|
||||
BaseURL: "https://api.siliconflow.cn/v1",
|
||||
ImplementsResponses: false,
|
||||
},
|
||||
"Together": {
|
||||
Name: "Together",
|
||||
BaseURL: "https://api.together.xyz/v1",
|
||||
ImplementsResponses: false,
|
||||
},
|
||||
}
|
||||
|
||||
// GetProviderByName returns the provider configuration for a given name with O(1) lookup
|
||||
|
||||
@@ -86,9 +86,10 @@ func (o *Session) String() (ret string) {
|
||||
ret += fmt.Sprintf("\n--- \n[%v]\n%v", message.Role, message.Content)
|
||||
if message.MultiContent != nil {
|
||||
for _, part := range message.MultiContent {
|
||||
if part.Type == chat.ChatMessagePartTypeImageURL {
|
||||
switch part.Type {
|
||||
case chat.ChatMessagePartTypeImageURL:
|
||||
ret += fmt.Sprintf("\n%v: %v", part.Type, *part.ImageURL)
|
||||
} else if part.Type == chat.ChatMessagePartTypeText {
|
||||
case chat.ChatMessagePartTypeText:
|
||||
ret += fmt.Sprintf("\n%v: %v", part.Type, part.Text)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/plugins"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
@@ -55,7 +57,12 @@ type PatternsLoader struct {
|
||||
|
||||
func (o *PatternsLoader) configure() (err error) {
|
||||
o.pathPatternsPrefix = fmt.Sprintf("%v/", o.DefaultFolder.Value)
|
||||
o.tempPatternsFolder = filepath.Join(os.TempDir(), o.DefaultFolder.Value)
|
||||
// Use a consistent temp folder name regardless of the source path structure
|
||||
tempDir, err := os.MkdirTemp("", "fabric-patterns-")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temporary patterns folder: %w", err)
|
||||
}
|
||||
o.tempPatternsFolder = tempDir
|
||||
|
||||
return
|
||||
}
|
||||
@@ -85,18 +92,44 @@ func (o *PatternsLoader) Setup() (err error) {
|
||||
func (o *PatternsLoader) PopulateDB() (err error) {
|
||||
fmt.Printf("Downloading patterns and Populating %s...\n", o.Patterns.Dir)
|
||||
fmt.Println()
|
||||
|
||||
originalPath := o.DefaultFolder.Value
|
||||
if err = o.gitCloneAndCopy(); err != nil {
|
||||
return
|
||||
return fmt.Errorf("failed to download patterns from git repository: %w", err)
|
||||
}
|
||||
|
||||
// If the path was migrated during gitCloneAndCopy, we need to save the updated configuration
|
||||
if o.DefaultFolder.Value != originalPath {
|
||||
fmt.Printf("💾 Saving updated configuration (path changed from '%s' to '%s')...\n", originalPath, o.DefaultFolder.Value)
|
||||
// The configuration will be saved by the calling code after this returns successfully
|
||||
}
|
||||
|
||||
if err = o.movePatterns(); err != nil {
|
||||
return
|
||||
return fmt.Errorf("failed to move patterns to config directory: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("✅ Successfully downloaded and installed patterns to %s\n", o.Patterns.Dir)
|
||||
|
||||
// Create the unique patterns file after patterns are successfully moved
|
||||
if err = o.createUniquePatternsFile(); err != nil {
|
||||
return fmt.Errorf("failed to create unique patterns file: %w", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// PersistPatterns copies custom patterns to the updated patterns directory
|
||||
func (o *PatternsLoader) PersistPatterns() (err error) {
|
||||
// Check if patterns directory exists, if not, nothing to persist
|
||||
if _, err = os.Stat(o.Patterns.Dir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// No existing patterns directory, nothing to persist
|
||||
return nil
|
||||
}
|
||||
// Return unexpected errors (e.g., permission issues)
|
||||
return fmt.Errorf("failed to access patterns directory '%s': %w", o.Patterns.Dir, err)
|
||||
}
|
||||
|
||||
var currentPatterns []os.DirEntry
|
||||
if currentPatterns, err = os.ReadDir(o.Patterns.Dir); err != nil {
|
||||
return
|
||||
@@ -108,15 +141,28 @@ func (o *PatternsLoader) PersistPatterns() (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
for _, currentPattern := range currentPatterns {
|
||||
for _, newPattern := range newPatterns {
|
||||
if currentPattern.Name() == newPattern.Name() {
|
||||
break
|
||||
}
|
||||
err = copy.Copy(filepath.Join(o.Patterns.Dir, newPattern.Name()), filepath.Join(newPatternsFolder, newPattern.Name()))
|
||||
// Create a map of new patterns for faster lookup
|
||||
newPatternNames := make(map[string]bool)
|
||||
for _, newPattern := range newPatterns {
|
||||
if newPattern.IsDir() {
|
||||
newPatternNames[newPattern.Name()] = true
|
||||
}
|
||||
}
|
||||
return
|
||||
|
||||
// Copy custom patterns that don't exist in the new download
|
||||
for _, currentPattern := range currentPatterns {
|
||||
if currentPattern.IsDir() && !newPatternNames[currentPattern.Name()] {
|
||||
// This is a custom pattern, preserve it
|
||||
src := filepath.Join(o.Patterns.Dir, currentPattern.Name())
|
||||
dst := filepath.Join(newPatternsFolder, currentPattern.Name())
|
||||
if copyErr := copy.Copy(src, dst); copyErr != nil {
|
||||
fmt.Printf("Warning: failed to preserve custom pattern '%s': %v\n", currentPattern.Name(), copyErr)
|
||||
} else {
|
||||
fmt.Printf("Preserved custom pattern: %s\n", currentPattern.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// movePatterns copies the new patterns into the config directory
|
||||
@@ -134,8 +180,29 @@ func (o *PatternsLoader) movePatterns() (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// Verify that patterns were actually copied before creating the loaded marker
|
||||
var entries []os.DirEntry
|
||||
if entries, err = os.ReadDir(o.Patterns.Dir); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Count actual pattern directories (exclude the loaded file itself)
|
||||
patternCount := 0
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
patternCount++
|
||||
}
|
||||
}
|
||||
|
||||
if patternCount == 0 {
|
||||
err = fmt.Errorf("no patterns were successfully copied to %s", o.Patterns.Dir)
|
||||
return
|
||||
}
|
||||
|
||||
//create an empty file to indicate that the patterns have been updated if not exists
|
||||
_, _ = os.Create(o.loadedFilePath)
|
||||
if _, err = os.Create(o.loadedFilePath); err != nil {
|
||||
return fmt.Errorf("failed to create loaded marker file '%s': %w", o.loadedFilePath, err)
|
||||
}
|
||||
|
||||
err = os.RemoveAll(patternsDir)
|
||||
return
|
||||
@@ -147,15 +214,155 @@ func (o *PatternsLoader) gitCloneAndCopy() (err error) {
|
||||
return fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
|
||||
// Use the helper to fetch files
|
||||
fmt.Printf("Cloning repository %s (path: %s)...\n", o.DefaultGitRepoUrl.Value, o.DefaultFolder.Value)
|
||||
|
||||
// Try to fetch files with the current path
|
||||
err = githelper.FetchFilesFromRepo(githelper.FetchOptions{
|
||||
RepoURL: o.DefaultGitRepoUrl.Value,
|
||||
PathPrefix: o.DefaultFolder.Value,
|
||||
DestDir: o.tempPatternsFolder,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download patterns: %w", err)
|
||||
return fmt.Errorf("failed to download patterns from %s: %w", o.DefaultGitRepoUrl.Value, err)
|
||||
}
|
||||
|
||||
// Check if patterns were downloaded
|
||||
if patternCount, checkErr := o.countPatternsInDirectory(o.tempPatternsFolder); checkErr != nil {
|
||||
return fmt.Errorf("failed to read temp patterns directory: %w", checkErr)
|
||||
} else if patternCount == 0 {
|
||||
// No patterns found with current path, try automatic migration
|
||||
if migrationErr := o.tryPathMigration(); migrationErr != nil {
|
||||
return fmt.Errorf("no patterns found in repository at path %s and migration failed: %w", o.DefaultFolder.Value, migrationErr)
|
||||
}
|
||||
// Migration successful, try downloading again
|
||||
return o.gitCloneAndCopy()
|
||||
} else {
|
||||
fmt.Printf("Downloaded %d patterns to temporary directory\n", patternCount)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryPathMigration attempts to migrate from old pattern paths to new restructured paths
|
||||
func (o *PatternsLoader) tryPathMigration() (err error) {
|
||||
// Check if current path is the old "patterns" path
|
||||
if o.DefaultFolder.Value == "patterns" {
|
||||
fmt.Println("🔄 Detected old pattern path 'patterns', trying migration to 'data/patterns'...")
|
||||
|
||||
// Try the new restructured path
|
||||
newPath := "data/patterns"
|
||||
testTempFolder := filepath.Join(os.TempDir(), "fabric-patterns-test")
|
||||
|
||||
// Clean up any existing test temp folder
|
||||
if err := os.RemoveAll(testTempFolder); err != nil {
|
||||
fmt.Printf("Warning: failed to remove test temporary folder '%s': %v\n", testTempFolder, err)
|
||||
}
|
||||
|
||||
// Test if the new path works
|
||||
testErr := githelper.FetchFilesFromRepo(githelper.FetchOptions{
|
||||
RepoURL: o.DefaultGitRepoUrl.Value,
|
||||
PathPrefix: newPath,
|
||||
DestDir: testTempFolder,
|
||||
})
|
||||
|
||||
if testErr == nil {
|
||||
// Check if patterns exist in the new path
|
||||
if patternCount, countErr := o.countPatternsInDirectory(testTempFolder); countErr == nil && patternCount > 0 {
|
||||
fmt.Printf("✅ Found %d patterns at new path '%s', updating configuration...\n", patternCount, newPath)
|
||||
|
||||
// Update the configuration
|
||||
o.DefaultFolder.Value = newPath
|
||||
// Clean up the main temp folder and replace it with the test one
|
||||
os.RemoveAll(o.tempPatternsFolder)
|
||||
if renameErr := os.Rename(testTempFolder, o.tempPatternsFolder); renameErr != nil {
|
||||
// If rename fails, try copy
|
||||
if copyErr := copy.Copy(testTempFolder, o.tempPatternsFolder); copyErr != nil {
|
||||
return fmt.Errorf("failed to move test patterns to temp folder: %w", copyErr)
|
||||
}
|
||||
os.RemoveAll(testTempFolder)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up test folder
|
||||
os.RemoveAll(testTempFolder)
|
||||
}
|
||||
|
||||
return fmt.Errorf("unable to find patterns at current path '%s' or migrate to new structure", o.DefaultFolder.Value)
|
||||
}
|
||||
|
||||
// countPatternsInDirectory counts the number of pattern directories in a given directory
|
||||
func (o *PatternsLoader) countPatternsInDirectory(dir string) (int, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
patternCount := 0
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
patternCount++
|
||||
}
|
||||
}
|
||||
|
||||
return patternCount, nil
|
||||
}
|
||||
|
||||
// createUniquePatternsFile creates the unique_patterns.txt file with all pattern names
|
||||
func (o *PatternsLoader) createUniquePatternsFile() (err error) {
|
||||
// Read patterns from the main patterns directory
|
||||
entries, err := os.ReadDir(o.Patterns.Dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read patterns directory: %w", err)
|
||||
}
|
||||
|
||||
patternNamesMap := make(map[string]bool) // Use map to avoid duplicates
|
||||
|
||||
// Add patterns from main directory
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
patternNamesMap[entry.Name()] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Add patterns from custom patterns directory if it exists
|
||||
if o.Patterns.CustomPatternsDir != "" {
|
||||
if customEntries, customErr := os.ReadDir(o.Patterns.CustomPatternsDir); customErr == nil {
|
||||
for _, entry := range customEntries {
|
||||
if entry.IsDir() {
|
||||
patternNamesMap[entry.Name()] = true
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "📂 Also included patterns from custom directory: %s\n", o.Patterns.CustomPatternsDir)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Could not read custom patterns directory %s: %v\n", o.Patterns.CustomPatternsDir, customErr)
|
||||
}
|
||||
}
|
||||
|
||||
if len(patternNamesMap) == 0 {
|
||||
if o.Patterns.CustomPatternsDir != "" {
|
||||
return fmt.Errorf("no patterns found in directories %s and %s", o.Patterns.Dir, o.Patterns.CustomPatternsDir)
|
||||
}
|
||||
return fmt.Errorf("no patterns found in directory %s", o.Patterns.Dir)
|
||||
}
|
||||
|
||||
// Convert map to sorted slice
|
||||
var patternNames []string
|
||||
for name := range patternNamesMap {
|
||||
patternNames = append(patternNames, name)
|
||||
}
|
||||
|
||||
// Sort patterns alphabetically for consistent output
|
||||
sort.Strings(patternNames)
|
||||
|
||||
// Join pattern names with newlines
|
||||
content := strings.Join(patternNames, "\n") + "\n"
|
||||
if err = os.WriteFile(o.Patterns.UniquePatternsFilePath, []byte(content), 0644); err != nil {
|
||||
return fmt.Errorf("failed to write unique patterns file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("📝 Created unique patterns file with %d patterns\n", len(patternNames))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -71,3 +71,21 @@ func IsSymlinkToDir(path string) bool {
|
||||
|
||||
return false // Regular directories should not be treated as symlinks
|
||||
}
|
||||
|
||||
// GetDefaultConfigPath returns the default path for the configuration file
|
||||
// if it exists, otherwise returns an empty string.
|
||||
func GetDefaultConfigPath() (string, error) {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not determine user home directory: %w", err)
|
||||
}
|
||||
|
||||
defaultConfigPath := filepath.Join(homeDir, ".fabric.yaml")
|
||||
if _, err := os.Stat(defaultConfigPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", nil // Return no error for non-existent config path
|
||||
}
|
||||
return "", fmt.Errorf("error accessing default config path: %w", err)
|
||||
}
|
||||
return defaultConfigPath, nil
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{
|
||||
self,
|
||||
lib,
|
||||
buildGoApplication,
|
||||
go,
|
||||
@@ -8,8 +9,8 @@
|
||||
buildGoApplication {
|
||||
pname = "fabric-ai";
|
||||
version = import ./version.nix;
|
||||
src = ../../../.;
|
||||
pwd = ../../../.;
|
||||
src = self;
|
||||
pwd = self;
|
||||
modules = ./gomod2nix.toml;
|
||||
|
||||
doCheck = false;
|
||||
|
||||
@@ -100,6 +100,9 @@ schema = 3
|
||||
[mod."github.com/cloudwego/base64x"]
|
||||
version = "v0.1.5"
|
||||
hash = "sha256-MyUYTveN48DhnL8mwAgCRuMExLct98uzSPsmYlfaa4I="
|
||||
[mod."github.com/coder/websocket"]
|
||||
version = "v1.8.13"
|
||||
hash = "sha256-NbF0aPhy8YR3jRM6LMMQTtkeGTFba0eIBPAUsqI9KOk="
|
||||
[mod."github.com/cyphar/filepath-securejoin"]
|
||||
version = "v0.4.1"
|
||||
hash = "sha256-NOV6MfbkcQbfhNmfADQw2SJmZ6q1nw0wwg8Pm2tf2DM="
|
||||
@@ -163,6 +166,12 @@ schema = 3
|
||||
[mod."github.com/google/generative-ai-go"]
|
||||
version = "v0.20.1"
|
||||
hash = "sha256-9bSpEs4kByhgyTKiHdOY5muYjGBTluA1LvEjw2gSoLI="
|
||||
[mod."github.com/google/go-github/v66"]
|
||||
version = "v66.0.0"
|
||||
hash = "sha256-o4usfbApXwTuwIFMECagJwK2H4UMJbCpdyGdWZ5VUpI="
|
||||
[mod."github.com/google/go-querystring"]
|
||||
version = "v1.1.0"
|
||||
hash = "sha256-itsKgKghuX26czU79cK6C2n+lc27jm5Dw1XbIRgwZJY="
|
||||
[mod."github.com/google/s2a-go"]
|
||||
version = "v0.1.9"
|
||||
hash = "sha256-0AdSpSTso4bATmM/9qamWzKrVtOLDf7afvDhoiT/UpA="
|
||||
@@ -175,6 +184,12 @@ schema = 3
|
||||
[mod."github.com/googleapis/gax-go/v2"]
|
||||
version = "v2.14.2"
|
||||
hash = "sha256-QyY7wuCkrOJCJIf9Q884KD/BC3vk/QtQLXeLeNPt750="
|
||||
[mod."github.com/hasura/go-graphql-client"]
|
||||
version = "v0.14.4"
|
||||
hash = "sha256-TBNYIfC2CI0cVu7aZcHSWc6ZkgdkWSSfoCXqoAJT8jw="
|
||||
[mod."github.com/inconshreveable/mousetrap"]
|
||||
version = "v1.1.0"
|
||||
hash = "sha256-XWlYH0c8IcxAwQTnIi6WYqq44nOKUylSWxWO/vi+8pE="
|
||||
[mod."github.com/jbenet/go-context"]
|
||||
version = "v0.0.0-20150711004518-d14ea06fba99"
|
||||
hash = "sha256-VANNCWNNpARH/ILQV9sCQsBWgyL2iFT+4AHZREpxIWE="
|
||||
@@ -199,6 +214,9 @@ schema = 3
|
||||
[mod."github.com/mattn/go-isatty"]
|
||||
version = "v0.0.20"
|
||||
hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
|
||||
[mod."github.com/mattn/go-sqlite3"]
|
||||
version = "v1.14.28"
|
||||
hash = "sha256-mskU1xki6J1Fj6ItNgY/XNetB4Ta4jufEr4+JvTd7qs="
|
||||
[mod."github.com/modern-go/concurrent"]
|
||||
version = "v0.0.0-20180306012644-bacd9c7ef1dd"
|
||||
hash = "sha256-OTySieAgPWR4oJnlohaFTeK1tRaVp/b0d1rYY8xKMzo="
|
||||
@@ -221,8 +239,8 @@ schema = 3
|
||||
version = "v2.2.4"
|
||||
hash = "sha256-8qQIPldbsS5RO8v/FW/se3ZsAyvLzexiivzJCbGRg2Q="
|
||||
[mod."github.com/pjbgf/sha1cd"]
|
||||
version = "v0.3.2"
|
||||
hash = "sha256-jdbiRhU8xc1C5c8m7BSCj71PUXHY3f7TWFfxDKKpUMk="
|
||||
version = "v0.4.0"
|
||||
hash = "sha256-a+KXfvy1KEna9yJZ+rKXzyTT0A3hg6+yvgqQKD0xXFQ="
|
||||
[mod."github.com/pkg/errors"]
|
||||
version = "v0.9.1"
|
||||
hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw="
|
||||
@@ -241,6 +259,12 @@ schema = 3
|
||||
[mod."github.com/skeema/knownhosts"]
|
||||
version = "v1.3.1"
|
||||
hash = "sha256-kjqQDzuncQNTuOYegqVZExwuOt/Z73m2ST7NZFEKixI="
|
||||
[mod."github.com/spf13/cobra"]
|
||||
version = "v1.9.1"
|
||||
hash = "sha256-dzEqquABE3UqZmJuj99244QjvfojS8cFlsPr/MXQGj0="
|
||||
[mod."github.com/spf13/pflag"]
|
||||
version = "v1.0.6"
|
||||
hash = "sha256-NjrK0FZPIfO/p2xtL1J7fOBQNTZAPZOC6Cb4aMMvhxI="
|
||||
[mod."github.com/stretchr/testify"]
|
||||
version = "v1.10.0"
|
||||
hash = "sha256-fJ4gnPr0vnrOhjQYQwJ3ARDKPsOtA7d4olQmQWR+wpI="
|
||||
@@ -289,6 +313,9 @@ schema = 3
|
||||
[mod."golang.org/x/crypto"]
|
||||
version = "v0.39.0"
|
||||
hash = "sha256-FtwjbVoAhZkx7F2hmzi9Y0J87CVVhWcrZzun+zWQLzc="
|
||||
[mod."golang.org/x/exp"]
|
||||
version = "v0.0.0-20250531010427-b6e5de432a8b"
|
||||
hash = "sha256-QaFfjyB+pogCkUkJskR9xnXwkCOU828XJRrzwwLm6Ms="
|
||||
[mod."golang.org/x/net"]
|
||||
version = "v0.41.0"
|
||||
hash = "sha256-6/pi8rNmGvBFzkJQXkXkMfL1Bjydhg3BgAMYDyQ/Uvg="
|
||||
@@ -296,14 +323,14 @@ schema = 3
|
||||
version = "v0.30.0"
|
||||
hash = "sha256-btD7BUtQpOswusZY5qIU90uDo38buVrQ0tmmQ8qNHDg="
|
||||
[mod."golang.org/x/sync"]
|
||||
version = "v0.15.0"
|
||||
hash = "sha256-Jf4ehm8H8YAWY6mM151RI5CbG7JcOFtmN0AZx4bE3UE="
|
||||
version = "v0.16.0"
|
||||
hash = "sha256-sqKDRESeMzLe0jWGWltLZL/JIgrn0XaIeBWCzVN3Bks="
|
||||
[mod."golang.org/x/sys"]
|
||||
version = "v0.33.0"
|
||||
hash = "sha256-wlOzIOUgAiGAtdzhW/KPl/yUVSH/lvFZfs5XOuJ9LOQ="
|
||||
version = "v0.34.0"
|
||||
hash = "sha256-5rZ7p8IaGli5X1sJbfIKOcOEwY4c0yQhinJPh2EtK50="
|
||||
[mod."golang.org/x/text"]
|
||||
version = "v0.26.0"
|
||||
hash = "sha256-N+27nBCyGvje0yCTlUzZoVZ0LRxx4AJ+eBlrFQVRlFQ="
|
||||
version = "v0.27.0"
|
||||
hash = "sha256-VX0rOh6L3qIvquKSGjfZQFU8URNtGvkNvxE7OZtboW8="
|
||||
[mod."golang.org/x/time"]
|
||||
version = "v0.12.0"
|
||||
hash = "sha256-Cp3oxrCMH2wyxjzr5SHVmyhgaoUuSl56Uy00Q7DYEpw="
|
||||
|
||||
@@ -1 +1 @@
|
||||
"1.4.241"
|
||||
"1.4.256"
|
||||
|
||||
@@ -16,16 +16,16 @@
|
||||
gomod2nix
|
||||
goEnv
|
||||
|
||||
(pkgs.writeShellScriptBin "update" ''
|
||||
(pkgs.writeShellScriptBin "update-mod" ''
|
||||
go get -u
|
||||
go mod tidy
|
||||
gomod2nix generate
|
||||
gomod2nix generate --outdir nix/pkgs/fabric
|
||||
'')
|
||||
];
|
||||
|
||||
shellHook = ''
|
||||
echo -e "\033[0;32;4mHeper commands:\033[0m"
|
||||
echo "'update' instead of 'go get -u && go mod tidy'"
|
||||
echo -e "\033[0;32;4mHelper commands:\033[0m"
|
||||
echo "'update-mod' instead of 'go get -u && go mod tidy && gomod2nix generate --outdir nix/pkgs/fabric'"
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1861,6 +1861,16 @@
|
||||
"CR THINKING",
|
||||
"SELF"
|
||||
]
|
||||
},
|
||||
{
|
||||
"patternName": "generate_code_rules",
|
||||
"description": "Extracts a list of best practices rules for AI coding assisted tools.",
|
||||
"tags": [
|
||||
"ANALYSIS",
|
||||
"EXTRACT",
|
||||
"DEVELOPMENT",
|
||||
"AI"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -903,6 +903,10 @@
|
||||
{
|
||||
"patternName": "t_check_dunning_kruger",
|
||||
"pattern_extract": "# IDENTITY You are an expert at understanding deep context about a person or entity, and then creating wisdom from that context combined with the instruction or question given in the input. # STEPS 1. Read the incoming TELOS File thoroughly. Fully understand everything about this person or entity. 2. Deeply study the input instruction or question. 3. Spend significant time and effort thinking about how these two are related, and what would be the best possible output for the person who sent the input. 4. Evaluate the input against the Dunning-Kruger effect and input's prior beliefs. Explore cognitive bias, subjective ability and objective ability for: low-ability areas where the input owner overestimate their knowledge or skill; and the opposite, high-ability areas where the input owner underestimate their knowledge or skill. # EXAMPLE In education, students who overestimate their understanding of a topic may not seek help or put in the necessary effort, while high-achieving students might doubt their abilities. In healthcare, overconfident practitioners might make critical errors, and underconfident practitioners might delay crucial decisions. In politics, politicians with limited expertise might propose simplistic solutions and ignore expert advice. END OF EXAMPLE # OUTPUT - In a section called OVERESTIMATION OF COMPETENCE, output a set of 10, 16-word bullets, that capture the principal misinterpretation of lack of knowledge or skill which are leading the input owner to believe they are more knowledgeable or skilled than they actually are. - In a section called UNDERESTIMATION OF COMPETENCE, output a set of 10, 16-word bullets,that capture the principal misinterpreation of underestimation of their knowledge or skill which are preventing the input owner to see opportunities. - In a section called METACOGNITIVIVE SKILLS, output a set of 10-word bullets that expose areas where the input owner struggles to accuratelly assess their own performance and may not be aware of the gap between their actual ability and their perceived ability. - In a section called IMPACT ON DECISION MAKING, output a set of 10-word bullets exposing facts, biases, traces of behavior based on overinflated self-assessment, that can lead to poor decisions. - At the end summarize the findings and give the input owner a motivational and constructive perspective on how they can start to tackle principal 5 gaps in their perceived skills and knowledge competencies. Don't be over simplistic. # OUTPUT INSTRUCTIONS 1. Only output valid, basic Markdown. No special formatting or italics or bolding or anything. 2. Do not output any content other than the sections above. Nothing else."
|
||||
},
|
||||
{
|
||||
"patternName": "generate_code_rules",
|
||||
"pattern_extract": "# IDENTITY AND PURPOSE You are a senior developer and expert prompt engineer. Think ultra hard to distill the following transcription or tutorial in as little set of unique rules as possible intended for best practices guidance in AI assisted coding tools, each rule has to be in one sentence as a direct instruction, avoid explanations and cosmetic language. Output in Markdown, I prefer bullet dash (-). --- # TRANSCRIPT"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1861,6 +1861,16 @@
|
||||
"CR THINKING",
|
||||
"SELF"
|
||||
]
|
||||
},
|
||||
{
|
||||
"patternName": "generate_code_rules",
|
||||
"description": "Extracts a list of best practices rules for AI coding assisted tools.",
|
||||
"tags": [
|
||||
"ANALYSIS",
|
||||
"EXTRACT",
|
||||
"DEVELOPMENT",
|
||||
"AI"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user