Compare commits

...

8 Commits

Author SHA1 Message Date
github-actions[bot]
17d863fd57 Update version to v1.4.227 and commit 2025-07-04 22:40:01 +00:00
Daniel Miessler 🛡️
7c9dbfd343 Merge pull request #1572 from ksylvan/0704-OpenAI-Image-Generation-Tool
Add Image Generation Support to Fabric
2025-07-04 15:38:27 -07:00
Kayvan Sylvan
d9260bdf26 chore: refactor image generation constants for clarity and reuse
### CHANGES

- Define `ImageGenerationResponseType` constant for response handling
- Define `ImageGenerationToolType` constant for tool type usage
- Update `addImageGenerationTool` to use defined constants
- Refactor `extractAndSaveImages` to use response type constant
2025-07-04 15:14:46 -07:00
Kayvan Sylvan
63a0cfeb1e feat: add web search and image file support to fabric CLI
## CHANGES

- Add web search tool for Anthropic and OpenAI models
- Add search location parameter for web search results
- Add image file output option with format support
- Update zsh completion with new search and image flags
- Update bash completion with new option handling logic
- Update fish completion with search and image descriptions
- Support PNG, JPG, JPEG, GIF, BMP image formats
2025-07-04 14:49:40 -07:00
Kayvan Sylvan
12fc6e2000 feat: add image generation support with OpenAI image generation model
## CHANGES

- Add `--image-file` flag for saving generated images
- Implement image generation tool integration with OpenAI
- Support image editing with attachment input files
- Add comprehensive test coverage for image features
- Update documentation with image generation examples
- Fix HTML formatting issues in README
- Improve PowerShell code block indentation
- Clean up help text formatting and spacing
2025-07-04 14:36:55 -07:00
Daniel Miessler
fe5900a5dc Fixed ul tag applier. 2025-07-04 14:25:54 -07:00
Daniel Miessler
1b6b8e3d72 Updated ul tag prompt. 2025-07-04 14:21:25 -07:00
Daniel Miessler
c85301cb1f Added the UL tags pattern. 2025-07-04 14:17:43 -07:00
13 changed files with 363 additions and 85 deletions

165
README.md
View File

@@ -15,9 +15,7 @@ Fabric is graciously supported by…
[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/danielmiessler/fabric)
<div align="center">
<p class="align center">
<h4><code>fabric</code> is an open-source framework for augmenting humans using AI.</h4>
</p>
</div>
[Updates](#updates) •
@@ -41,9 +39,9 @@ Since the start of modern AI in late 2022 we've seen an **_extraordinary_** numb
It's all really exciting and powerful, but _it's not easy to integrate this functionality into our lives._
<p class="align center">
<div class="align center">
<h4>In other words, AI doesn't have a capabilities problem—it has an <em>integration</em> problem.</h4>
</p>
</div>
**Fabric was created to address this by creating and organizing the fundamental units of AI—the prompts themselves!**
@@ -120,6 +118,8 @@ Keep in mind that many of these were recorded when Fabric was Python-based, so r
> - Web search is available for both Anthropic and OpenAI providers
> - Previous plugin-level search configurations have been removed in favor of the new flag-based approach.
> - If you used the previous approach, consider cleaning up your `~/.config/fabric/.env` file, removing the unused `ANTHROPIC_WEB_SEARCH_TOOL_ENABLED` and `ANTHROPIC_WEB_SEARCH_TOOL_LOCATION` variables.
> - Fabric now supports image generation using the `--image-file` flag with OpenAI models
> - Image generation works with both text prompts and input images (via `--attachment`) for image editing tasks
>
>
>June 17, 2025
@@ -292,88 +292,88 @@ yt() {
You can add the below code for the equivalent aliases inside PowerShell by running `notepad $PROFILE` inside a PowerShell window:
```powershell
# Path to the patterns directory
$patternsPath = Join-Path $HOME ".config/fabric/patterns"
foreach ($patternDir in Get-ChildItem -Path $patternsPath -Directory) {
$patternName = $patternDir.Name
```powershell
# Path to the patterns directory
$patternsPath = Join-Path $HOME ".config/fabric/patterns"
foreach ($patternDir in Get-ChildItem -Path $patternsPath -Directory) {
$patternName = $patternDir.Name
# Dynamically define a function for each pattern
$functionDefinition = @"
function $patternName {
[CmdletBinding()]
param(
[Parameter(ValueFromPipeline = `$true)]
[string] `$InputObject,
# Dynamically define a function for each pattern
$functionDefinition = @"
function $patternName {
[CmdletBinding()]
param(
[Parameter(ValueFromPipeline = `$true)]
[string] `$InputObject,
[Parameter(ValueFromRemainingArguments = `$true)]
[String[]] `$patternArgs
)
[Parameter(ValueFromRemainingArguments = `$true)]
[String[]] `$patternArgs
)
begin {
# Initialize an array to collect pipeline input
`$collector = @()
}
begin {
# Initialize an array to collect pipeline input
`$collector = @()
}
process {
# Collect pipeline input objects
if (`$InputObject) {
`$collector += `$InputObject
process {
# Collect pipeline input objects
if (`$InputObject) {
`$collector += `$InputObject
}
}
end {
# Join all pipeline input into a single string, separated by newlines
`$pipelineContent = `$collector -join "`n"
# If there's pipeline input, include it in the call to fabric
if (`$pipelineContent) {
`$pipelineContent | fabric --pattern $patternName `$patternArgs
} else {
# No pipeline input; just call fabric with the additional args
fabric --pattern $patternName `$patternArgs
}
}
}
end {
# Join all pipeline input into a single string, separated by newlines
`$pipelineContent = `$collector -join "`n"
# If there's pipeline input, include it in the call to fabric
if (`$pipelineContent) {
`$pipelineContent | fabric --pattern $patternName `$patternArgs
} else {
# No pipeline input; just call fabric with the additional args
fabric --pattern $patternName `$patternArgs
}
}
}
"@
# Add the function to the current session
Invoke-Expression $functionDefinition
}
# Define the 'yt' function as well
function yt {
[CmdletBinding()]
param(
[Parameter()]
[Alias("timestamps")]
[switch]$t,
[Parameter(Position = 0, ValueFromPipeline = $true)]
[string]$videoLink
)
begin {
$transcriptFlag = "--transcript"
if ($t) {
$transcriptFlag = "--transcript-with-timestamps"
}
"@
# Add the function to the current session
Invoke-Expression $functionDefinition
}
process {
if (-not $videoLink) {
Write-Error "Usage: yt [-t | --timestamps] youtube-link"
return
}
}
# Define the 'yt' function as well
function yt {
[CmdletBinding()]
param(
[Parameter()]
[Alias("timestamps")]
[switch]$t,
end {
if ($videoLink) {
# Execute and allow output to flow through the pipeline
fabric -y $videoLink $transcriptFlag
[Parameter(Position = 0, ValueFromPipeline = $true)]
[string]$videoLink
)
begin {
$transcriptFlag = "--transcript"
if ($t) {
$transcriptFlag = "--transcript-with-timestamps"
}
}
process {
if (-not $videoLink) {
Write-Error "Usage: yt [-t | --timestamps] youtube-link"
return
}
}
end {
if ($videoLink) {
# Execute and allow output to flow through the pipeline
fabric -y $videoLink $transcriptFlag
}
}
}
}
```
```
This also creates a `yt` alias that allows you to use `yt https://www.youtube.com/watch?v=4b0iet22VIk` to get transcripts, comments, and metadata.
@@ -493,7 +493,6 @@ fabric -h
```
```plaintext
Usage:
fabric [OPTIONS]
@@ -508,7 +507,9 @@ Application Options:
-T, --topp= Set top P (default: 0.9)
-s, --stream Stream
-P, --presencepenalty= Set presence penalty (default: 0.0)
-r, --raw Use the defaults of the model without sending chat options (like temperature etc.) and use the user role instead of the system role for patterns.
-r, --raw Use the defaults of the model without sending chat options (like
temperature etc.) and use the user role instead of the system role for
patterns.
-F, --frequencypenalty= Set frequency penalty (default: 0.0)
-l, --listpatterns List all patterns
-L, --listmodels List all available models
@@ -522,9 +523,12 @@ Application Options:
--output-session Output the entire session (also a temporary one) to the output file
-n, --latest= Number of latest patterns to list (default: 0)
-d, --changeDefaultModel Change default model
-y, --youtube= YouTube video or play list "URL" to grab transcript, comments from it and send to chat or print it put to the console and store it in the output file
-y, --youtube= YouTube video or play list "URL" to grab transcript, comments from it
and send to chat or print it put to the console and store it in the
output file
--playlist Prefer playlist over video if both ids are present in the URL
--transcript Grab transcript from YouTube video and send to chat (it is used per default).
--transcript Grab transcript from YouTube video and send to chat (it is used per
default).
--transcript-with-timestamps Grab transcript from YouTube video with timestamps and send to chat
--comments Grab comments from YouTube video and send to chat
--metadata Output video metadata
@@ -552,6 +556,9 @@ Application Options:
--liststrategies List all strategies
--listvendors List all vendors
--shell-complete-list Output raw list without headers/formatting (for shell completion)
--search Enable web search tool for supported models (Anthropic, OpenAI)
--search-location= Set location for web search results (e.g., 'America/Los_Angeles')
--image-file= Save generated image to specified file path (e.g., 'output.png')
Help Options:
-h, --help Show this help message

View File

@@ -76,6 +76,7 @@ type Flags struct {
ShellCompleteOutput bool `long:"shell-complete-list" description:"Output raw list without headers/formatting (for shell completion)"`
Search bool `long:"search" description:"Enable web search tool for supported models (Anthropic, OpenAI)"`
SearchLocation string `long:"search-location" description:"Set location for web search results (e.g., 'America/Los_Angeles')"`
ImageFile string `long:"image-file" description:"Save generated image to specified file path (e.g., 'output.png')"`
}
var debug = false
@@ -267,6 +268,7 @@ func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {
ModelContextLength: o.ModelContextLength,
Search: o.Search,
SearchLocation: o.SearchLocation,
ImageFile: o.ImageFile,
}
return
}

View File

@@ -28,6 +28,7 @@ type ChatOptions struct {
MaxTokens int
Search bool
SearchLocation string
ImageFile string
}
// NormalizeMessages remove empty messages and ensure messages order user-assist-user

View File

@@ -96,6 +96,9 @@ _fabric() {
'(--api-key)--api-key[API key used to secure server routes]:api-key:' \
'(--config)--config[Path to YAML config file]:config file:_files -g "*.yaml *.yml"' \
'(--version)--version[Print current version]' \
'(--search)--search[Enable web search tool for supported models (Anthropic, OpenAI)]' \
'(--search-location)--search-location[Set location for web search results]:location:' \
'(--image-file)--image-file[Save generated image to specified file path]:image file:_files -g "*.png *.jpg *.jpeg *.gif *.bmp"' \
'(--listextensions)--listextensions[List all registered extensions]' \
'(--addextension)--addextension[Register a new extension from config file path]:config file:_files -g "*.yaml *.yml"' \
'(--rmextension)--rmextension[Remove a registered extension by name]:extension:_fabric_extensions' \

View File

@@ -13,7 +13,7 @@ _fabric() {
_get_comp_words_by_ref -n : cur prev words cword
# Define all possible options/flags
local opts="--pattern -p --variable -v --context -C --session --attachment -a --setup -S --temperature -t --topp -T --stream -s --presencepenalty -P --raw -r --frequencypenalty -F --listpatterns -l --listmodels -L --listcontexts -x --listsessions -X --updatepatterns -U --copy -c --model -m --modelContextLength --output -o --output-session --latest -n --changeDefaultModel -d --youtube -y --playlist --transcript --transcript-with-timestamps --comments --metadata --language -g --scrape_url -u --scrape_question -q --seed -e --wipecontext -w --wipesession -W --printcontext --printsession --readability --input-has-vars --dry-run --serve --serveOllama --address --api-key --config --version --listextensions --addextension --rmextension --strategy --liststrategies --listvendors --shell-complete-list --help -h"
local opts="--pattern -p --variable -v --context -C --session --attachment -a --setup -S --temperature -t --topp -T --stream -s --presencepenalty -P --raw -r --frequencypenalty -F --listpatterns -l --listmodels -L --listcontexts -x --listsessions -X --updatepatterns -U --copy -c --model -m --modelContextLength --output -o --output-session --latest -n --changeDefaultModel -d --youtube -y --playlist --transcript --transcript-with-timestamps --comments --metadata --language -g --scrape_url -u --scrape_question -q --seed -e --wipecontext -w --wipesession -W --printcontext --printsession --readability --input-has-vars --dry-run --serve --serveOllama --address --api-key --config --search --search-location --image-file --version --listextensions --addextension --rmextension --strategy --liststrategies --listvendors --shell-complete-list --help -h"
# Helper function for dynamic completions
_fabric_get_list() {
@@ -63,12 +63,12 @@ _fabric() {
return 0
;;
# Options requiring file/directory paths
-a | --attachment | -o | --output | --config | --addextension)
-a | --attachment | -o | --output | --config | --addextension | --image-file)
_filedir
return 0
;;
# Options requiring simple arguments (no specific completion logic here)
-v | --variable | -t | --temperature | -T | --topp | -P | --presencepenalty | -F | --frequencypenalty | --modelContextLength | -n | --latest | -y | --youtube | -g | --language | -u | --scrape_url | -q | --scrape_question | -e | --seed | --address | --api-key)
-v | --variable | -t | --temperature | -T | --topp | -P | --presencepenalty | -F | --frequencypenalty | --modelContextLength | -n | --latest | -y | --youtube | -g | --language | -u | --scrape_url | -q | --scrape_question | -e | --seed | --address | --api-key | --search-location)
# No specific completion suggestions, user types the value
return 0
;;

View File

@@ -60,6 +60,8 @@ complete -c fabric -l printsession -d "Print session" -a "(__fabric_get_sessions
complete -c fabric -l address -d "The address to bind the REST API (default: :8080)"
complete -c fabric -l api-key -d "API key used to secure server routes"
complete -c fabric -l config -d "Path to YAML config file" -r -a "*.yaml *.yml"
complete -c fabric -l search-location -d "Set location for web search results (e.g., 'America/Los_Angeles')"
complete -c fabric -l image-file -d "Save generated image to specified file path (e.g., 'output.png')" -r -a "*.png *.jpg *.jpeg *.gif *.bmp"
complete -c fabric -l addextension -d "Register a new extension from config file path" -r -a "*.yaml *.yml"
complete -c fabric -l rmextension -d "Remove a registered extension by name" -a "(__fabric_get_extensions)"
complete -c fabric -l strategy -d "Choose a strategy from the available strategies" -a "(__fabric_get_strategies)"
@@ -84,6 +86,7 @@ complete -c fabric -l metadata -d "Output video metadata"
complete -c fabric -l readability -d "Convert HTML input into a clean, readable view"
complete -c fabric -l input-has-vars -d "Apply variables to user input"
complete -c fabric -l dry-run -d "Show what would be sent to the model without actually sending it"
complete -c fabric -l search -d "Enable web search tool for supported models (Anthropic, OpenAI)"
complete -c fabric -l serve -d "Serve the Fabric Rest API"
complete -c fabric -l serveOllama -d "Serve the Fabric Rest API with ollama endpoints"
complete -c fabric -l version -d "Print current version"

View File

@@ -1 +1 @@
"1.4.226"
"1.4.227"

View File

@@ -0,0 +1,48 @@
# IDENTITY
You are a superintelligent expert on content of all forms, with deep understanding of which topics, categories, themes, and tags apply to any piece of content.
# GOAL
Your goal is to output a JSON object called tags, with the following tags applied if the content is significantly about their topic.
- **future** - Posts about the future, predictions, emerging trends
- **politics** - Political topics, elections, governance, policy
- **cybersecurity** - Security, hacking, vulnerabilities, infosec
- **books** - Book reviews, reading lists, literature
- **society** - Social issues, cultural observations, human behavior
- **science** - Scientific topics, research, discoveries
- **philosophy** - Philosophical discussions, ethics, meaning
- **nationalsecurity** - Defense, intelligence, geopolitics
- **ai** - Artificial intelligence, machine learning, automation
- **culture** - Cultural commentary, trends, observations
- **personal** - Personal stories, experiences, reflections
- **innovation** - New ideas, inventions, breakthroughs
- **business** - Business, entrepreneurship, economics
- **meaning** - Purpose, existential topics, life meaning
- **technology** - General tech topics, tools, gadgets
- **ethics** - Moral questions, ethical dilemmas
- **productivity** - Efficiency, time management, workflows
- **writing** - Writing craft, process, tips
- **creativity** - Creative process, artistic expression
# STEPS
1. Deeply understand the content and its themes and categories and topics.
2. Evaluate the list of tags above.
3. Determine which tags apply to the content.
4. Output the "tags" JSON object.
# NOTES
- It's ok, and quite normal, for multiple tags to apply—which is why this is tags and not categories
- All AI posts should have the technology tag, and that's ok. But not all technology posts are about AI, and therefore the AI tag needs to be evaluated separately. That goes for all potentially nested or conflicted tags.
- Be a bit conservative in applying tags. If a piece of content is only tangentially related to a tag, don't include it.
# OUTPUT INSTRUCTIONS
- Output ONLY the JSON object, and nothing else.
- That means DO NOT OUTPUT the ```json format indicator. ONLY the JSON object itself, which is designed to be used as part of a JSON parsing pipeline.

View File

@@ -82,6 +82,9 @@ func (c *Client) formatOptions(opts *common.ChatOptions) string {
builder.WriteString(fmt.Sprintf("SearchLocation: %s\n", opts.SearchLocation))
}
}
if opts.ImageFile != "" {
builder.WriteString(fmt.Sprintf("ImageFile: %s\n", opts.ImageFile))
}
return builder.String()
}

View File

@@ -134,6 +134,12 @@ func (o *Client) sendResponses(ctx context.Context, msgs []*chat.ChatCompletionM
if resp, err = o.ApiClient.Responses.New(ctx, req); err != nil {
return
}
// Extract and save images if requested
if err = o.extractAndSaveImages(resp, opts); err != nil {
return
}
ret = o.extractText(resp)
return
}
@@ -183,6 +189,9 @@ func (o *Client) buildResponseParams(
},
}
// Add tools if enabled
var tools []responses.ToolUnionParam
// Add web search tool if enabled
if opts.Search {
webSearchTool := responses.ToolParamOfWebSearchPreview("web_search_preview")
@@ -195,7 +204,14 @@ func (o *Client) buildResponseParams(
}
}
ret.Tools = []responses.ToolUnionParam{webSearchTool}
tools = append(tools, webSearchTool)
}
// Add image generation tool if needed
tools = o.addImageGenerationTool(opts, tools)
if len(tools) > 0 {
ret.Tools = tools
}
if !opts.Raw {

View File

@@ -0,0 +1,81 @@
package openai
// This file contains helper methods for image generation and processing
// using OpenAI's Responses API and Image API.
import (
"encoding/base64"
"fmt"
"os"
"path/filepath"
"github.com/danielmiessler/fabric/common"
"github.com/openai/openai-go/responses"
)
// ImageGenerationResponseType is the type used for image generation calls in responses
const ImageGenerationResponseType = "image_generation_call"
const ImageGenerationToolType = "image_generation"
// addImageGenerationTool adds the image generation tool to the request if needed
func (o *Client) addImageGenerationTool(opts *common.ChatOptions, tools []responses.ToolUnionParam) []responses.ToolUnionParam {
// Check if the request seems to be asking for image generation
if o.shouldUseImageGeneration(opts) {
imageGenTool := responses.ToolUnionParam{
OfImageGeneration: &responses.ToolImageGenerationParam{
Type: ImageGenerationToolType,
Model: "gpt-image-1",
OutputFormat: "png",
Quality: "auto",
Size: "auto",
},
}
tools = append(tools, imageGenTool)
}
return tools
}
// shouldUseImageGeneration determines if image generation should be enabled
// This is a heuristic based on the presence of --image-file flag
func (o *Client) shouldUseImageGeneration(opts *common.ChatOptions) bool {
return opts.ImageFile != ""
}
// extractAndSaveImages extracts generated images from the response and saves them
func (o *Client) extractAndSaveImages(resp *responses.Response, opts *common.ChatOptions) error {
if opts.ImageFile == "" {
return nil // No image file specified, skip saving
}
// Extract image data from response
for _, item := range resp.Output {
if item.Type == ImageGenerationResponseType {
imageCall := item.AsImageGenerationCall()
if imageCall.Status == "completed" && imageCall.Result != "" {
// Decode base64 image data
imageData, err := base64.StdEncoding.DecodeString(imageCall.Result)
if err != nil {
return fmt.Errorf("failed to decode image data: %w", err)
}
// Ensure directory exists
dir := filepath.Dir(opts.ImageFile)
if dir != "." {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", dir, err)
}
}
// Save image to file
if err := os.WriteFile(opts.ImageFile, imageData, 0644); err != nil {
return fmt.Errorf("failed to save image to %s: %w", opts.ImageFile, err)
}
fmt.Printf("Image saved to: %s\n", opts.ImageFile)
return nil
}
}
}
return nil
}

View File

@@ -0,0 +1,114 @@
package openai
import (
"testing"
"github.com/danielmiessler/fabric/chat"
"github.com/danielmiessler/fabric/common"
"github.com/openai/openai-go/responses"
"github.com/stretchr/testify/assert"
)
func TestShouldUseImageGeneration(t *testing.T) {
client := NewClient()
// Test with image file specified
opts := &common.ChatOptions{
ImageFile: "output.png",
}
assert.True(t, client.shouldUseImageGeneration(opts), "Should use image generation when image file is specified")
// Test without image file
opts = &common.ChatOptions{
ImageFile: "",
}
assert.False(t, client.shouldUseImageGeneration(opts), "Should not use image generation when no image file is specified")
}
func TestAddImageGenerationTool(t *testing.T) {
client := NewClient()
// Test with image generation enabled
opts := &common.ChatOptions{
ImageFile: "output.png",
}
tools := []responses.ToolUnionParam{}
result := client.addImageGenerationTool(opts, tools)
assert.Len(t, result, 1, "Should add one image generation tool")
assert.NotNil(t, result[0].OfImageGeneration, "Should have image generation tool")
assert.Equal(t, "image_generation", string(result[0].OfImageGeneration.Type))
assert.Equal(t, "gpt-image-1", result[0].OfImageGeneration.Model)
assert.Equal(t, "png", result[0].OfImageGeneration.OutputFormat)
// Test without image generation
opts = &common.ChatOptions{
ImageFile: "",
}
tools = []responses.ToolUnionParam{}
result = client.addImageGenerationTool(opts, tools)
assert.Len(t, result, 0, "Should not add image generation tool when not needed")
}
func TestBuildResponseParams_WithImageGeneration(t *testing.T) {
client := NewClient()
opts := &common.ChatOptions{
Model: "gpt-image-1",
ImageFile: "output.png",
}
msgs := []*chat.ChatCompletionMessage{
{Role: "user", Content: "Generate an image of a cat"},
}
params := client.buildResponseParams(msgs, opts)
assert.NotNil(t, params.Tools, "Expected tools when image generation is enabled")
// Should have image generation tool
hasImageTool := false
for _, tool := range params.Tools {
if tool.OfImageGeneration != nil {
hasImageTool = true
assert.Equal(t, "image_generation", string(tool.OfImageGeneration.Type))
assert.Equal(t, "gpt-image-1", tool.OfImageGeneration.Model)
break
}
}
assert.True(t, hasImageTool, "Should have image generation tool")
}
func TestBuildResponseParams_WithBothSearchAndImage(t *testing.T) {
client := NewClient()
opts := &common.ChatOptions{
Model: "gpt-image-1",
Search: true,
SearchLocation: "America/Los_Angeles",
ImageFile: "output.png",
}
msgs := []*chat.ChatCompletionMessage{
{Role: "user", Content: "Search for cat images and generate one"},
}
params := client.buildResponseParams(msgs, opts)
assert.NotNil(t, params.Tools, "Expected tools when both search and image generation are enabled")
assert.Len(t, params.Tools, 2, "Should have both search and image generation tools")
hasSearchTool := false
hasImageTool := false
for _, tool := range params.Tools {
if tool.OfWebSearchPreview != nil {
hasSearchTool = true
}
if tool.OfImageGeneration != nil {
hasImageTool = true
}
}
assert.True(t, hasSearchTool, "Should have web search tool")
assert.True(t, hasImageTool, "Should have image generation tool")
}

View File

@@ -1,3 +1,3 @@
package main
var version = "v1.4.226"
var version = "v1.4.227"