diff --git a/README.md b/README.md index 2f5e7f51..cd46e974 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,7 @@ Fabric is graciously supported by… [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/danielmiessler/fabric)
-

fabric is an open-source framework for augmenting humans using AI.

-

[Updates](#updates) • @@ -41,9 +39,9 @@ Since the start of modern AI in late 2022 we've seen an **_extraordinary_** numb It's all really exciting and powerful, but _it's not easy to integrate this functionality into our lives._ -

+

In other words, AI doesn't have a capabilities problem—it has an integration problem.

-

+
**Fabric was created to address this by creating and organizing the fundamental units of AI—the prompts themselves!** @@ -120,6 +118,8 @@ Keep in mind that many of these were recorded when Fabric was Python-based, so r > - Web search is available for both Anthropic and OpenAI providers > - Previous plugin-level search configurations have been removed in favor of the new flag-based approach. > - If you used the previous approach, consider cleaning up your `~/.config/fabric/.env` file, removing the unused `ANTHROPIC_WEB_SEARCH_TOOL_ENABLED` and `ANTHROPIC_WEB_SEARCH_TOOL_LOCATION` variables. +> - Fabric now supports image generation using the `--image-file` flag with OpenAI models +> - Image generation works with both text prompts and input images (via `--attachment`) for image editing tasks > > >June 17, 2025 @@ -292,88 +292,88 @@ yt() { You can add the below code for the equivalent aliases inside PowerShell by running `notepad $PROFILE` inside a PowerShell window: -```powershell -# Path to the patterns directory -$patternsPath = Join-Path $HOME ".config/fabric/patterns" -foreach ($patternDir in Get-ChildItem -Path $patternsPath -Directory) { - $patternName = $patternDir.Name + ```powershell + # Path to the patterns directory + $patternsPath = Join-Path $HOME ".config/fabric/patterns" + foreach ($patternDir in Get-ChildItem -Path $patternsPath -Directory) { + $patternName = $patternDir.Name - # Dynamically define a function for each pattern - $functionDefinition = @" -function $patternName { - [CmdletBinding()] - param( - [Parameter(ValueFromPipeline = `$true)] - [string] `$InputObject, + # Dynamically define a function for each pattern + $functionDefinition = @" + function $patternName { + [CmdletBinding()] + param( + [Parameter(ValueFromPipeline = `$true)] + [string] `$InputObject, - [Parameter(ValueFromRemainingArguments = `$true)] - [String[]] `$patternArgs - ) + [Parameter(ValueFromRemainingArguments = `$true)] + [String[]] `$patternArgs + ) - begin { - # Initialize an array to collect pipeline input - `$collector = @() - } + begin { + # Initialize an array to collect pipeline input + `$collector = @() + } - process { - # Collect pipeline input objects - if (`$InputObject) { - `$collector += `$InputObject + process { + # Collect pipeline input objects + if (`$InputObject) { + `$collector += `$InputObject + } + } + + end { + # Join all pipeline input into a single string, separated by newlines + `$pipelineContent = `$collector -join "`n" + + # If there's pipeline input, include it in the call to fabric + if (`$pipelineContent) { + `$pipelineContent | fabric --pattern $patternName `$patternArgs + } else { + # No pipeline input; just call fabric with the additional args + fabric --pattern $patternName `$patternArgs + } } } - - end { - # Join all pipeline input into a single string, separated by newlines - `$pipelineContent = `$collector -join "`n" - - # If there's pipeline input, include it in the call to fabric - if (`$pipelineContent) { - `$pipelineContent | fabric --pattern $patternName `$patternArgs - } else { - # No pipeline input; just call fabric with the additional args - fabric --pattern $patternName `$patternArgs - } - } -} -"@ - # Add the function to the current session - Invoke-Expression $functionDefinition -} - -# Define the 'yt' function as well -function yt { - [CmdletBinding()] - param( - [Parameter()] - [Alias("timestamps")] - [switch]$t, - - [Parameter(Position = 0, ValueFromPipeline = $true)] - [string]$videoLink - ) - - begin { - $transcriptFlag = "--transcript" - if ($t) { - $transcriptFlag = "--transcript-with-timestamps" - } + "@ + # Add the function to the current session + Invoke-Expression $functionDefinition } - process { - if (-not $videoLink) { - Write-Error "Usage: yt [-t | --timestamps] youtube-link" - return - } - } + # Define the 'yt' function as well + function yt { + [CmdletBinding()] + param( + [Parameter()] + [Alias("timestamps")] + [switch]$t, - end { - if ($videoLink) { - # Execute and allow output to flow through the pipeline - fabric -y $videoLink $transcriptFlag + [Parameter(Position = 0, ValueFromPipeline = $true)] + [string]$videoLink + ) + + begin { + $transcriptFlag = "--transcript" + if ($t) { + $transcriptFlag = "--transcript-with-timestamps" + } + } + + process { + if (-not $videoLink) { + Write-Error "Usage: yt [-t | --timestamps] youtube-link" + return + } + } + + end { + if ($videoLink) { + # Execute and allow output to flow through the pipeline + fabric -y $videoLink $transcriptFlag + } } } -} -``` + ``` This also creates a `yt` alias that allows you to use `yt https://www.youtube.com/watch?v=4b0iet22VIk` to get transcripts, comments, and metadata. @@ -493,7 +493,6 @@ fabric -h ``` ```plaintext - Usage: fabric [OPTIONS] @@ -508,7 +507,9 @@ Application Options: -T, --topp= Set top P (default: 0.9) -s, --stream Stream -P, --presencepenalty= Set presence penalty (default: 0.0) - -r, --raw Use the defaults of the model without sending chat options (like temperature etc.) and use the user role instead of the system role for patterns. + -r, --raw Use the defaults of the model without sending chat options (like + temperature etc.) and use the user role instead of the system role for + patterns. -F, --frequencypenalty= Set frequency penalty (default: 0.0) -l, --listpatterns List all patterns -L, --listmodels List all available models @@ -522,9 +523,12 @@ Application Options: --output-session Output the entire session (also a temporary one) to the output file -n, --latest= Number of latest patterns to list (default: 0) -d, --changeDefaultModel Change default model - -y, --youtube= YouTube video or play list "URL" to grab transcript, comments from it and send to chat or print it put to the console and store it in the output file + -y, --youtube= YouTube video or play list "URL" to grab transcript, comments from it + and send to chat or print it put to the console and store it in the + output file --playlist Prefer playlist over video if both ids are present in the URL - --transcript Grab transcript from YouTube video and send to chat (it is used per default). + --transcript Grab transcript from YouTube video and send to chat (it is used per + default). --transcript-with-timestamps Grab transcript from YouTube video with timestamps and send to chat --comments Grab comments from YouTube video and send to chat --metadata Output video metadata @@ -552,6 +556,9 @@ Application Options: --liststrategies List all strategies --listvendors List all vendors --shell-complete-list Output raw list without headers/formatting (for shell completion) + --search Enable web search tool for supported models (Anthropic, OpenAI) + --search-location= Set location for web search results (e.g., 'America/Los_Angeles') + --image-file= Save generated image to specified file path (e.g., 'output.png') Help Options: -h, --help Show this help message diff --git a/cli/flags.go b/cli/flags.go index 0aa1be43..72cfceba 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -76,6 +76,7 @@ type Flags struct { ShellCompleteOutput bool `long:"shell-complete-list" description:"Output raw list without headers/formatting (for shell completion)"` Search bool `long:"search" description:"Enable web search tool for supported models (Anthropic, OpenAI)"` SearchLocation string `long:"search-location" description:"Set location for web search results (e.g., 'America/Los_Angeles')"` + ImageFile string `long:"image-file" description:"Save generated image to specified file path (e.g., 'output.png')"` } var debug = false @@ -267,6 +268,7 @@ func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) { ModelContextLength: o.ModelContextLength, Search: o.Search, SearchLocation: o.SearchLocation, + ImageFile: o.ImageFile, } return } diff --git a/common/domain.go b/common/domain.go index 44357e0b..0488eeee 100644 --- a/common/domain.go +++ b/common/domain.go @@ -28,6 +28,7 @@ type ChatOptions struct { MaxTokens int Search bool SearchLocation string + ImageFile string } // NormalizeMessages remove empty messages and ensure messages order user-assist-user diff --git a/plugins/ai/dryrun/dryrun.go b/plugins/ai/dryrun/dryrun.go index c3ff5402..1ca19e64 100644 --- a/plugins/ai/dryrun/dryrun.go +++ b/plugins/ai/dryrun/dryrun.go @@ -82,6 +82,9 @@ func (c *Client) formatOptions(opts *common.ChatOptions) string { builder.WriteString(fmt.Sprintf("SearchLocation: %s\n", opts.SearchLocation)) } } + if opts.ImageFile != "" { + builder.WriteString(fmt.Sprintf("ImageFile: %s\n", opts.ImageFile)) + } return builder.String() } diff --git a/plugins/ai/openai/openai.go b/plugins/ai/openai/openai.go index 24c3d22e..4d9e638c 100644 --- a/plugins/ai/openai/openai.go +++ b/plugins/ai/openai/openai.go @@ -134,6 +134,12 @@ func (o *Client) sendResponses(ctx context.Context, msgs []*chat.ChatCompletionM if resp, err = o.ApiClient.Responses.New(ctx, req); err != nil { return } + + // Extract and save images if requested + if err = o.extractAndSaveImages(resp, opts); err != nil { + return + } + ret = o.extractText(resp) return } @@ -183,6 +189,9 @@ func (o *Client) buildResponseParams( }, } + // Add tools if enabled + var tools []responses.ToolUnionParam + // Add web search tool if enabled if opts.Search { webSearchTool := responses.ToolParamOfWebSearchPreview("web_search_preview") @@ -195,7 +204,14 @@ func (o *Client) buildResponseParams( } } - ret.Tools = []responses.ToolUnionParam{webSearchTool} + tools = append(tools, webSearchTool) + } + + // Add image generation tool if needed + tools = o.addImageGenerationTool(opts, tools) + + if len(tools) > 0 { + ret.Tools = tools } if !opts.Raw { diff --git a/plugins/ai/openai/openai_image.go b/plugins/ai/openai/openai_image.go new file mode 100644 index 00000000..1824d7db --- /dev/null +++ b/plugins/ai/openai/openai_image.go @@ -0,0 +1,77 @@ +package openai + +// This file contains helper methods for image generation and processing +// using OpenAI's Responses API and Image API. + +import ( + "encoding/base64" + "fmt" + "os" + "path/filepath" + + "github.com/danielmiessler/fabric/common" + "github.com/openai/openai-go/responses" +) + +// addImageGenerationTool adds the image generation tool to the request if needed +func (o *Client) addImageGenerationTool(opts *common.ChatOptions, tools []responses.ToolUnionParam) []responses.ToolUnionParam { + // Check if the request seems to be asking for image generation + if o.shouldUseImageGeneration(opts) { + imageGenTool := responses.ToolUnionParam{ + OfImageGeneration: &responses.ToolImageGenerationParam{ + Type: "image_generation", + Model: "gpt-image-1", + OutputFormat: "png", + Quality: "auto", + Size: "auto", + }, + } + tools = append(tools, imageGenTool) + } + return tools +} + +// shouldUseImageGeneration determines if image generation should be enabled +// This is a heuristic based on the presence of --image-file flag +func (o *Client) shouldUseImageGeneration(opts *common.ChatOptions) bool { + return opts.ImageFile != "" +} + +// extractAndSaveImages extracts generated images from the response and saves them +func (o *Client) extractAndSaveImages(resp *responses.Response, opts *common.ChatOptions) error { + if opts.ImageFile == "" { + return nil // No image file specified, skip saving + } + + // Extract image data from response + for _, item := range resp.Output { + if item.Type == "image_generation_call" { + imageCall := item.AsImageGenerationCall() + if imageCall.Status == "completed" && imageCall.Result != "" { + // Decode base64 image data + imageData, err := base64.StdEncoding.DecodeString(imageCall.Result) + if err != nil { + return fmt.Errorf("failed to decode image data: %w", err) + } + + // Ensure directory exists + dir := filepath.Dir(opts.ImageFile) + if dir != "." { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", dir, err) + } + } + + // Save image to file + if err := os.WriteFile(opts.ImageFile, imageData, 0644); err != nil { + return fmt.Errorf("failed to save image to %s: %w", opts.ImageFile, err) + } + + fmt.Printf("Image saved to: %s\n", opts.ImageFile) + return nil + } + } + } + + return nil +} diff --git a/plugins/ai/openai/openai_image_test.go b/plugins/ai/openai/openai_image_test.go new file mode 100644 index 00000000..9a88d543 --- /dev/null +++ b/plugins/ai/openai/openai_image_test.go @@ -0,0 +1,114 @@ +package openai + +import ( + "testing" + + "github.com/danielmiessler/fabric/chat" + "github.com/danielmiessler/fabric/common" + "github.com/openai/openai-go/responses" + "github.com/stretchr/testify/assert" +) + +func TestShouldUseImageGeneration(t *testing.T) { + client := NewClient() + + // Test with image file specified + opts := &common.ChatOptions{ + ImageFile: "output.png", + } + assert.True(t, client.shouldUseImageGeneration(opts), "Should use image generation when image file is specified") + + // Test without image file + opts = &common.ChatOptions{ + ImageFile: "", + } + assert.False(t, client.shouldUseImageGeneration(opts), "Should not use image generation when no image file is specified") +} + +func TestAddImageGenerationTool(t *testing.T) { + client := NewClient() + + // Test with image generation enabled + opts := &common.ChatOptions{ + ImageFile: "output.png", + } + tools := []responses.ToolUnionParam{} + result := client.addImageGenerationTool(opts, tools) + + assert.Len(t, result, 1, "Should add one image generation tool") + assert.NotNil(t, result[0].OfImageGeneration, "Should have image generation tool") + assert.Equal(t, "image_generation", string(result[0].OfImageGeneration.Type)) + assert.Equal(t, "gpt-image-1", result[0].OfImageGeneration.Model) + assert.Equal(t, "png", result[0].OfImageGeneration.OutputFormat) + + // Test without image generation + opts = &common.ChatOptions{ + ImageFile: "", + } + tools = []responses.ToolUnionParam{} + result = client.addImageGenerationTool(opts, tools) + + assert.Len(t, result, 0, "Should not add image generation tool when not needed") +} + +func TestBuildResponseParams_WithImageGeneration(t *testing.T) { + client := NewClient() + opts := &common.ChatOptions{ + Model: "gpt-image-1", + ImageFile: "output.png", + } + + msgs := []*chat.ChatCompletionMessage{ + {Role: "user", Content: "Generate an image of a cat"}, + } + + params := client.buildResponseParams(msgs, opts) + + assert.NotNil(t, params.Tools, "Expected tools when image generation is enabled") + + // Should have image generation tool + hasImageTool := false + for _, tool := range params.Tools { + if tool.OfImageGeneration != nil { + hasImageTool = true + assert.Equal(t, "image_generation", string(tool.OfImageGeneration.Type)) + assert.Equal(t, "gpt-image-1", tool.OfImageGeneration.Model) + break + } + } + assert.True(t, hasImageTool, "Should have image generation tool") +} + +func TestBuildResponseParams_WithBothSearchAndImage(t *testing.T) { + client := NewClient() + opts := &common.ChatOptions{ + Model: "gpt-image-1", + Search: true, + SearchLocation: "America/Los_Angeles", + ImageFile: "output.png", + } + + msgs := []*chat.ChatCompletionMessage{ + {Role: "user", Content: "Search for cat images and generate one"}, + } + + params := client.buildResponseParams(msgs, opts) + + assert.NotNil(t, params.Tools, "Expected tools when both search and image generation are enabled") + assert.Len(t, params.Tools, 2, "Should have both search and image generation tools") + + hasSearchTool := false + hasImageTool := false + + for _, tool := range params.Tools { + if tool.OfWebSearchPreview != nil { + hasSearchTool = true + } + if tool.OfImageGeneration != nil { + hasImageTool = true + } + } + + assert.True(t, hasSearchTool, "Should have web search tool") + assert.True(t, hasImageTool, "Should have image generation tool") +}