mirror of
https://github.com/danielmiessler/Fabric.git
synced 2026-01-11 07:18:03 -05:00
Compare commits
35 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4badaa4c85 | ||
|
|
bf6be964fd | ||
|
|
cdbcb0a512 | ||
|
|
f81cf193a2 | ||
|
|
cba56fcde6 | ||
|
|
72cbd13917 | ||
|
|
dc722f9724 | ||
|
|
1a35f32a48 | ||
|
|
65bd2753c2 | ||
|
|
570c9a9404 | ||
|
|
15151fe9ee | ||
|
|
2aad4caf9b | ||
|
|
289fda8c74 | ||
|
|
fd40778472 | ||
|
|
bc1641a68c | ||
|
|
5cf15d22d3 | ||
|
|
2b2a25daaa | ||
|
|
75a7f25642 | ||
|
|
8bab58f225 | ||
|
|
8ec006e02c | ||
|
|
2508dc6397 | ||
|
|
7670df35ad | ||
|
|
3b9782f942 | ||
|
|
3fca3489fb | ||
|
|
bb2d58eae0 | ||
|
|
87df7dc383 | ||
|
|
1d69afa1c9 | ||
|
|
96c18b4c99 | ||
|
|
dd5173963b | ||
|
|
da1c8ec979 | ||
|
|
ac97f9984f | ||
|
|
181b812eaf | ||
|
|
fe94165d31 | ||
|
|
16e92690aa | ||
|
|
1c33799aa8 |
8
.vscode/settings.json
vendored
8
.vscode/settings.json
vendored
@@ -53,6 +53,7 @@
|
||||
"hasura",
|
||||
"hormozi",
|
||||
"Hormozi's",
|
||||
"horts",
|
||||
"HTMLURL",
|
||||
"jaredmontoya",
|
||||
"jessevdk",
|
||||
@@ -74,10 +75,12 @@
|
||||
"markmap",
|
||||
"matplotlib",
|
||||
"mattn",
|
||||
"mbed",
|
||||
"Miessler",
|
||||
"nometa",
|
||||
"numpy",
|
||||
"ollama",
|
||||
"openaiapi",
|
||||
"opencode",
|
||||
"openrouter",
|
||||
"otiai",
|
||||
@@ -104,6 +107,7 @@
|
||||
"stretchr",
|
||||
"talkpanel",
|
||||
"Telos",
|
||||
"testpattern",
|
||||
"Thacker",
|
||||
"tidwall",
|
||||
"topp",
|
||||
@@ -113,11 +117,13 @@
|
||||
"updatepatterns",
|
||||
"videoid",
|
||||
"webp",
|
||||
"WEBVTT",
|
||||
"wipecontext",
|
||||
"wipesession",
|
||||
"writeups",
|
||||
"xclip",
|
||||
"yourpatternname"
|
||||
"yourpatternname",
|
||||
"youtu"
|
||||
],
|
||||
"cSpell.ignorePaths": ["go.mod", ".gitignore", "CHANGELOG.md"],
|
||||
"markdownlint.config": {
|
||||
|
||||
69
CHANGELOG.md
69
CHANGELOG.md
@@ -1,5 +1,74 @@
|
||||
# Changelog
|
||||
|
||||
## v1.4.259 (2025-07-18)
|
||||
|
||||
### PR [#1633](https://github.com/danielmiessler/Fabric/pull/1633) by [ksylvan](https://github.com/ksylvan): YouTube VTT Processing Enhancement
|
||||
|
||||
- Fix: prevent duplicate segments in VTT file processing by adding deduplication map to track seen segments
|
||||
- Feat: enhance VTT duplicate filtering to allow legitimate repeated content with configurable time gap detection
|
||||
- Feat: improve timestamp parsing to handle fractional seconds and optional seconds/milliseconds formats
|
||||
- Chore: refactor timestamp regex to global scope and improve performance by avoiding repeated compilation
|
||||
- Fix: Youtube VTT parsing gap test and extract seconds parsing logic into reusable function
|
||||
|
||||
### Direct commits
|
||||
|
||||
- Docs: Update CHANGELOG after v1.4.258
|
||||
|
||||
## v1.4.258 (2025-07-17)
|
||||
|
||||
### PR [#1629](https://github.com/danielmiessler/Fabric/pull/1629) by [ksylvan](https://github.com/ksylvan): Create Default (empty) .env in ~/.config/fabric on Demand
|
||||
|
||||
- Add startup check to initialize config and .env file automatically
|
||||
- Introduce ensureEnvFile function to create ~/.config/fabric/.env if missing
|
||||
- Add directory creation for config path in ensureEnvFile
|
||||
- Integrate setup flag in CLI to call ensureEnvFile on demand
|
||||
- Improve error handling and permissions in ensureEnvFile function
|
||||
|
||||
### Direct commits
|
||||
|
||||
- Update README and CHANGELOG after v1.4.257
|
||||
|
||||
## v1.4.257 (2025-07-17)
|
||||
|
||||
### PR [#1628](https://github.com/danielmiessler/Fabric/pull/1628) by [ksylvan](https://github.com/ksylvan): Introduce CLI Flag to Disable OpenAI Responses API
|
||||
|
||||
- Add `--disable-responses-api` CLI flag for OpenAI control and llama-server compatibility
|
||||
- Implement `SetResponsesAPIEnabled` method in OpenAI client with configuration control
|
||||
- Update default config path to `~/.config/fabric/config.yaml`
|
||||
- Add CLI completions for new API flag across zsh, bash, and fish shells
|
||||
- Update CHANGELOG after v1.4.256 release
|
||||
|
||||
## v1.4.256 (2025-07-17)
|
||||
|
||||
### PR [#1624](https://github.com/danielmiessler/Fabric/pull/1624) by [ksylvan](https://github.com/ksylvan): Feature: Add Automatic ~/.fabric.yaml Config Detection
|
||||
|
||||
- Implement default ~/.fabric.yaml config file detection
|
||||
- Add support for short flag parsing with dashes
|
||||
- Improve dry run output formatting and config path error handling
|
||||
- Refactor dry run response construction into helper method
|
||||
- Extract flag parsing logic into separate extractFlag function
|
||||
|
||||
### Direct commits
|
||||
|
||||
- Docs: Update CHANGELOG after v1.4.255
|
||||
|
||||
## v1.4.255 (2025-07-16)
|
||||
|
||||
### Direct commits
|
||||
|
||||
- Merge branch 'danielmiessler:main' into main
|
||||
- Chore: add more paths to update-version-andcreate-tag workflow to reduce unnecessary tagging
|
||||
|
||||
## v1.4.254 (2025-07-16)
|
||||
|
||||
### PR [#1621](https://github.com/danielmiessler/Fabric/pull/1621) by [robertocarvajal](https://github.com/robertocarvajal): Adds generate code rules pattern
|
||||
|
||||
- Adds generate code rules pattern
|
||||
|
||||
### Direct commits
|
||||
|
||||
- Docs: Update CHANGELOG after v1.4.253
|
||||
|
||||
## v1.4.253 (2025-07-16)
|
||||
|
||||
### PR [#1620](https://github.com/danielmiessler/Fabric/pull/1620) by [ksylvan](https://github.com/ksylvan): Update Shell Completions for New Think-Block Suppression Options
|
||||
|
||||
@@ -544,10 +544,13 @@ Application Options:
|
||||
--image-compression= Compression level 0-100 for JPEG/WebP formats (default: not set)
|
||||
--image-background= Background type: opaque, transparent (default: opaque, only for
|
||||
PNG/WebP)
|
||||
--suppress-think Suppress text enclosed in thinking tags
|
||||
--think-start-tag= Start tag for thinking sections (default: <think>)
|
||||
--think-end-tag= End tag for thinking sections (default: </think>)
|
||||
--disable-responses-api Disable OpenAI Responses API (default: false)
|
||||
|
||||
Help Options:
|
||||
-h, --help Show this help message
|
||||
|
||||
```
|
||||
|
||||
## Our approach to prompting
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
package main
|
||||
|
||||
var version = "v1.4.255"
|
||||
var version = "v1.4.260"
|
||||
|
||||
Binary file not shown.
@@ -113,6 +113,7 @@ _fabric() {
|
||||
'(--suppress-think)--suppress-think[Suppress text enclosed in thinking tags]' \
|
||||
'(--think-start-tag)--think-start-tag[Start tag for thinking sections (default: <think>)]:start tag:' \
|
||||
'(--think-end-tag)--think-end-tag[End tag for thinking sections (default: </think>)]:end tag:' \
|
||||
'(--disable-responses-api)--disable-responses-api[Disable OpenAI Responses API (default: false)]' \
|
||||
'(-h --help)'{-h,--help}'[Show this help message]' \
|
||||
'*:arguments:'
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ _fabric() {
|
||||
_get_comp_words_by_ref -n : cur prev words cword
|
||||
|
||||
# Define all possible options/flags
|
||||
local opts="--pattern -p --variable -v --context -C --session --attachment -a --setup -S --temperature -t --topp -T --stream -s --presencepenalty -P --raw -r --frequencypenalty -F --listpatterns -l --listmodels -L --listcontexts -x --listsessions -X --updatepatterns -U --copy -c --model -m --modelContextLength --output -o --output-session --latest -n --changeDefaultModel -d --youtube -y --playlist --transcript --transcript-with-timestamps --comments --metadata --language -g --scrape_url -u --scrape_question -q --seed -e --wipecontext -w --wipesession -W --printcontext --printsession --readability --input-has-vars --dry-run --serve --serveOllama --address --api-key --config --search --search-location --image-file --image-size --image-quality --image-compression --image-background --suppress-think --think-start-tag --think-end-tag --version --listextensions --addextension --rmextension --strategy --liststrategies --listvendors --shell-complete-list --help -h"
|
||||
local opts="--pattern -p --variable -v --context -C --session --attachment -a --setup -S --temperature -t --topp -T --stream -s --presencepenalty -P --raw -r --frequencypenalty -F --listpatterns -l --listmodels -L --listcontexts -x --listsessions -X --updatepatterns -U --copy -c --model -m --modelContextLength --output -o --output-session --latest -n --changeDefaultModel -d --youtube -y --playlist --transcript --transcript-with-timestamps --comments --metadata --language -g --scrape_url -u --scrape_question -q --seed -e --wipecontext -w --wipesession -W --printcontext --printsession --readability --input-has-vars --dry-run --serve --serveOllama --address --api-key --config --search --search-location --image-file --image-size --image-quality --image-compression --image-background --suppress-think --think-start-tag --think-end-tag --disable-responses-api --version --listextensions --addextension --rmextension --strategy --liststrategies --listvendors --shell-complete-list --help -h"
|
||||
|
||||
# Helper function for dynamic completions
|
||||
_fabric_get_list() {
|
||||
|
||||
@@ -101,4 +101,5 @@ complete -c fabric -l liststrategies -d "List all strategies"
|
||||
complete -c fabric -l listvendors -d "List all vendors"
|
||||
complete -c fabric -l shell-complete-list -d "Output raw list without headers/formatting (for shell completion)"
|
||||
complete -c fabric -l suppress-think -d "Suppress text enclosed in thinking tags"
|
||||
complete -c fabric -l disable-responses-api -d "Disable OpenAI Responses API (default: false)"
|
||||
complete -c fabric -s h -l help -d "Show this help message"
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/danielmiessler/fabric/internal/core"
|
||||
"github.com/danielmiessler/fabric/internal/plugins/ai/openai"
|
||||
"github.com/danielmiessler/fabric/internal/tools/converter"
|
||||
"github.com/danielmiessler/fabric/internal/tools/youtube"
|
||||
)
|
||||
@@ -18,6 +19,12 @@ func Cli(version string) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.Setup {
|
||||
if err = ensureEnvFile(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if currentFlags.Version {
|
||||
fmt.Println(version)
|
||||
return
|
||||
@@ -36,6 +43,11 @@ func Cli(version string) (err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Configure OpenAI Responses API setting based on CLI flag
|
||||
if registry != nil {
|
||||
configureOpenAIResponsesAPI(registry, currentFlags.DisableResponsesAPI)
|
||||
}
|
||||
|
||||
// Handle setup and server commands
|
||||
var handled bool
|
||||
if handled, err = handleSetupAndServerCommands(currentFlags, registry, version); err != nil || handled {
|
||||
@@ -142,3 +154,21 @@ func WriteOutput(message string, outputFile string) (err error) {
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// configureOpenAIResponsesAPI configures the OpenAI client's Responses API setting based on the CLI flag
|
||||
func configureOpenAIResponsesAPI(registry *core.PluginRegistry, disableResponsesAPI bool) {
|
||||
// Find the OpenAI vendor in the registry
|
||||
if registry != nil && registry.VendorsAll != nil {
|
||||
for _, vendor := range registry.VendorsAll.Vendors {
|
||||
if vendor.GetName() == "OpenAI" {
|
||||
// Type assertion to access the OpenAI-specific method
|
||||
if openaiClient, ok := vendor.(*openai.Client); ok {
|
||||
// Invert the disable flag to get the enable flag
|
||||
enableResponsesAPI := !disableResponsesAPI
|
||||
openaiClient.SetResponsesAPIEnabled(enableResponsesAPI)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,3 +24,7 @@ raw: false
|
||||
suppressThink: false
|
||||
thinkStartTag: "<think>"
|
||||
thinkEndTag: "</think>"
|
||||
|
||||
# OpenAI Responses API settings
|
||||
# (use this for llama-server or other OpenAI-compatible local servers)
|
||||
disableResponsesAPI: true
|
||||
|
||||
@@ -86,6 +86,7 @@ type Flags struct {
|
||||
SuppressThink bool `long:"suppress-think" yaml:"suppressThink" description:"Suppress text enclosed in thinking tags"`
|
||||
ThinkStartTag string `long:"think-start-tag" yaml:"thinkStartTag" description:"Start tag for thinking sections" default:"<think>"`
|
||||
ThinkEndTag string `long:"think-end-tag" yaml:"thinkEndTag" description:"End tag for thinking sections" default:"</think>"`
|
||||
DisableResponsesAPI bool `long:"disable-responses-api" yaml:"disableResponsesAPI" description:"Disable OpenAI Responses API (default: false)"`
|
||||
}
|
||||
|
||||
var debug = false
|
||||
@@ -102,26 +103,34 @@ func Init() (ret *Flags, err error) {
|
||||
usedFlags := make(map[string]bool)
|
||||
yamlArgsScan := os.Args[1:]
|
||||
|
||||
// Get list of fields that have yaml tags, could be in yaml config
|
||||
yamlFields := make(map[string]bool)
|
||||
// Create mapping from flag names (both short and long) to yaml tag names
|
||||
flagToYamlTag := make(map[string]string)
|
||||
t := reflect.TypeOf(Flags{})
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
if yamlTag := t.Field(i).Tag.Get("yaml"); yamlTag != "" {
|
||||
yamlFields[yamlTag] = true
|
||||
//Debugf("Found yaml-configured field: %s\n", yamlTag)
|
||||
field := t.Field(i)
|
||||
yamlTag := field.Tag.Get("yaml")
|
||||
if yamlTag != "" {
|
||||
longTag := field.Tag.Get("long")
|
||||
shortTag := field.Tag.Get("short")
|
||||
if longTag != "" {
|
||||
flagToYamlTag[longTag] = yamlTag
|
||||
Debugf("Mapped long flag %s to yaml tag %s\n", longTag, yamlTag)
|
||||
}
|
||||
if shortTag != "" {
|
||||
flagToYamlTag[shortTag] = yamlTag
|
||||
Debugf("Mapped short flag %s to yaml tag %s\n", shortTag, yamlTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scan args for that are provided by cli and might be in yaml
|
||||
for _, arg := range yamlArgsScan {
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
flag := strings.TrimPrefix(arg, "--")
|
||||
if i := strings.Index(flag, "="); i > 0 {
|
||||
flag = flag[:i]
|
||||
}
|
||||
if yamlFields[flag] {
|
||||
usedFlags[flag] = true
|
||||
Debugf("CLI flag used: %s\n", flag)
|
||||
flag := extractFlag(arg)
|
||||
|
||||
if flag != "" {
|
||||
if yamlTag, exists := flagToYamlTag[flag]; exists {
|
||||
usedFlags[yamlTag] = true
|
||||
Debugf("CLI flag used: %s (yaml: %s)\n", flag, yamlTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -134,6 +143,16 @@ func Init() (ret *Flags, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check to see if a ~/.config/fabric/config.yaml config file exists (only when user didn't specify a config)
|
||||
if ret.Config == "" {
|
||||
// Default to ~/.config/fabric/config.yaml if no config specified
|
||||
if defaultConfigPath, err := util.GetDefaultConfigPath(); err == nil && defaultConfigPath != "" {
|
||||
ret.Config = defaultConfigPath
|
||||
} else if err != nil {
|
||||
Debugf("Could not determine default config path: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// If config specified, load and apply YAML for unused flags
|
||||
if ret.Config != "" {
|
||||
var yamlFlags *Flags
|
||||
@@ -168,7 +187,6 @@ func Init() (ret *Flags, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Handle stdin and messages
|
||||
// Handle stdin and messages
|
||||
info, _ := os.Stdin.Stat()
|
||||
pipedToStdin := (info.Mode() & os.ModeCharDevice) == 0
|
||||
@@ -188,6 +206,22 @@ func Init() (ret *Flags, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func extractFlag(arg string) string {
|
||||
var flag string
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
flag = strings.TrimPrefix(arg, "--")
|
||||
if i := strings.Index(flag, "="); i > 0 {
|
||||
flag = flag[:i]
|
||||
}
|
||||
} else if strings.HasPrefix(arg, "-") && len(arg) > 1 {
|
||||
flag = strings.TrimPrefix(arg, "-")
|
||||
if i := strings.Index(flag, "="); i > 0 {
|
||||
flag = flag[:i]
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
func assignWithConversion(targetField, sourceField reflect.Value) error {
|
||||
// Handle string source values
|
||||
if sourceField.Kind() == reflect.String {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
@@ -8,6 +9,9 @@ import (
|
||||
"github.com/danielmiessler/fabric/internal/plugins/db/fsdb"
|
||||
)
|
||||
|
||||
const ConfigDirPerms os.FileMode = 0755
|
||||
const EnvFilePerms os.FileMode = 0644
|
||||
|
||||
// initializeFabric initializes the fabric database and plugin registry
|
||||
func initializeFabric() (registry *core.PluginRegistry, err error) {
|
||||
var homedir string
|
||||
@@ -26,3 +30,27 @@ func initializeFabric() (registry *core.PluginRegistry, err error) {
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// ensureEnvFile checks for the default ~/.config/fabric/.env file and creates it
|
||||
// along with the parent directory if it does not exist.
|
||||
func ensureEnvFile() (err error) {
|
||||
var homedir string
|
||||
if homedir, err = os.UserHomeDir(); err != nil {
|
||||
return fmt.Errorf("could not determine user home directory: %w", err)
|
||||
}
|
||||
configDir := filepath.Join(homedir, ".config", "fabric")
|
||||
envPath := filepath.Join(configDir, ".env")
|
||||
|
||||
if _, statErr := os.Stat(envPath); statErr != nil {
|
||||
if !os.IsNotExist(statErr) {
|
||||
return fmt.Errorf("could not stat .env file: %w", statErr)
|
||||
}
|
||||
if err = os.MkdirAll(configDir, ConfigDirPerms); err != nil {
|
||||
return fmt.Errorf("could not create config directory: %w", err)
|
||||
}
|
||||
if err = os.WriteFile(envPath, []byte{}, EnvFilePerms); err != nil {
|
||||
return fmt.Errorf("could not create .env file: %w", err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ func (o *Chatter) Send(request *domain.ChatRequest, opts *domain.ChatOptions) (s
|
||||
}
|
||||
}
|
||||
|
||||
if opts.SuppressThink {
|
||||
if opts.SuppressThink && !o.DryRun {
|
||||
message = domain.StripThinkBlocks(message, opts.ThinkStartTag, opts.ThinkEndTag)
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"github.com/danielmiessler/fabric/internal/plugins"
|
||||
)
|
||||
|
||||
const DryRunResponse = "Dry run: Fake response sent by DryRun plugin\n"
|
||||
|
||||
type Client struct {
|
||||
*plugins.PluginBase
|
||||
}
|
||||
@@ -85,27 +87,37 @@ func (c *Client) formatOptions(opts *domain.ChatOptions) string {
|
||||
if opts.ImageFile != "" {
|
||||
builder.WriteString(fmt.Sprintf("ImageFile: %s\n", opts.ImageFile))
|
||||
}
|
||||
if opts.SuppressThink {
|
||||
builder.WriteString("SuppressThink: enabled\n")
|
||||
builder.WriteString(fmt.Sprintf("Thinking Start Tag: %s\n", opts.ThinkStartTag))
|
||||
builder.WriteString(fmt.Sprintf("Thinking End Tag: %s\n", opts.ThinkEndTag))
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func (c *Client) SendStream(msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions, channel chan string) error {
|
||||
func (c *Client) constructRequest(msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString("Dry run: Would send the following request:\n\n")
|
||||
builder.WriteString(c.formatMessages(msgs))
|
||||
builder.WriteString(c.formatOptions(opts))
|
||||
|
||||
channel <- builder.String()
|
||||
close(channel)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func (c *Client) SendStream(msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions, channel chan string) error {
|
||||
defer close(channel)
|
||||
request := c.constructRequest(msgs, opts)
|
||||
channel <- request
|
||||
channel <- "\n"
|
||||
channel <- DryRunResponse
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) Send(_ context.Context, msgs []*chat.ChatCompletionMessage, opts *domain.ChatOptions) (string, error) {
|
||||
fmt.Println("Dry run: Would send the following request:")
|
||||
fmt.Print(c.formatMessages(msgs))
|
||||
fmt.Print(c.formatOptions(opts))
|
||||
request := c.constructRequest(msgs, opts)
|
||||
|
||||
return "", nil
|
||||
return request + "\n" + DryRunResponse, nil
|
||||
}
|
||||
|
||||
func (c *Client) Setup() error {
|
||||
|
||||
@@ -13,6 +13,7 @@ func NewClient() (ret *Client) {
|
||||
ret = &Client{}
|
||||
ret.Client = openai.NewClientCompatibleNoSetupQuestions("Exolab", ret.configure)
|
||||
|
||||
ret.ApiKey = ret.AddSetupQuestion("API Key", false)
|
||||
ret.ApiBaseURL = ret.AddSetupQuestion("API Base URL", true)
|
||||
ret.ApiBaseURL.Value = "http://localhost:52415"
|
||||
|
||||
|
||||
@@ -66,6 +66,11 @@ type Client struct {
|
||||
ImplementsResponses bool // Whether this provider supports the Responses API
|
||||
}
|
||||
|
||||
// SetResponsesAPIEnabled configures whether to use the Responses API
|
||||
func (o *Client) SetResponsesAPIEnabled(enabled bool) {
|
||||
o.ImplementsResponses = enabled
|
||||
}
|
||||
|
||||
func (o *Client) configure() (ret error) {
|
||||
opts := []option.RequestOption{option.WithAPIKey(o.ApiKey.Value)}
|
||||
if o.ApiBaseURL.Value != "" {
|
||||
|
||||
61
internal/tools/youtube/timestamp_test.go
Normal file
61
internal/tools/youtube/timestamp_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package youtube
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseTimestampToSeconds(t *testing.T) {
|
||||
tests := []struct {
|
||||
timestamp string
|
||||
expected int
|
||||
shouldErr bool
|
||||
}{
|
||||
{"00:30", 30, false},
|
||||
{"01:30", 90, false},
|
||||
{"01:05:30", 3930, false}, // 1 hour 5 minutes 30 seconds
|
||||
{"10:00", 600, false},
|
||||
{"invalid", 0, true},
|
||||
{"1:2:3:4", 0, true}, // too many parts
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result, err := parseTimestampToSeconds(test.timestamp)
|
||||
|
||||
if test.shouldErr {
|
||||
if err == nil {
|
||||
t.Errorf("Expected error for timestamp %s, but got none", test.timestamp)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error for timestamp %s: %v", test.timestamp, err)
|
||||
}
|
||||
if result != test.expected {
|
||||
t.Errorf("For timestamp %s, expected %d seconds, got %d", test.timestamp, test.expected, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldIncludeRepeat(t *testing.T) {
|
||||
tests := []struct {
|
||||
lastTimestamp string
|
||||
currentTimestamp string
|
||||
expected bool
|
||||
description string
|
||||
}{
|
||||
{"00:30", "01:30", true, "60 second gap should allow repeat"},
|
||||
{"00:30", "00:45", true, "15 second gap should allow repeat"},
|
||||
{"01:00", "01:10", true, "10 second gap should allow repeat (boundary case)"},
|
||||
{"01:00", "01:09", false, "9 second gap should not allow repeat"},
|
||||
{"00:30", "00:35", false, "5 second gap should not allow repeat"},
|
||||
{"invalid", "01:30", true, "invalid timestamp should err on side of inclusion"},
|
||||
{"01:30", "invalid", true, "invalid timestamp should err on side of inclusion"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result := shouldIncludeRepeat(test.lastTimestamp, test.currentTimestamp)
|
||||
if result != test.expected {
|
||||
t.Errorf("%s: expected %v, got %v", test.description, test.expected, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,15 @@ import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
var timestampRegex *regexp.Regexp
|
||||
|
||||
const TimeGapForRepeats = 10 // seconds
|
||||
|
||||
func init() {
|
||||
// Match timestamps like "00:00:01.234" or just numbers or sequence numbers
|
||||
timestampRegex = regexp.MustCompile(`^\d+$|^\d{1,2}:\d{2}(:\d{2})?(\.\d{3})?$`)
|
||||
}
|
||||
|
||||
func NewYouTube() (ret *YouTube) {
|
||||
|
||||
label := "YouTube"
|
||||
@@ -180,6 +189,7 @@ func (o *YouTube) readAndCleanVTTFile(filename string) (ret string, err error) {
|
||||
// Convert VTT to plain text
|
||||
lines := strings.Split(string(content), "\n")
|
||||
var textBuilder strings.Builder
|
||||
seenSegments := make(map[string]struct{})
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
@@ -193,8 +203,11 @@ func (o *YouTube) readAndCleanVTTFile(filename string) (ret string, err error) {
|
||||
// Remove VTT formatting tags
|
||||
line = removeVTTTags(line)
|
||||
if line != "" {
|
||||
textBuilder.WriteString(line)
|
||||
textBuilder.WriteString(" ")
|
||||
if _, exists := seenSegments[line]; !exists {
|
||||
textBuilder.WriteString(line)
|
||||
textBuilder.WriteString(" ")
|
||||
seenSegments[line] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -215,6 +228,10 @@ func (o *YouTube) readAndFormatVTTWithTimestamps(filename string) (ret string, e
|
||||
lines := strings.Split(string(content), "\n")
|
||||
var textBuilder strings.Builder
|
||||
var currentTimestamp string
|
||||
// Track content with timestamps to allow repeats after significant time gaps
|
||||
// This preserves legitimate repeated content (choruses, recurring phrases, etc.)
|
||||
// while still filtering out immediate duplicates from VTT formatting issues
|
||||
seenSegments := make(map[string]string) // text -> last timestamp seen
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
@@ -246,7 +263,20 @@ func (o *YouTube) readAndFormatVTTWithTimestamps(filename string) (ret string, e
|
||||
// Remove VTT formatting tags
|
||||
cleanText := removeVTTTags(line)
|
||||
if cleanText != "" && currentTimestamp != "" {
|
||||
textBuilder.WriteString(fmt.Sprintf("[%s] %s\n", currentTimestamp, cleanText))
|
||||
// Check if we should include this segment
|
||||
shouldInclude := true
|
||||
if lastTimestamp, exists := seenSegments[cleanText]; exists {
|
||||
// Calculate time difference to determine if this is a legitimate repeat
|
||||
if !shouldIncludeRepeat(lastTimestamp, currentTimestamp) {
|
||||
shouldInclude = false
|
||||
}
|
||||
}
|
||||
|
||||
if shouldInclude {
|
||||
timestampedLine := fmt.Sprintf("[%s] %s", currentTimestamp, cleanText)
|
||||
textBuilder.WriteString(timestampedLine + "\n")
|
||||
seenSegments[cleanText] = currentTimestamp
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -268,8 +298,6 @@ func formatVTTTimestamp(vttTime string) string {
|
||||
}
|
||||
|
||||
func isTimeStamp(s string) bool {
|
||||
// Match timestamps like "00:00:01.234" or just numbers
|
||||
timestampRegex := regexp.MustCompile(`^\d+$|^\d{2}:\d{2}:\d{2}`)
|
||||
return timestampRegex.MatchString(s)
|
||||
}
|
||||
|
||||
@@ -279,6 +307,76 @@ func removeVTTTags(s string) string {
|
||||
return tagRegex.ReplaceAllString(s, "")
|
||||
}
|
||||
|
||||
// shouldIncludeRepeat determines if repeated content should be included based on time gap
|
||||
func shouldIncludeRepeat(lastTimestamp, currentTimestamp string) bool {
|
||||
// Parse timestamps to calculate time difference
|
||||
lastSeconds, err1 := parseTimestampToSeconds(lastTimestamp)
|
||||
currentSeconds, err2 := parseTimestampToSeconds(currentTimestamp)
|
||||
|
||||
if err1 != nil || err2 != nil {
|
||||
// If we can't parse timestamps, err on the side of inclusion
|
||||
return true
|
||||
}
|
||||
|
||||
// Allow repeats if there's at least a TimeGapForRepeats gap
|
||||
// This threshold can be adjusted based on use case:
|
||||
// - 10 seconds works well for most content
|
||||
// - Could be made configurable in the future
|
||||
timeDiffSeconds := currentSeconds - lastSeconds
|
||||
return timeDiffSeconds >= TimeGapForRepeats
|
||||
}
|
||||
|
||||
// parseTimestampToSeconds converts timestamp string (HH:MM:SS or MM:SS) to total seconds
|
||||
func parseTimestampToSeconds(timestamp string) (int, error) {
|
||||
parts := strings.Split(timestamp, ":")
|
||||
if len(parts) < 2 || len(parts) > 3 {
|
||||
return 0, fmt.Errorf("invalid timestamp format: %s", timestamp)
|
||||
}
|
||||
|
||||
var hours, minutes, seconds int
|
||||
var err error
|
||||
|
||||
if len(parts) == 3 {
|
||||
// HH:MM:SS format
|
||||
if hours, err = strconv.Atoi(parts[0]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if minutes, err = strconv.Atoi(parts[1]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if seconds, err = parseSeconds(parts[2]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
// MM:SS format
|
||||
if minutes, err = strconv.Atoi(parts[0]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if seconds, err = parseSeconds(parts[1]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return hours*3600 + minutes*60 + seconds, nil
|
||||
}
|
||||
|
||||
func parseSeconds(seconds_str string) (int, error) {
|
||||
var seconds int
|
||||
var err error
|
||||
if strings.Contains(seconds_str, ".") {
|
||||
// Handle fractional seconds
|
||||
second_parts := strings.Split(seconds_str, ".")
|
||||
if seconds, err = strconv.Atoi(second_parts[0]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
if seconds, err = strconv.Atoi(seconds_str); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return seconds, nil
|
||||
}
|
||||
|
||||
func (o *YouTube) GrabComments(videoId string) (ret []string, err error) {
|
||||
if err = o.initService(); err != nil {
|
||||
return
|
||||
|
||||
@@ -71,3 +71,21 @@ func IsSymlinkToDir(path string) bool {
|
||||
|
||||
return false // Regular directories should not be treated as symlinks
|
||||
}
|
||||
|
||||
// GetDefaultConfigPath returns the default path for the configuration file
|
||||
// if it exists, otherwise returns an empty string.
|
||||
func GetDefaultConfigPath() (string, error) {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not determine user home directory: %w", err)
|
||||
}
|
||||
|
||||
defaultConfigPath := filepath.Join(homeDir, ".config", "fabric", "config.yaml")
|
||||
if _, err := os.Stat(defaultConfigPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", nil // Return no error for non-existent config path
|
||||
}
|
||||
return "", fmt.Errorf("error accessing default config path: %w", err)
|
||||
}
|
||||
return defaultConfigPath, nil
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
"1.4.255"
|
||||
"1.4.260"
|
||||
|
||||
Reference in New Issue
Block a user