diff --git a/core/plugin_registry.go b/core/plugin_registry.go index a3abb5b5..c88f1b1c 100644 --- a/core/plugin_registry.go +++ b/core/plugin_registry.go @@ -31,6 +31,7 @@ import ( "github.com/danielmiessler/fabric/plugins/db/fsdb" "github.com/danielmiessler/fabric/plugins/template" "github.com/danielmiessler/fabric/plugins/tools" + "github.com/danielmiessler/fabric/plugins/tools/custom_patterns" "github.com/danielmiessler/fabric/plugins/tools/jina" "github.com/danielmiessler/fabric/plugins/tools/lang" "github.com/danielmiessler/fabric/plugins/tools/youtube" @@ -69,6 +70,7 @@ func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry, err error) { VendorManager: ai.NewVendorsManager(), VendorsAll: ai.NewVendorsManager(), PatternsLoader: tools.NewPatternsLoader(db.Patterns), + CustomPatterns: custom_patterns.NewCustomPatterns(), YouTube: youtube.NewYouTube(), Language: lang.NewLanguage(), Jina: jina.NewClient(), @@ -138,6 +140,7 @@ type PluginRegistry struct { VendorsAll *ai.VendorsManager Defaults *tools.Defaults PatternsLoader *tools.PatternsLoader + CustomPatterns *custom_patterns.CustomPatterns YouTube *youtube.YouTube Language *lang.Language Jina *jina.Client @@ -151,6 +154,7 @@ func (o *PluginRegistry) SaveEnvFile() (err error) { o.Defaults.Settings.FillEnvFileContent(&envFileContent) o.PatternsLoader.SetupFillEnvFileContent(&envFileContent) + o.CustomPatterns.SetupFillEnvFileContent(&envFileContent) o.Strategies.SetupFillEnvFileContent(&envFileContent) for _, vendor := range o.VendorManager.Vendors { @@ -183,7 +187,7 @@ func (o *PluginRegistry) Setup() (err error) { return vendor })...) - groupsPlugins.AddGroupItems("Tools", o.Defaults, o.Jina, o.Language, o.PatternsLoader, o.Strategies, o.YouTube) + groupsPlugins.AddGroupItems("Tools", o.CustomPatterns, o.Defaults, o.Jina, o.Language, o.PatternsLoader, o.Strategies, o.YouTube) for { groupsPlugins.Print(false) diff --git a/plugins/db/fsdb/db.go b/plugins/db/fsdb/db.go index 6ee8534f..e2f1451e 100644 --- a/plugins/db/fsdb/db.go +++ b/plugins/db/fsdb/db.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/joho/godotenv" @@ -15,10 +16,22 @@ func NewDb(dir string) (db *Db) { db.EnvFilePath = db.FilePath(".env") + // Check for custom patterns directory from environment variable + customPatternsDir := os.Getenv("CUSTOM_PATTERNS_DIRECTORY") + if customPatternsDir != "" { + // Expand home directory if needed + if strings.HasPrefix(customPatternsDir, "~/") { + if homeDir, err := os.UserHomeDir(); err == nil { + customPatternsDir = filepath.Join(homeDir, customPatternsDir[2:]) + } + } + } + db.Patterns = &PatternsEntity{ StorageEntity: &StorageEntity{Label: "Patterns", Dir: db.FilePath("patterns"), ItemIsDir: true}, SystemPatternFile: "system.md", UniquePatternsFilePath: db.FilePath("unique_patterns.txt"), + CustomPatternsDir: customPatternsDir, } db.Sessions = &SessionsEntity{ diff --git a/plugins/db/fsdb/patterns.go b/plugins/db/fsdb/patterns.go index 01b0e7a9..b1698d5b 100644 --- a/plugins/db/fsdb/patterns.go +++ b/plugins/db/fsdb/patterns.go @@ -16,6 +16,7 @@ type PatternsEntity struct { *StorageEntity SystemPatternFile string UniquePatternsFilePath string + CustomPatternsDir string } // Pattern represents a single pattern with its metadata @@ -43,7 +44,7 @@ func (o *PatternsEntity) GetApplyVariables( } // Use the resolved absolute path to get the pattern - pattern, err = o.getFromFile(absPath) + pattern, _ = o.getFromFile(absPath) } else { // Otherwise, get the pattern from the database pattern, err = o.getFromDB(source) @@ -89,6 +90,19 @@ func (o *PatternsEntity) applyVariables( // retrieves a pattern from the database by name func (o *PatternsEntity) getFromDB(name string) (ret *Pattern, err error) { + // First check custom patterns directory if it exists + if o.CustomPatternsDir != "" { + customPatternPath := filepath.Join(o.CustomPatternsDir, name, o.SystemPatternFile) + if pattern, customErr := os.ReadFile(customPatternPath); customErr == nil { + ret = &Pattern{ + Name: name, + Pattern: string(pattern), + } + return ret, nil + } + } + + // Fallback to main patterns directory patternPath := filepath.Join(o.Dir, name, o.SystemPatternFile) var pattern []byte @@ -145,6 +159,48 @@ func (o *PatternsEntity) getFromFile(pathStr string) (pattern *Pattern, err erro return } +// GetNames overrides StorageEntity.GetNames to include custom patterns directory +func (o *PatternsEntity) GetNames() (ret []string, err error) { + // Get names from main patterns directory + mainNames, err := o.StorageEntity.GetNames() + if err != nil { + return nil, err + } + + // Create a map to track unique pattern names (custom patterns override main ones) + nameMap := make(map[string]bool) + for _, name := range mainNames { + nameMap[name] = true + } + + // Get names from custom patterns directory if it exists + if o.CustomPatternsDir != "" { + // Create a temporary StorageEntity for the custom directory + customStorage := &StorageEntity{ + Dir: o.CustomPatternsDir, + ItemIsDir: o.StorageEntity.ItemIsDir, + FileExtension: o.StorageEntity.FileExtension, + } + + customNames, customErr := customStorage.GetNames() + if customErr == nil { + // Add custom patterns, they will override main patterns with same name + for _, name := range customNames { + nameMap[name] = true + } + } + // Ignore errors from custom directory (it might not exist) + } + + // Convert map keys back to slice + ret = make([]string, 0, len(nameMap)) + for name := range nameMap { + ret = append(ret, name) + } + + return ret, nil +} + // Get required for Storage interface func (o *PatternsEntity) Get(name string) (*Pattern, error) { // Use GetPattern with no variables diff --git a/plugins/db/fsdb/patterns_test.go b/plugins/db/fsdb/patterns_test.go index 47427440..650b2247 100644 --- a/plugins/db/fsdb/patterns_test.go +++ b/plugins/db/fsdb/patterns_test.go @@ -162,3 +162,123 @@ func TestPatternsEntity_Save(t *testing.T) { require.NoError(t, err) assert.Equal(t, content, data) } + +func TestPatternsEntity_CustomPatterns(t *testing.T) { + // Create main patterns directory + mainDir, err := os.MkdirTemp("", "test-main-patterns-*") + require.NoError(t, err) + defer os.RemoveAll(mainDir) + + // Create custom patterns directory + customDir, err := os.MkdirTemp("", "test-custom-patterns-*") + require.NoError(t, err) + defer os.RemoveAll(customDir) + + entity := &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: mainDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + CustomPatternsDir: customDir, + } + + // Create a pattern in main directory + createTestPattern(t, &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: mainDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + }, "main-pattern", "Main pattern content") + + // Create a pattern in custom directory + createTestPattern(t, &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: customDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + }, "custom-pattern", "Custom pattern content") + + // Create a pattern with same name in both directories (custom should override) + createTestPattern(t, &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: mainDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + }, "shared-pattern", "Main shared pattern") + + createTestPattern(t, &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: customDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + }, "shared-pattern", "Custom shared pattern") + + // Test GetNames includes both directories + names, err := entity.GetNames() + require.NoError(t, err) + assert.Contains(t, names, "main-pattern") + assert.Contains(t, names, "custom-pattern") + assert.Contains(t, names, "shared-pattern") + + // Test that custom pattern overrides main pattern + pattern, err := entity.getFromDB("shared-pattern") + require.NoError(t, err) + assert.Equal(t, "Custom shared pattern", pattern.Pattern) + + // Test that main pattern is accessible when not overridden + pattern, err = entity.getFromDB("main-pattern") + require.NoError(t, err) + assert.Equal(t, "Main pattern content", pattern.Pattern) + + // Test that custom pattern is accessible + pattern, err = entity.getFromDB("custom-pattern") + require.NoError(t, err) + assert.Equal(t, "Custom pattern content", pattern.Pattern) +} + +func TestPatternsEntity_CustomPatternsEmpty(t *testing.T) { + // Test behavior when custom patterns directory is empty or doesn't exist + mainDir, err := os.MkdirTemp("", "test-main-patterns-*") + require.NoError(t, err) + defer os.RemoveAll(mainDir) + + entity := &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: mainDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + CustomPatternsDir: "/nonexistent/directory", + } + + // Create a pattern in main directory + createTestPattern(t, &PatternsEntity{ + StorageEntity: &StorageEntity{ + Dir: mainDir, + Label: "patterns", + ItemIsDir: true, + }, + SystemPatternFile: "system.md", + }, "main-pattern", "Main pattern content") + + // Test GetNames works even with nonexistent custom directory + names, err := entity.GetNames() + require.NoError(t, err) + assert.Contains(t, names, "main-pattern") + + // Test that main pattern is accessible + pattern, err := entity.getFromDB("main-pattern") + require.NoError(t, err) + assert.Equal(t, "Main pattern content", pattern.Pattern) +} diff --git a/plugins/tools/custom_patterns/custom_patterns.go b/plugins/tools/custom_patterns/custom_patterns.go new file mode 100644 index 00000000..ee5898c4 --- /dev/null +++ b/plugins/tools/custom_patterns/custom_patterns.go @@ -0,0 +1,61 @@ +package custom_patterns + +import ( + "os" + "path/filepath" + "strings" + + "github.com/danielmiessler/fabric/plugins" +) + +func NewCustomPatterns() (ret *CustomPatterns) { + label := "Custom Patterns" + ret = &CustomPatterns{} + + ret.PluginBase = &plugins.PluginBase{ + Name: label, + SetupDescription: "Custom Patterns - Set directory for your custom patterns (optional)", + EnvNamePrefix: plugins.BuildEnvVariablePrefix(label), + ConfigureCustom: ret.configure, + } + + ret.CustomPatternsDir = ret.AddSetupQuestionCustom("Directory", false, + "Enter the path to your custom patterns directory (leave empty to skip)") + + return +} + +type CustomPatterns struct { + *plugins.PluginBase + CustomPatternsDir *plugins.SetupQuestion +} + +func (o *CustomPatterns) configure() error { + if o.CustomPatternsDir.Value != "" { + // Expand home directory if needed + if strings.HasPrefix(o.CustomPatternsDir.Value, "~/") { + if homeDir, err := os.UserHomeDir(); err == nil { + o.CustomPatternsDir.Value = filepath.Join(homeDir, o.CustomPatternsDir.Value[2:]) + } + } + + // Convert to absolute path + if absPath, err := filepath.Abs(o.CustomPatternsDir.Value); err == nil { + o.CustomPatternsDir.Value = absPath + } + + // Create the directory if it doesn't exist + if err := os.MkdirAll(o.CustomPatternsDir.Value, 0755); err != nil { + // If we can't create it, clear the value to avoid errors + o.CustomPatternsDir.Value = "" + } + } + + return nil +} + +// IsConfigured returns true if a custom patterns directory has been set +func (o *CustomPatterns) IsConfigured() bool { + // Check if the plugin has been configured with a directory + return o.CustomPatternsDir.Value != "" +} diff --git a/plugins/tools/custom_patterns/custom_patterns_test.go b/plugins/tools/custom_patterns/custom_patterns_test.go new file mode 100644 index 00000000..c76cb817 --- /dev/null +++ b/plugins/tools/custom_patterns/custom_patterns_test.go @@ -0,0 +1,79 @@ +package custom_patterns + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewCustomPatterns(t *testing.T) { + plugin := NewCustomPatterns() + + assert.NotNil(t, plugin) + assert.Equal(t, "Custom Patterns", plugin.GetName()) + assert.Equal(t, "Custom Patterns - Set directory for your custom patterns (optional)", plugin.GetSetupDescription()) + assert.False(t, plugin.IsConfigured()) // Should not be configured initially +} +func TestCustomPatterns_Configure(t *testing.T) { + plugin := NewCustomPatterns() + + // Test with empty directory (should work) + plugin.CustomPatternsDir.Value = "" + err := plugin.configure() + assert.NoError(t, err) + + // Test with home directory expansion + plugin.CustomPatternsDir.Value = "~/test-patterns" + err = plugin.configure() + assert.NoError(t, err) + + homeDir, _ := os.UserHomeDir() + expectedPath := filepath.Join(homeDir, "test-patterns") + absExpected, _ := filepath.Abs(expectedPath) + assert.Equal(t, absExpected, plugin.CustomPatternsDir.Value) + + // Clean up + os.RemoveAll(plugin.CustomPatternsDir.Value) +} + +func TestCustomPatterns_ConfigureWithTempDir(t *testing.T) { + plugin := NewCustomPatterns() + + // Test with a temporary directory + tmpDir, err := os.MkdirTemp("", "test-custom-patterns-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + plugin.CustomPatternsDir.Value = tmpDir + err = plugin.configure() + assert.NoError(t, err) + + absPath, _ := filepath.Abs(tmpDir) + assert.Equal(t, absPath, plugin.CustomPatternsDir.Value) + + // Verify directory exists + info, err := os.Stat(plugin.CustomPatternsDir.Value) + assert.NoError(t, err) + assert.True(t, info.IsDir()) + + // Should be configured now + assert.True(t, plugin.IsConfigured()) +} + +func TestCustomPatterns_IsConfigured(t *testing.T) { + plugin := NewCustomPatterns() + + // Initially not configured + assert.False(t, plugin.IsConfigured()) + + // Set a directory + plugin.CustomPatternsDir.Value = "/some/path" + assert.True(t, plugin.IsConfigured()) + + // Clear the directory + plugin.CustomPatternsDir.Value = "" + assert.False(t, plugin.IsConfigured()) +}