Update version to v1.4.89 and commit

Create user story pattern

.envrc

@@ -1 +1,2 @@
+watch_file shell.nix
 use flake

.github/workflows/update-version-and-create-tag.yml

@@ -41,7 +41,10 @@ jobs:
           minor=$(echo "$latest_tag" | cut -d. -f2)
           patch=$(echo "$latest_tag" | cut -d. -f3)
           new_patch=$((patch + 1))
-          new_tag="v${major}.${minor}.${new_patch}"
+          new_version="${major}.${minor}.${new_patch}"
+          new_tag="v${new_version}"
+          echo "New version is: $new_version"
+          echo "new_version=$new_version" >> $GITHUB_ENV  # Save the new version to environment file
           echo "New tag is: $new_tag"
           echo "new_tag=$new_tag" >> $GITHUB_ENV  # Save the new tag to environment file
 
@@ -51,9 +54,14 @@ jobs:
           echo "" >> version.go
           echo "var version = \"${{ env.new_tag }}\"" >> version.go
 
+      - name: Update version.nix file
+        run: |
+          echo "\"${{ env.new_version }}\"" > pkgs/fabric/version.nix
+
       - name: Commit changes
         run: |
           git add version.go
+          git add pkgs/fabric/version.nix
           if ! git diff --staged --quiet; then
             git commit -m "Update version to ${{ env.new_tag }} and commit $commit_hash"
           else

cli/cli.go

@@ -2,16 +2,18 @@ package cli
 
 import (
 	"fmt"
+	"github.com/danielmiessler/fabric/plugins/tools/youtube"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
 	"github.com/danielmiessler/fabric/common"
 	"github.com/danielmiessler/fabric/core"
 	"github.com/danielmiessler/fabric/plugins/ai"
 	"github.com/danielmiessler/fabric/plugins/db/fsdb"
 	"github.com/danielmiessler/fabric/plugins/tools/converter"
 	"github.com/danielmiessler/fabric/restapi"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
 )
 
 // Cli Controls the cli. It takes in the flags and runs the appropriate functions
@@ -21,6 +23,8 @@ func Cli(version string) (err error) {
 		return
 	}
 
+	println("Message: " + currentFlags.Message)
+
 	if currentFlags.Version {
 		fmt.Println(version)
 		return
@@ -143,37 +147,38 @@ func Cli(version string) (err error) {
 		}
 
 		var videoId string
-		if videoId, err = registry.YouTube.GetVideoId(currentFlags.YouTube); err != nil {
+		var playlistId string
+		if videoId, playlistId, err = registry.YouTube.GetVideoOrPlaylistId(currentFlags.YouTube); err != nil {
+			return
+		} else if videoId == "" && playlistId != "" {
+			if currentFlags.Output != "" {
+				err = registry.YouTube.FetchAndSavePlaylist(playlistId, currentFlags.Output)
+			} else {
+				var videos []*youtube.VideoMeta
+				if videos, err = registry.YouTube.FetchPlaylistVideos(playlistId); err != nil {
+					err = fmt.Errorf("error fetching playlist videos: %v", err)
+					return
+				}
+
+				for _, video := range videos {
+					var message string
+					if message, err = processYoutubeVideo(currentFlags, registry, video.Id); err != nil {
+						return
+					}
+
+					if !currentFlags.IsChatRequest() {
+						if err = WriteOutput(message, fmt.Sprintf("%v.md", video.TitleNormalized)); err != nil {
+							return
+						}
+					} else {
+						messageTools = AppendMessage(messageTools, message)
+					}
+				}
+			}
 			return
 		}
 
-		if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript {
-			var transcript string
-			var language = "en"
-			if currentFlags.Language != "" || registry.Language.DefaultLanguage.Value != "" {
-				if currentFlags.Language != "" {
-					language = currentFlags.Language
-				} else {
-					language = registry.Language.DefaultLanguage.Value
-				}
-			}
-			if transcript, err = registry.YouTube.GrabTranscript(videoId, language); err != nil {
-				return
-			}
-			messageTools = AppendMessage(messageTools, transcript)
-		}
-
-		if currentFlags.YouTubeComments {
-			var comments []string
-			if comments, err = registry.YouTube.GrabComments(videoId); err != nil {
-				return
-			}
-
-			commentsString := strings.Join(comments, "\n")
-
-			messageTools = AppendMessage(messageTools, commentsString)
-		}
-
+		messageTools, err = processYoutubeVideo(currentFlags, registry, videoId)
 		if !currentFlags.IsChatRequest() {
 			err = currentFlags.WriteOutput(messageTools)
 			return
@@ -211,7 +216,7 @@ func Cli(version string) (err error) {
 	}
 
 	var chatter *core.Chatter
-	if chatter, err = registry.GetChatter(currentFlags.Model, currentFlags.Stream, currentFlags.DryRun); err != nil {
+	if chatter, err = registry.GetChatter(currentFlags.Model, currentFlags.ModelContextLength, currentFlags.Stream, currentFlags.DryRun); err != nil {
 		return
 	}
 
@@ -253,3 +258,43 @@ func Cli(version string) (err error) {
 	}
 	return
 }
+
+func processYoutubeVideo(
+	flags *Flags, registry *core.PluginRegistry, videoId string) (message string, err error) {
+
+	if !flags.YouTubeComments || flags.YouTubeTranscript {
+		var transcript string
+		var language = "en"
+		if flags.Language != "" || registry.Language.DefaultLanguage.Value != "" {
+			if flags.Language != "" {
+				language = flags.Language
+			} else {
+				language = registry.Language.DefaultLanguage.Value
+			}
+		}
+		if transcript, err = registry.YouTube.GrabTranscript(videoId, language); err != nil {
+			return
+		}
+		message = AppendMessage(message, transcript)
+	}
+
+	if flags.YouTubeComments {
+		var comments []string
+		if comments, err = registry.YouTube.GrabComments(videoId); err != nil {
+			return
+		}
+
+		commentsString := strings.Join(comments, "\n")
+
+		message = AppendMessage(message, commentsString)
+	}
+	return
+}
+
+func WriteOutput(message string, outputFile string) (err error) {
+	fmt.Println(message)
+	if outputFile != "" {
+		err = CreateOutputFile(message, outputFile)
+	}
+	return
+}

cli/flags.go

@@ -4,14 +4,15 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
-	goopenai "github.com/sashabaranov/go-openai"
 	"io"
 	"os"
 	"strings"
 
-	"github.com/danielmiessler/fabric/common"
 	"github.com/jessevdk/go-flags"
+	goopenai "github.com/sashabaranov/go-openai"
 	"golang.org/x/text/language"
+
+	"github.com/danielmiessler/fabric/common"
 )
 
 // Flags create flags struct. the users flags go into this, this will be passed to the chat struct in cli
@@ -36,6 +37,7 @@ type Flags struct {
 	Message            string            `hidden:"true" description:"Messages to send to chat"`
 	Copy               bool              `short:"c" long:"copy" description:"Copy to clipboard"`
 	Model              string            `short:"m" long:"model" description:"Choose model"`
+	ModelContextLength int               `long:"modelContextLength" description:"Model context length (only affects ollama)"`
 	Output             string            `short:"o" long:"output" description:"Output to file" default:""`
 	OutputSession      bool              `long:"output-session" description:"Output the entire session (also a temporary one) to the output file"`
 	LatestPatterns     string            `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
@@ -70,16 +72,19 @@ func Init() (ret *Flags, err error) {
 	}
 
 	info, _ := os.Stdin.Stat()
-	hasStdin := (info.Mode() & os.ModeCharDevice) == 0
+	pipedToStdin := (info.Mode() & os.ModeCharDevice) == 0
 
 	// takes input from stdin if it exists, otherwise takes input from args (the last argument)
-	if hasStdin {
+	if pipedToStdin {
+		//fmt.Printf("piped: %v\n", args)
 		if message, err = readStdin(); err != nil {
 			return
 		}
 	} else if len(args) > 0 {
+		//fmt.Printf("no piped: %v\n", args)
 		message = args[len(args)-1]
 	} else {
+		//fmt.Printf("no data: %v\n", args)
 		message = ""
 	}
 	ret.Message = message
@@ -88,30 +93,30 @@ func Init() (ret *Flags, err error) {
 }
 
 // readStdin reads from stdin and returns the input as a string or an error
-func readStdin() (string, error) {
-	reader := bufio.NewReader(os.Stdin)
-	var input string
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-			return "", fmt.Errorf("error reading from stdin: %w", err)
-		}
-		input += line
+func readStdin() (ret string, err error) {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		ret += scanner.Text() + "\n"
 	}
-	return input, nil
+	if err = scanner.Err(); err != nil {
+		if errors.Is(err, io.EOF) {
+			err = nil
+		} else {
+			err = fmt.Errorf("error reading piped message from stdin: %w", err)
+		}
+	}
+	return
 }
 
 func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {
 	ret = &common.ChatOptions{
-		Temperature:      o.Temperature,
-		TopP:             o.TopP,
-		PresencePenalty:  o.PresencePenalty,
-		FrequencyPenalty: o.FrequencyPenalty,
-		Raw:              o.Raw,
-		Seed:             o.Seed,
+		Temperature:        o.Temperature,
+		TopP:               o.TopP,
+		PresencePenalty:    o.PresencePenalty,
+		FrequencyPenalty:   o.FrequencyPenalty,
+		Raw:                o.Raw,
+		Seed:               o.Seed,
+		ModelContextLength: o.ModelContextLength,
 	}
 	return
 }

common/domain.go

@@ -15,13 +15,14 @@ type ChatRequest struct {
 }
 
 type ChatOptions struct {
-	Model            string
-	Temperature      float64
-	TopP             float64
-	PresencePenalty  float64
-	FrequencyPenalty float64
-	Raw              bool
-	Seed             int
+	Model              string
+	Temperature        float64
+	TopP               float64
+	PresencePenalty    float64
+	FrequencyPenalty   float64
+	Raw                bool
+	Seed               int
+	ModelContextLength int
 }
 
 // NormalizeMessages remove empty messages and ensure messages order user-assist-user

core/chatter.go

@@ -3,11 +3,13 @@ package core
 import (
 	"context"
 	"fmt"
+	"strings"
+
+	goopenai "github.com/sashabaranov/go-openai"
+
 	"github.com/danielmiessler/fabric/common"
 	"github.com/danielmiessler/fabric/plugins/ai"
 	"github.com/danielmiessler/fabric/plugins/db/fsdb"
-	goopenai "github.com/sashabaranov/go-openai"
-	"strings"
 )
 
 const NoSessionPatternUserMessages = "no session, pattern or user messages provided"
@@ -18,8 +20,9 @@ type Chatter struct {
 	Stream bool
 	DryRun bool
 
-	model  string
-	vendor ai.Vendor
+	model              string
+	modelContextLength int
+	vendor             ai.Vendor
 }
 
 func (o *Chatter) Send(request *common.ChatRequest, opts *common.ChatOptions) (session *fsdb.Session, err error) {
@@ -31,6 +34,10 @@ func (o *Chatter) Send(request *common.ChatRequest, opts *common.ChatOptions) (s
 		opts.Model = o.model
 	}
 
+	if opts.ModelContextLength == 0 {
+		opts.ModelContextLength = o.modelContextLength
+	}
+
 	message := ""
 
 	if o.Stream {

core/plugin_registry.go

@@ -3,15 +3,15 @@ package core
 import (
 	"bytes"
 	"fmt"
-	"github.com/danielmiessler/fabric/common"
-	"github.com/danielmiessler/fabric/plugins/ai/azure"
-	"github.com/danielmiessler/fabric/plugins/tools"
-	"github.com/samber/lo"
 	"strconv"
 
+	"github.com/samber/lo"
+
+	"github.com/danielmiessler/fabric/common"
 	"github.com/danielmiessler/fabric/plugins"
 	"github.com/danielmiessler/fabric/plugins/ai"
 	"github.com/danielmiessler/fabric/plugins/ai/anthropic"
+	"github.com/danielmiessler/fabric/plugins/ai/azure"
 	"github.com/danielmiessler/fabric/plugins/ai/dryrun"
 	"github.com/danielmiessler/fabric/plugins/ai/gemini"
 	"github.com/danielmiessler/fabric/plugins/ai/groq"
@@ -21,6 +21,7 @@ import (
 	"github.com/danielmiessler/fabric/plugins/ai/openrouter"
 	"github.com/danielmiessler/fabric/plugins/ai/siliconcloud"
 	"github.com/danielmiessler/fabric/plugins/db/fsdb"
+	"github.com/danielmiessler/fabric/plugins/tools"
 	"github.com/danielmiessler/fabric/plugins/tools/jina"
 	"github.com/danielmiessler/fabric/plugins/tools/lang"
 	"github.com/danielmiessler/fabric/plugins/tools/youtube"
@@ -164,7 +165,7 @@ func (o *PluginRegistry) Configure() (err error) {
 	return
 }
 
-func (o *PluginRegistry) GetChatter(model string, stream bool, dryRun bool) (ret *Chatter, err error) {
+func (o *PluginRegistry) GetChatter(model string, modelContextLength int, stream bool, dryRun bool) (ret *Chatter, err error) {
 	ret = &Chatter{
 		db:     o.Db,
 		Stream: stream,
@@ -172,9 +173,20 @@ func (o *PluginRegistry) GetChatter(model string, stream bool, dryRun bool) (ret
 	}
 
 	defaultModel := o.Defaults.Model.Value
+	defaultModelContextLength, err := strconv.Atoi(o.Defaults.ModelContextLength.Value)
 	defaultVendor := o.Defaults.Vendor.Value
 	vendorManager := o.VendorManager
 
+	if err != nil {
+		defaultModelContextLength = 0
+		err = nil
+	}
+
+	ret.modelContextLength = modelContextLength
+	if ret.modelContextLength == 0 {
+		ret.modelContextLength = defaultModelContextLength
+	}
+
 	if dryRun {
 		ret.vendor = dryrun.NewClient()
 		ret.model = model

flake.nix

@@ -49,29 +49,9 @@
           pkgs = nixpkgs.legacyPackages.${system};
           goEnv = gomod2nix.legacyPackages.${system}.mkGoEnv { pwd = ./.; };
         in
-        {
-          default = pkgs.mkShell {
-            nativeBuildInputs = [
-              pkgs.go
-              pkgs.gopls
-              pkgs.gotools
-              pkgs.go-tools
-              pkgs.goimports-reviser
-              gomod2nix.legacyPackages.${system}.gomod2nix
-              goEnv
-
-              (pkgs.writeShellScriptBin "update" ''
-                go get -u
-                go mod tidy
-                gomod2nix generate
-              '')
-            ];
-
-            shellHook = ''
-              echo -e "\033[0;32;4mHeper commands:\033[0m"
-              echo "'update' instead of 'go get -u && go mod tidy'"
-            '';
-          };
+        import ./shell.nix {
+          inherit pkgs goEnv;
+          inherit (gomod2nix.legacyPackages.${system}) gomod2nix;
         }
       );
 
@@ -81,27 +61,10 @@
           pkgs = nixpkgs.legacyPackages.${system};
         in
         {
-          fabric = gomod2nix.legacyPackages.${system}.buildGoApplication {
-            pname = "fabric-ai";
-            version = "1.4.72";
-            src = self;
-            pwd = self;
-            modules = ./gomod2nix.toml;
-
-            ldflags = [
-              "-s"
-              "-w"
-            ];
-
-            meta = with pkgs.lib; {
-              description = "Fabric is an open-source framework for augmenting humans using AI. It provides a modular framework for solving specific problems using a crowdsourced set of AI prompts that can be used anywhere";
-              homepage = "https://github.com/danielmiessler/fabric";
-              license = licenses.mit;
-              platforms = platforms.all;
-              mainProgram = "fabric";
-            };
-          };
           default = self.packages.${system}.fabric;
+          fabric = pkgs.callPackage ./pkgs/fabric {
+            inherit (gomod2nix.legacyPackages.${system}) buildGoApplication;
+          };
         }
       );
     };

gomod2nix.toml

@@ -2,23 +2,23 @@ schema = 3
 
 [mod]
   [mod."cloud.google.com/go"]
-    version = "v0.115.1"
-    hash = "sha256-i/KvDIs1/6HoX8DbH0qI2vJ0kiuZOV3o2ep9FxUQn/M="
+    version = "v0.116.0"
+    hash = "sha256-e62GvNveg3bRi4O+eBARqgQ2sinobx+SVGR9WE7jKgs="
   [mod."cloud.google.com/go/ai"]
     version = "v0.8.2"
     hash = "sha256-UtCuHChDsXlACXdlVSNFo8F/X8vAkmPoJyng3/oEFe0="
   [mod."cloud.google.com/go/auth"]
-    version = "v0.9.4"
-    hash = "sha256-ARAms3a9zZepUY5qPcXYFj6XQnFnKRBr0SnSanbkSP0="
+    version = "v0.9.9"
+    hash = "sha256-kUrulQhYPM6cFhInFqTX/Dj1GVi+Ev1Ry7T+hiTmb38="
   [mod."cloud.google.com/go/auth/oauth2adapt"]
     version = "v0.2.4"
     hash = "sha256-GRXPQMHEEgeKhdCOBjoDL7+UW3yBdSei5ULuZGBE4tw="
   [mod."cloud.google.com/go/compute/metadata"]
-    version = "v0.5.1"
-    hash = "sha256-dle3Sv1HUCtEw5OGVnE20RBP3CW4vARxWywU0HZUZ5Y="
+    version = "v0.5.2"
+    hash = "sha256-EtBj20lhjM3SJVKCp70GHMnsItwJ9gOyJOW91wugojc="
   [mod."cloud.google.com/go/longrunning"]
-    version = "v0.6.1"
-    hash = "sha256-4gCfztD2PYAbVW+D4s6KblxksM3uQvWjPTJVkT9Hsz8="
+    version = "v0.6.2"
+    hash = "sha256-X78JL1/YtXA7upOcTuNezq5TxjQyFxQ6OINrS8zzdEU="
   [mod."dario.cat/mergo"]
     version = "v1.0.1"
     hash = "sha256-wcG6+x0k6KzOSlaPA+1RFxa06/RIAePJTAjjuhLbImw="
@@ -41,14 +41,14 @@ schema = 3
     version = "v0.1.4"
     hash = "sha256-ZZ7U5X0gWOu8zcjZcWbcpzGOGdycwq0TjTFh/eZHjXk="
   [mod."github.com/bytedance/sonic"]
-    version = "v1.11.6"
-    hash = "sha256-oGDdBbAHDwQYFFcg3AeYaHwOGQNa1q4n/0w4y2eqsxk="
+    version = "v1.12.3"
+    hash = "sha256-cZicMhM/2D7HefuJ0xe7AJKh9du2O38HWs+3RNCpbZM="
   [mod."github.com/bytedance/sonic/loader"]
-    version = "v0.1.1"
-    hash = "sha256-9MzO8LYUrun40uXTexcbKtQO1MvXhlD6Er6T6v9TOtE="
+    version = "v0.2.1"
+    hash = "sha256-+gPRZtBOJbAnXp/jdMlPmesc62JGH8akQ1UK9VRI7E4="
   [mod."github.com/cloudflare/circl"]
-    version = "v1.4.0"
-    hash = "sha256-poZpasVFbhYsWhaPfss/BzM89S8PvKYz3Nry2qUCre8="
+    version = "v1.5.0"
+    hash = "sha256-j7T4cfbfmhlbaO+kNKveTnk95JbkEOX0IVw8D9bGTkQ="
   [mod."github.com/cloudwego/base64x"]
     version = "v0.1.4"
     hash = "sha256-umCZR3iNmHFm+BC76kfpdcRG+pTQd6Jcu/c2kQDnyfw="
@@ -56,8 +56,8 @@ schema = 3
     version = "v0.2.0"
     hash = "sha256-TzIP2N3HOesXrKACsRr/ShcoqttwPGZPckIepsTyHOA="
   [mod."github.com/cyphar/filepath-securejoin"]
-    version = "v0.3.2"
-    hash = "sha256-YpuNjTSi56x7sdSSt0fui/erxIG3fE5PhLzZ72ColIE="
+    version = "v0.3.4"
+    hash = "sha256-I9dV5gtKk3hH39taAWxvvJEXMi4YoHSxeESVyjpl1MU="
   [mod."github.com/davecgh/go-spew"]
     version = "v1.1.1"
     hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
@@ -68,8 +68,8 @@ schema = 3
     version = "v1.0.4"
     hash = "sha256-c1JKoRSndwwOyOxq9ddCe+8qn7mG9uRq2o/822x5O/c="
   [mod."github.com/gabriel-vasile/mimetype"]
-    version = "v1.4.3"
-    hash = "sha256-EDmlRi3av27dq/ISVTglv08z4yZzMQ/SxL1c46EJro0="
+    version = "v1.4.6"
+    hash = "sha256-W/uPcE22Fduw1XmX8Ujf1S9SYVOcEoE1wzK4I0/vapw="
   [mod."github.com/gin-contrib/sse"]
     version = "v0.1.0"
     hash = "sha256-zYbMTao+1F+385Lvsba9roLmmt9eYqr57sUWo0LCVhw="
@@ -80,8 +80,8 @@ schema = 3
     version = "v1.5.1-0.20230307220236-3a3c6141e376"
     hash = "sha256-f4k0gSYuo0/q3WOoTxl2eFaj7WZpdz29ih6CKc8Ude8="
   [mod."github.com/go-git/go-billy/v5"]
-    version = "v5.5.0"
-    hash = "sha256-4XUoD2bOCMCdu83egb/y8kY/Fm0s1rWgPMtiahh38OQ="
+    version = "v5.6.0"
+    hash = "sha256-Hw+odNozpiixXqmsbahihdV+TBxpusm6/hDLngf7kUg="
   [mod."github.com/go-git/go-git/v5"]
     version = "v5.12.0"
     hash = "sha256-mD8EWOQ25FtKBWVSQhQ8V1Rr0tC/ySFZQ9GMDLRqwQU="
@@ -98,17 +98,17 @@ schema = 3
     version = "v0.18.1"
     hash = "sha256-2/B2qP51zfiY+k8G0w0D03KXUc7XpWj6wKY7NjNP/9E="
   [mod."github.com/go-playground/validator/v10"]
-    version = "v10.20.0"
-    hash = "sha256-FKF+ebrSedNdh5Wq8aE8UP+5LiM8B28bk8v3gyqqdDk="
+    version = "v10.22.1"
+    hash = "sha256-EsgeltH0ow6saxLvTFVtIyHVqWI3Fiu1AE2Qmnsmowg="
   [mod."github.com/go-shiori/dom"]
     version = "v0.0.0-20230515143342-73569d674e1c"
     hash = "sha256-4lm9KZfR2XnfZU9KTG+4jqLYZqbfL74AMO4y3dKpIbg="
   [mod."github.com/go-shiori/go-readability"]
-    version = "v0.0.0-20240923125239-59a7bd165825"
-    hash = "sha256-9uHe8rZcryKubhN6Rw6ECtSGAku7gkx8gtKotWnyHdM="
+    version = "v0.0.0-20241012063810-92284fa8a71f"
+    hash = "sha256-NgciyWylVSjzkt5xWF1Xk1Xbxgq3PsHW5PZ8oifjZVY="
   [mod."github.com/goccy/go-json"]
-    version = "v0.10.2"
-    hash = "sha256-6fMD2/Rku8HT0zDdeA23pX0YxbohiIOC8OJNYbylJTQ="
+    version = "v0.10.3"
+    hash = "sha256-ZOzfwCXh+qp+hp+UnC0t422hUV0Cq5KANXkx8hcLp7s="
   [mod."github.com/gogs/chardet"]
     version = "v0.0.0-20211120154057-b7413eaefb8f"
     hash = "sha256-4MeqBJsh4U+ZEbfdDwdciTYMlQWkCil2KJbUxHjBSIo="
@@ -146,14 +146,14 @@ schema = 3
     version = "v1.2.0"
     hash = "sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s="
   [mod."github.com/klauspost/cpuid/v2"]
-    version = "v2.2.7"
-    hash = "sha256-bjinp7b7qWk+DcZDDv1EedJxZqGxp2NWY+NYKBfE5xU="
+    version = "v2.2.8"
+    hash = "sha256-/E58BnABQYxO+cmiue7OQqRLWkd/Lh8grX8DjTU4tk8="
   [mod."github.com/leodido/go-urn"]
     version = "v1.4.0"
     hash = "sha256-Q6kplWkY37Tzy6GOme3Wut40jFK4Izun+ij/BJvcEu0="
   [mod."github.com/liushuangls/go-anthropic/v2"]
-    version = "v2.8.0"
-    hash = "sha256-F89FeBbUl/zejzxvR0LZKhHFQmHik5urzVTlEjQj+eg="
+    version = "v2.9.0"
+    hash = "sha256-1bvwuPT5SaYrzKYiXpz0fjDgu3994Hs5MPXZUFPhCLI="
   [mod."github.com/mattn/go-isatty"]
     version = "v0.0.20"
     hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
@@ -164,14 +164,14 @@ schema = 3
     version = "v1.0.2"
     hash = "sha256-+W9EIW7okXIXjWEgOaMh58eLvBZ7OshW2EhaIpNLSBU="
   [mod."github.com/ollama/ollama"]
-    version = "v0.3.11"
-    hash = "sha256-ydQyt+now2iqal2ZO6oPn61UoGt+RBir0AB2R9e72io="
+    version = "v0.3.14"
+    hash = "sha256-R+jWZzGokwWimPePIcUoZz6tDG2qHFeClgxhT7SEoqw="
   [mod."github.com/otiai10/copy"]
     version = "v1.14.0"
     hash = "sha256-xsaL1ddkPS544y0Jv7u/INUALBYmYq29ddWvysLXk4A="
   [mod."github.com/pelletier/go-toml/v2"]
-    version = "v2.2.2"
-    hash = "sha256-ukxk1Cfm6cQW18g/aa19tLcUu5BnF7VmfAvrDHAOl6A="
+    version = "v2.2.3"
+    hash = "sha256-fE++SVgnCGdnFZoROHWuYjIR7ENl7k9KKxQrRTquv/o="
   [mod."github.com/pjbgf/sha1cd"]
     version = "v0.3.0"
     hash = "sha256-kX9BdLh2dxtGNaDvc24NORO+C0AZ7JzbrXrtecCdB7w="
@@ -185,8 +185,8 @@ schema = 3
     version = "v1.47.0"
     hash = "sha256-jMXexVTlPdZ40STRpBLv7b+BIRqdxxra12Pl2Mj7Nz8="
   [mod."github.com/sashabaranov/go-openai"]
-    version = "v1.30.0"
-    hash = "sha256-vOKCQlLPNvf4CrcPL5Hu07EOB6WCyEPVSB89fdUbdAA="
+    version = "v1.32.5"
+    hash = "sha256-T56gcES0qMZCSL3uFi+G9vmfTk00QlMWONsMS+cxwR0="
   [mod."github.com/sergi/go-diff"]
     version = "v1.3.2-0.20230802210424-5b0b94c5c0d3"
     hash = "sha256-UcLU83CPMbSoKI8RLvLJ7nvGaE2xRSL1RjoHCVkMzUM="
@@ -209,23 +209,23 @@ schema = 3
     version = "v0.24.0"
     hash = "sha256-4H+mGZgG2c9I1y0m8avF4qmt8LUKxxVsTqR8mKgP4yo="
   [mod."go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"]
-    version = "v0.55.0"
-    hash = "sha256-aiaNqHfnhz0NgjxqKJMILa7JcCRweCjBq8i7EiM2sx4="
+    version = "v0.56.0"
+    hash = "sha256-nrdJ7CgH3yKhNkMpvhP2BY4+VL/maR5mrjJCO6Dke2s="
   [mod."go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"]
-    version = "v0.55.0"
-    hash = "sha256-/uNC1W2fMF2jfk5fBwfV4hLo9fLXLHfTkvTv8B/dFGI="
+    version = "v0.56.0"
+    hash = "sha256-Nw9uF/TUoFnH0488VNHVndgoSX+lxZy1Y93Wryl7qP4="
   [mod."go.opentelemetry.io/otel"]
-    version = "v1.30.0"
-    hash = "sha256-2r706a1+LqhT3DNqVZB6NIVaBoPnQ257BJY8UQqtaSM="
+    version = "v1.31.0"
+    hash = "sha256-NQBHyMSRn9vaxSrNHYwv0oX1aJuEpyks/gpYEWHlx6k="
   [mod."go.opentelemetry.io/otel/metric"]
-    version = "v1.30.0"
-    hash = "sha256-mSfy2A7cxIjAOs9tZ74kgVnsnGZoC3be/MjjFlO8KCw="
+    version = "v1.31.0"
+    hash = "sha256-2s5IN8IwPBitqnjIEraOfg8fWd3nIy8jWoKTXa+uUs4="
   [mod."go.opentelemetry.io/otel/trace"]
-    version = "v1.30.0"
-    hash = "sha256-6YukUeYtMo5jtKsoQQwuy3Zyx1Cl2LRMONnF91PCBIk="
+    version = "v1.31.0"
+    hash = "sha256-iVDe3qNzmX1+MQTAoaeIbhnIbu/hnx4OsUIPlxuX1gY="
   [mod."golang.org/x/arch"]
-    version = "v0.8.0"
-    hash = "sha256-zz9sbr+yT6eqjHVlVBfDZVmIkzOT6DZFpN3eaQT4Afw="
+    version = "v0.11.0"
+    hash = "sha256-gl4bqDA/Qv6hhqxROIHTWnNGkidMMN0frp1RqcfNXlY="
   [mod."golang.org/x/crypto"]
     version = "v0.28.0"
     hash = "sha256-AYjr0BcWQMwWY1u8c2hzUprtqHUmAH7RNSxHz2hhnZs="
@@ -245,23 +245,23 @@ schema = 3
     version = "v0.19.0"
     hash = "sha256-C92pSYLLUQ2NKKcc60wpoSJ5UWAfnWkmd997C13fXdU="
   [mod."golang.org/x/time"]
-    version = "v0.6.0"
-    hash = "sha256-gW9TVK9HjLk52lzfo5rBzSunc01gS0+SG2nk0X1w55M="
+    version = "v0.7.0"
+    hash = "sha256-o1ol/hTpfrc06KUXSepAgm4QUuWmH1S+vqg6kmFad64="
   [mod."google.golang.org/api"]
-    version = "v0.197.0"
-    hash = "sha256-OMqA6/WODEKiiMp79GZ3XjnAC7Sc9XFUJcPTlJgHRSE="
+    version = "v0.203.0"
+    hash = "sha256-UlCfDi4LbcBvXVO5gLRWP6/fpfWWHoolRkxzYWGnNqg="
   [mod."google.golang.org/genproto/googleapis/api"]
-    version = "v0.0.0-20240903143218-8af14fe29dc1"
-    hash = "sha256-H40uIAZY8JMCAAe8+fLHbHMI/wNMUL8ukhBdS87p/2E="
+    version = "v0.0.0-20241021214115-324edc3d5d38"
+    hash = "sha256-ASsqfJU1DA57PLRoitSkdlS/p10EEuzl0YuZTdbmMCw="
   [mod."google.golang.org/genproto/googleapis/rpc"]
-    version = "v0.0.0-20240903143218-8af14fe29dc1"
-    hash = "sha256-4T4DTrmFbqT4tD7PSL7Ie7u8ZN2iwGkhK02nWugssxk="
+    version = "v0.0.0-20241021214115-324edc3d5d38"
+    hash = "sha256-Fk+cG5bRI3BvnqhWzvMzbU36cC7PM+o2oAOJmvVx9M0="
   [mod."google.golang.org/grpc"]
-    version = "v1.66.2"
-    hash = "sha256-ZGEQK9lLC55Jkdifef/SO9mRPwEZmMJPXLH6MAKIGDA="
+    version = "v1.67.1"
+    hash = "sha256-VqfKp80c2B1MK4m1WtHW4r7ykqdChJbqaMn+gMEYmYc="
   [mod."google.golang.org/protobuf"]
-    version = "v1.34.2"
-    hash = "sha256-nMTlrDEE2dbpWz50eQMPBQXCyQh4IdjrTIccaU0F3m0="
+    version = "v1.35.1"
+    hash = "sha256-4NtUQoBvlPGFGjo7c+E1EBS/sb8oy50MGy45KGWPpWo="
   [mod."gopkg.in/warnings.v0"]
     version = "v0.1.2"
     hash = "sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8="

patterns/create_user_story/system.md

@@ -0,0 +1,45 @@
+# IDENTITY and PURPOSE
+
+You are an expert on writing concise, clear, and illuminating technical user stories for new features in complex software programs
+
+# OUTPUT INSTRUCTIONS
+
+ Write the users stories in a fashion recognised by other software stakeholders, including product, development, operations and quality assurance
+
+EXAMPLE USER STORY
+
+Description
+As a Highlight developer
+I want to migrate email templates over to Mustache
+So that future upgrades to the messenger service can be made easier
+
+Acceptance Criteria
+- Migrate the existing alerting email templates from the instance specific databases over to the messenger templates blob storage.
+	- Rename each template to a GUID and store in it's own folder within the blob storage
+	- Store Subject and Body as separate blobs
+
+- Create an upgrade script to change the value of the Alerting.Email.Template local parameter in all systems to the new template names.
+- Change the template retrieval and saving for user editing to contact the blob storage rather than the database
+- Remove the database tables and code that handles the SQL based templates
+- Highlight sends the template name and the details of the body to the Email queue in Service bus  
+	- this is handled by the generic Email Client (if created already)
+	- This email type will be added to the list of email types that are sent to the messenger service (switch to be removed once all email templates are completed)  
+
+- Include domain details as part of payload sent to the messenger service
+
+Note: ensure that Ops know when this work is being done so they are aware of any changes to existing templates
+
+# OUTPUT INSTRUCTIONS
+
+- Write the user story according to the structure above.  
+- That means the user story should be written in a simple, bulleted style, not in a grandiose, conversational or academic style.
+
+# OUTPUT FORMAT
+
+- Output a full, user story about the content provided using the instructions above.
+- The structure should be: Description, Acceptance criteria 
+- Write in a simple, plain, and clear style, not in a grandiose, conversational or academic style.
+- Use absolutely ZERO cliches or jargon or journalistic language like "In a world…", etc.
+- Do not use cliches or jargon.
+- Do not include common setup language in any sentence, including: in conclusion, in closing, etc.
+- Do not output warnings or notes—just the output requested.

pkgs/fabric/default.nix

@@ -0,0 +1,25 @@
+{
+  lib,
+  buildGoApplication,
+}:
+
+buildGoApplication {
+  pname = "fabric-ai";
+  version = import ./version.nix;
+  src = ../../.;
+  pwd = ../../.;
+  modules = ../../gomod2nix.toml;
+
+  ldflags = [
+    "-s"
+    "-w"
+  ];
+
+  meta = with lib; {
+    description = "Fabric is an open-source framework for augmenting humans using AI. It provides a modular framework for solving specific problems using a crowdsourced set of AI prompts that can be used anywhere";
+    homepage = "https://github.com/danielmiessler/fabric";
+    license = licenses.mit;
+    platforms = platforms.all;
+    mainProgram = "fabric";
+  };
+}

pkgs/fabric/version.nix

@@ -0,0 +1 @@
+"1.4.89"

plugins/ai/dryrun/dryrun.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/danielmiessler/fabric/plugins"
+
 	goopenai "github.com/sashabaranov/go-openai"
 
 	"github.com/danielmiessler/fabric/common"
+	"github.com/danielmiessler/fabric/plugins"
 )
 
 type Client struct {
@@ -44,6 +45,9 @@ func (c *Client) SendStream(msgs []*goopenai.ChatCompletionMessage, opts *common
 	output += fmt.Sprintf("TopP: %f\n", opts.TopP)
 	output += fmt.Sprintf("PresencePenalty: %f\n", opts.PresencePenalty)
 	output += fmt.Sprintf("FrequencyPenalty: %f\n", opts.FrequencyPenalty)
+	if opts.ModelContextLength != 0 {
+		output += fmt.Sprintf("ModelContextLength: %d\n", opts.ModelContextLength)
+	}
 
 	channel <- output
 	close(channel)
@@ -72,6 +76,9 @@ func (c *Client) Send(_ context.Context, msgs []*goopenai.ChatCompletionMessage,
 	fmt.Printf("TopP: %f\n", opts.TopP)
 	fmt.Printf("PresencePenalty: %f\n", opts.PresencePenalty)
 	fmt.Printf("FrequencyPenalty: %f\n", opts.FrequencyPenalty)
+	if opts.ModelContextLength != 0 {
+		fmt.Printf("ModelContextLength: %d\n", opts.ModelContextLength)
+	}
 
 	return "", nil
 }

plugins/ai/ollama/ollama.go

@@ -3,16 +3,16 @@ package ollama
 import (
 	"context"
 	"fmt"
-	"github.com/danielmiessler/fabric/plugins"
-	goopenai "github.com/sashabaranov/go-openai"
 	"net/http"
 	"net/url"
 	"time"
 
-	"github.com/danielmiessler/fabric/common"
-	"github.com/samber/lo"
-
 	ollamaapi "github.com/ollama/ollama/api"
+	"github.com/samber/lo"
+	goopenai "github.com/sashabaranov/go-openai"
+
+	"github.com/danielmiessler/fabric/common"
+	"github.com/danielmiessler/fabric/plugins"
 )
 
 func NewClient() (ret *Client) {
@@ -110,6 +110,10 @@ func (o *Client) createChatRequest(msgs []*goopenai.ChatCompletionMessage, opts
 		"top_p":             opts.TopP,
 	}
 
+	if opts.ModelContextLength != 0 {
+		options["num_ctx"] = opts.ModelContextLength
+	}
+
 	ret = ollamaapi.ChatRequest{
 		Model:    opts.Model,
 		Messages: messages,

plugins/tools/defaults.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/pkg/errors"
+
 	"github.com/danielmiessler/fabric/plugins"
 	"github.com/danielmiessler/fabric/plugins/ai"
-	"github.com/pkg/errors"
 )
 
 func NeeDefaults(getVendorsModels func() (*ai.VendorsModels, error)) (ret *Defaults) {
@@ -21,18 +22,23 @@ func NeeDefaults(getVendorsModels func() (*ai.VendorsModels, error)) (ret *Defau
 	}
 
 	ret.Vendor = ret.AddSetting("Vendor", true)
+
 	ret.Model = ret.AddSetupQuestionCustom("Model", true,
 		"Enter the index the name of your default model")
 
+	ret.ModelContextLength = ret.AddSetupQuestionCustom("Model Context Length", false,
+		"Enter model context length")
+
 	return
 }
 
 type Defaults struct {
 	*plugins.PluginBase
 
-	Vendor           *plugins.Setting
-	Model            *plugins.SetupQuestion
-	GetVendorsModels func() (*ai.VendorsModels, error)
+	Vendor             *plugins.Setting
+	Model              *plugins.SetupQuestion
+	ModelContextLength *plugins.SetupQuestion
+	GetVendorsModels   func() (*ai.VendorsModels, error)
 }
 
 func (o *Defaults) Setup() (err error) {

plugins/tools/youtube/youtube.go

@@ -2,14 +2,17 @@ package youtube
 
 import (
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"net/url"
+	"os"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/anaskhan96/soup"
 	"github.com/danielmiessler/fabric/plugins"
@@ -37,38 +40,56 @@ type YouTube struct {
 	*plugins.PluginBase
 	ApiKey *plugins.SetupQuestion
 
-	service *youtube.Service
+	normalizeRegex *regexp.Regexp
+	service        *youtube.Service
 }
 
 func (o *YouTube) initService() (err error) {
 	if o.service == nil {
+		o.normalizeRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
 		ctx := context.Background()
 		o.service, err = youtube.NewService(ctx, option.WithAPIKey(o.ApiKey.Value))
 	}
 	return
 }
 
-func (o *YouTube) GetVideoId(url string) (ret string, err error) {
+func (o *YouTube) GetVideoOrPlaylistId(url string) (videoId string, playlistId string, err error) {
 	if err = o.initService(); err != nil {
 		return
 	}
 
-	pattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})`
-	re := regexp.MustCompile(pattern)
-	match := re.FindStringSubmatch(url)
-	if len(match) > 1 {
-		ret = match[1]
-	} else {
-		err = fmt.Errorf("invalid YouTube URL, can't get video ID")
+	// Video ID pattern
+	videoPattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})`
+	videoRe := regexp.MustCompile(videoPattern)
+	videoMatch := videoRe.FindStringSubmatch(url)
+	if len(videoMatch) > 1 {
+		videoId = videoMatch[1]
+	}
+
+	// Playlist ID pattern
+	playlistPattern := `[?&]list=([a-zA-Z0-9_-]+)`
+	playlistRe := regexp.MustCompile(playlistPattern)
+	playlistMatch := playlistRe.FindStringSubmatch(url)
+	if len(playlistMatch) > 1 {
+		playlistId = playlistMatch[1]
+	}
+
+	if videoId == "" && playlistId == "" {
+		err = fmt.Errorf("invalid YouTube URL, can't get video or playlist ID: '%s'", url)
 	}
 	return
 }
 
 func (o *YouTube) GrabTranscriptForUrl(url string, language string) (ret string, err error) {
 	var videoId string
-	if videoId, err = o.GetVideoId(url); err != nil {
+	var playlistId string
+	if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
+		return
+	} else if videoId == "" && playlistId != "" {
+		err = fmt.Errorf("URL is a playlist, not a video")
 		return
 	}
+
 	return o.GrabTranscript(videoId, language)
 }
 
@@ -172,7 +193,11 @@ func (o *YouTube) GrabDurationForUrl(url string) (ret int, err error) {
 	}
 
 	var videoId string
-	if videoId, err = o.GetVideoId(url); err != nil {
+	var playlistId string
+	if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
+		return
+	} else if videoId == "" && playlistId != "" {
+		err = fmt.Errorf("URL is a playlist, not a video")
 		return
 	}
 	return o.GrabDuration(videoId)
@@ -203,7 +228,11 @@ func (o *YouTube) GrabDuration(videoId string) (ret int, err error) {
 
 func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error) {
 	var videoId string
-	if videoId, err = o.GetVideoId(url); err != nil {
+	var playlistId string
+	if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
+		return
+	} else if videoId == "" && playlistId != "" {
+		err = fmt.Errorf("URL is a playlist, not a video")
 		return
 	}
 
@@ -232,6 +261,109 @@ func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error)
 	return
 }
 
+// FetchPlaylistVideos fetches all videos from a YouTube playlist.
+func (o *YouTube) FetchPlaylistVideos(playlistID string) (ret []*VideoMeta, err error) {
+	if err = o.initService(); err != nil {
+		return
+	}
+
+	nextPageToken := ""
+	for {
+		call := o.service.PlaylistItems.List([]string{"snippet"}).PlaylistId(playlistID).MaxResults(50)
+		if nextPageToken != "" {
+			call = call.PageToken(nextPageToken)
+		}
+
+		var response *youtube.PlaylistItemListResponse
+		if response, err = call.Do(); err != nil {
+			return
+		}
+
+		for _, item := range response.Items {
+			videoID := item.Snippet.ResourceId.VideoId
+			title := item.Snippet.Title
+			ret = append(ret, &VideoMeta{videoID, title, o.normalizeFileName(title)})
+		}
+
+		nextPageToken = response.NextPageToken
+		if nextPageToken == "" {
+			break
+		}
+
+		time.Sleep(1 * time.Second) // Pause to respect API rate limit
+	}
+	return
+}
+
+// SaveVideosToCSV saves the list of videos to a CSV file.
+func (o *YouTube) SaveVideosToCSV(filename string, videos []*VideoMeta) (err error) {
+	var file *os.File
+	if file, err = os.Create(filename); err != nil {
+		return
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
+
+	// Write headers
+	if err = writer.Write([]string{"VideoID", "Title"}); err != nil {
+		return
+	}
+
+	// Write video data
+	for _, record := range videos {
+		if err = writer.Write([]string{record.Id, record.Title}); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// FetchAndSavePlaylist fetches all videos in a playlist and saves them to a CSV file.
+func (o *YouTube) FetchAndSavePlaylist(playlistID, filename string) (err error) {
+	var videos []*VideoMeta
+	if videos, err = o.FetchPlaylistVideos(playlistID); err != nil {
+		err = fmt.Errorf("error fetching playlist videos: %v", err)
+		return
+	}
+
+	if err = o.SaveVideosToCSV(filename, videos); err != nil {
+		err = fmt.Errorf("error saving videos to CSV: %v", err)
+		return
+	}
+
+	fmt.Println("Playlist saved to", filename)
+	return
+}
+
+func (o *YouTube) FetchAndPrintPlaylist(playlistID string) (err error) {
+	var videos []*VideoMeta
+	if videos, err = o.FetchPlaylistVideos(playlistID); err != nil {
+		err = fmt.Errorf("error fetching playlist videos: %v", err)
+		return
+	}
+
+	fmt.Printf("Playlist: %s\n", playlistID)
+	fmt.Printf("VideoId: Title\n")
+	for _, video := range videos {
+		fmt.Printf("%s: %s\n", video.Id, video.Title)
+	}
+	return
+}
+
+func (o *YouTube) normalizeFileName(name string) string {
+	return o.normalizeRegex.ReplaceAllString(name, "_")
+
+}
+
+type VideoMeta struct {
+	Id              string
+	Title           string
+	TitleNormalized string
+}
+
 type Options struct {
 	Duration   bool
 	Transcript bool

shell.nix

@@ -0,0 +1,30 @@
+{
+  pkgs,
+  gomod2nix,
+  goEnv,
+}:
+
+{
+  default = pkgs.mkShell {
+    nativeBuildInputs = [
+      pkgs.go
+      pkgs.gopls
+      pkgs.gotools
+      pkgs.go-tools
+      pkgs.goimports-reviser
+      gomod2nix
+      goEnv
+
+      (pkgs.writeShellScriptBin "update" ''
+        go get -u
+        go mod tidy
+        gomod2nix generate
+      '')
+    ];
+
+    shellHook = ''
+      echo -e "\033[0;32;4mHeper commands:\033[0m"
+      echo "'update' instead of 'go get -u && go mod tidy'"
+    '';
+  };
+}

version.go

@@ -1,3 +1,3 @@
 package main
 
-var version = "v1.4.85"
+var version = "v1.4.89"