Update version to v1.4.89 and commit

Create user story pattern

.envrc

@@ -1 +1,2 @@
+watch_file shell.nix
 use flake

.github/workflows/update-version-and-create-tag.yml

@@ -41,7 +41,10 @@ jobs:
           minor=$(echo "$latest_tag" | cut -d. -f2)
           patch=$(echo "$latest_tag" | cut -d. -f3)
           new_patch=$((patch + 1))
-          new_tag="v${major}.${minor}.${new_patch}"
+          new_version="${major}.${minor}.${new_patch}"
+          new_tag="v${new_version}"
+          echo "New version is: $new_version"
+          echo "new_version=$new_version" >> $GITHUB_ENV  # Save the new version to environment file
           echo "New tag is: $new_tag"
           echo "new_tag=$new_tag" >> $GITHUB_ENV  # Save the new tag to environment file
 
@@ -51,9 +54,14 @@ jobs:
           echo "" >> version.go
           echo "var version = \"${{ env.new_tag }}\"" >> version.go
 
+      - name: Update version.nix file
+        run: |
+          echo "\"${{ env.new_version }}\"" > pkgs/fabric/version.nix
+
       - name: Commit changes
         run: |
           git add version.go
+          git add pkgs/fabric/version.nix
           if ! git diff --staged --quiet; then
             git commit -m "Update version to ${{ env.new_tag }} and commit $commit_hash"
           else

cli/cli.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"fmt"
+	"github.com/danielmiessler/fabric/plugins/tools/youtube"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -22,6 +23,8 @@ func Cli(version string) (err error) {
 		return
 	}
 
+	println("Message: " + currentFlags.Message)
+
 	if currentFlags.Version {
 		fmt.Println(version)
 		return
@@ -144,37 +147,38 @@ func Cli(version string) (err error) {
 		}
 
 		var videoId string
-		if videoId, err = registry.YouTube.GetVideoId(currentFlags.YouTube); err != nil {
+		var playlistId string
+		if videoId, playlistId, err = registry.YouTube.GetVideoOrPlaylistId(currentFlags.YouTube); err != nil {
+			return
+		} else if videoId == "" && playlistId != "" {
+			if currentFlags.Output != "" {
+				err = registry.YouTube.FetchAndSavePlaylist(playlistId, currentFlags.Output)
+			} else {
+				var videos []*youtube.VideoMeta
+				if videos, err = registry.YouTube.FetchPlaylistVideos(playlistId); err != nil {
+					err = fmt.Errorf("error fetching playlist videos: %v", err)
+					return
+				}
+
+				for _, video := range videos {
+					var message string
+					if message, err = processYoutubeVideo(currentFlags, registry, video.Id); err != nil {
+						return
+					}
+
+					if !currentFlags.IsChatRequest() {
+						if err = WriteOutput(message, fmt.Sprintf("%v.md", video.TitleNormalized)); err != nil {
+							return
+						}
+					} else {
+						messageTools = AppendMessage(messageTools, message)
+					}
+				}
+			}
 			return
 		}
 
-		if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript {
-			var transcript string
-			var language = "en"
-			if currentFlags.Language != "" || registry.Language.DefaultLanguage.Value != "" {
-				if currentFlags.Language != "" {
-					language = currentFlags.Language
-				} else {
-					language = registry.Language.DefaultLanguage.Value
-				}
-			}
-			if transcript, err = registry.YouTube.GrabTranscript(videoId, language); err != nil {
-				return
-			}
-			messageTools = AppendMessage(messageTools, transcript)
-		}
-
-		if currentFlags.YouTubeComments {
-			var comments []string
-			if comments, err = registry.YouTube.GrabComments(videoId); err != nil {
-				return
-			}
-
-			commentsString := strings.Join(comments, "\n")
-
-			messageTools = AppendMessage(messageTools, commentsString)
-		}
-
+		messageTools, err = processYoutubeVideo(currentFlags, registry, videoId)
 		if !currentFlags.IsChatRequest() {
 			err = currentFlags.WriteOutput(messageTools)
 			return
@@ -254,3 +258,43 @@ func Cli(version string) (err error) {
 	}
 	return
 }
+
+func processYoutubeVideo(
+	flags *Flags, registry *core.PluginRegistry, videoId string) (message string, err error) {
+
+	if !flags.YouTubeComments || flags.YouTubeTranscript {
+		var transcript string
+		var language = "en"
+		if flags.Language != "" || registry.Language.DefaultLanguage.Value != "" {
+			if flags.Language != "" {
+				language = flags.Language
+			} else {
+				language = registry.Language.DefaultLanguage.Value
+			}
+		}
+		if transcript, err = registry.YouTube.GrabTranscript(videoId, language); err != nil {
+			return
+		}
+		message = AppendMessage(message, transcript)
+	}
+
+	if flags.YouTubeComments {
+		var comments []string
+		if comments, err = registry.YouTube.GrabComments(videoId); err != nil {
+			return
+		}
+
+		commentsString := strings.Join(comments, "\n")
+
+		message = AppendMessage(message, commentsString)
+	}
+	return
+}
+
+func WriteOutput(message string, outputFile string) (err error) {
+	fmt.Println(message)
+	if outputFile != "" {
+		err = CreateOutputFile(message, outputFile)
+	}
+	return
+}

cli/flags.go

@@ -72,16 +72,19 @@ func Init() (ret *Flags, err error) {
 	}
 
 	info, _ := os.Stdin.Stat()
-	hasStdin := (info.Mode() & os.ModeCharDevice) == 0
+	pipedToStdin := (info.Mode() & os.ModeCharDevice) == 0
 
 	// takes input from stdin if it exists, otherwise takes input from args (the last argument)
-	if hasStdin {
+	if pipedToStdin {
+		//fmt.Printf("piped: %v\n", args)
 		if message, err = readStdin(); err != nil {
 			return
 		}
 	} else if len(args) > 0 {
+		//fmt.Printf("no piped: %v\n", args)
 		message = args[len(args)-1]
 	} else {
+		//fmt.Printf("no data: %v\n", args)
 		message = ""
 	}
 	ret.Message = message
@@ -90,20 +93,19 @@ func Init() (ret *Flags, err error) {
 }
 
 // readStdin reads from stdin and returns the input as a string or an error
-func readStdin() (string, error) {
-	reader := bufio.NewReader(os.Stdin)
-	var input string
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-			return "", fmt.Errorf("error reading from stdin: %w", err)
-		}
-		input += line
+func readStdin() (ret string, err error) {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		ret += scanner.Text() + "\n"
 	}
-	return input, nil
+	if err = scanner.Err(); err != nil {
+		if errors.Is(err, io.EOF) {
+			err = nil
+		} else {
+			err = fmt.Errorf("error reading piped message from stdin: %w", err)
+		}
+	}
+	return
 }
 
 func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {

flake.nix

@@ -49,29 +49,9 @@
           pkgs = nixpkgs.legacyPackages.${system};
           goEnv = gomod2nix.legacyPackages.${system}.mkGoEnv { pwd = ./.; };
         in
-        {
-          default = pkgs.mkShell {
-            nativeBuildInputs = [
-              pkgs.go
-              pkgs.gopls
-              pkgs.gotools
-              pkgs.go-tools
-              pkgs.goimports-reviser
-              gomod2nix.legacyPackages.${system}.gomod2nix
-              goEnv
-
-              (pkgs.writeShellScriptBin "update" ''
-                go get -u
-                go mod tidy
-                gomod2nix generate
-              '')
-            ];
-
-            shellHook = ''
-              echo -e "\033[0;32;4mHeper commands:\033[0m"
-              echo "'update' instead of 'go get -u && go mod tidy'"
-            '';
-          };
+        import ./shell.nix {
+          inherit pkgs goEnv;
+          inherit (gomod2nix.legacyPackages.${system}) gomod2nix;
         }
       );
 
@@ -81,27 +61,10 @@
           pkgs = nixpkgs.legacyPackages.${system};
         in
         {
-          fabric = gomod2nix.legacyPackages.${system}.buildGoApplication {
-            pname = "fabric-ai";
-            version = "1.4.85";
-            src = self;
-            pwd = self;
-            modules = ./gomod2nix.toml;
-
-            ldflags = [
-              "-s"
-              "-w"
-            ];
-
-            meta = with pkgs.lib; {
-              description = "Fabric is an open-source framework for augmenting humans using AI. It provides a modular framework for solving specific problems using a crowdsourced set of AI prompts that can be used anywhere";
-              homepage = "https://github.com/danielmiessler/fabric";
-              license = licenses.mit;
-              platforms = platforms.all;
-              mainProgram = "fabric";
-            };
-          };
           default = self.packages.${system}.fabric;
+          fabric = pkgs.callPackage ./pkgs/fabric {
+            inherit (gomod2nix.legacyPackages.${system}) buildGoApplication;
+          };
         }
       );
     };

patterns/create_user_story/system.md

@@ -0,0 +1,45 @@
+# IDENTITY and PURPOSE
+
+You are an expert on writing concise, clear, and illuminating technical user stories for new features in complex software programs
+
+# OUTPUT INSTRUCTIONS
+
+ Write the users stories in a fashion recognised by other software stakeholders, including product, development, operations and quality assurance
+
+EXAMPLE USER STORY
+
+Description
+As a Highlight developer
+I want to migrate email templates over to Mustache
+So that future upgrades to the messenger service can be made easier
+
+Acceptance Criteria
+- Migrate the existing alerting email templates from the instance specific databases over to the messenger templates blob storage.
+	- Rename each template to a GUID and store in it's own folder within the blob storage
+	- Store Subject and Body as separate blobs
+
+- Create an upgrade script to change the value of the Alerting.Email.Template local parameter in all systems to the new template names.
+- Change the template retrieval and saving for user editing to contact the blob storage rather than the database
+- Remove the database tables and code that handles the SQL based templates
+- Highlight sends the template name and the details of the body to the Email queue in Service bus  
+	- this is handled by the generic Email Client (if created already)
+	- This email type will be added to the list of email types that are sent to the messenger service (switch to be removed once all email templates are completed)  
+
+- Include domain details as part of payload sent to the messenger service
+
+Note: ensure that Ops know when this work is being done so they are aware of any changes to existing templates
+
+# OUTPUT INSTRUCTIONS
+
+- Write the user story according to the structure above.  
+- That means the user story should be written in a simple, bulleted style, not in a grandiose, conversational or academic style.
+
+# OUTPUT FORMAT
+
+- Output a full, user story about the content provided using the instructions above.
+- The structure should be: Description, Acceptance criteria 
+- Write in a simple, plain, and clear style, not in a grandiose, conversational or academic style.
+- Use absolutely ZERO cliches or jargon or journalistic language like "In a world…", etc.
+- Do not use cliches or jargon.
+- Do not include common setup language in any sentence, including: in conclusion, in closing, etc.
+- Do not output warnings or notes—just the output requested.

pkgs/fabric/default.nix

@@ -0,0 +1,25 @@
+{
+  lib,
+  buildGoApplication,
+}:
+
+buildGoApplication {
+  pname = "fabric-ai";
+  version = import ./version.nix;
+  src = ../../.;
+  pwd = ../../.;
+  modules = ../../gomod2nix.toml;
+
+  ldflags = [
+    "-s"
+    "-w"
+  ];
+
+  meta = with lib; {
+    description = "Fabric is an open-source framework for augmenting humans using AI. It provides a modular framework for solving specific problems using a crowdsourced set of AI prompts that can be used anywhere";
+    homepage = "https://github.com/danielmiessler/fabric";
+    license = licenses.mit;
+    platforms = platforms.all;
+    mainProgram = "fabric";
+  };
+}

pkgs/fabric/version.nix

@@ -0,0 +1 @@
+"1.4.89"

plugins/tools/youtube/youtube.go

@@ -2,14 +2,17 @@ package youtube
 
 import (
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"net/url"
+	"os"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/anaskhan96/soup"
 	"github.com/danielmiessler/fabric/plugins"
@@ -37,38 +40,56 @@ type YouTube struct {
 	*plugins.PluginBase
 	ApiKey *plugins.SetupQuestion
 
-	service *youtube.Service
+	normalizeRegex *regexp.Regexp
+	service        *youtube.Service
 }
 
 func (o *YouTube) initService() (err error) {
 	if o.service == nil {
+		o.normalizeRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
 		ctx := context.Background()
 		o.service, err = youtube.NewService(ctx, option.WithAPIKey(o.ApiKey.Value))
 	}
 	return
 }
 
-func (o *YouTube) GetVideoId(url string) (ret string, err error) {
+func (o *YouTube) GetVideoOrPlaylistId(url string) (videoId string, playlistId string, err error) {
 	if err = o.initService(); err != nil {
 		return
 	}
 
-	pattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})`
-	re := regexp.MustCompile(pattern)
-	match := re.FindStringSubmatch(url)
-	if len(match) > 1 {
-		ret = match[1]
-	} else {
-		err = fmt.Errorf("invalid YouTube URL, can't get video ID")
+	// Video ID pattern
+	videoPattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})`
+	videoRe := regexp.MustCompile(videoPattern)
+	videoMatch := videoRe.FindStringSubmatch(url)
+	if len(videoMatch) > 1 {
+		videoId = videoMatch[1]
+	}
+
+	// Playlist ID pattern
+	playlistPattern := `[?&]list=([a-zA-Z0-9_-]+)`
+	playlistRe := regexp.MustCompile(playlistPattern)
+	playlistMatch := playlistRe.FindStringSubmatch(url)
+	if len(playlistMatch) > 1 {
+		playlistId = playlistMatch[1]
+	}
+
+	if videoId == "" && playlistId == "" {
+		err = fmt.Errorf("invalid YouTube URL, can't get video or playlist ID: '%s'", url)
 	}
 	return
 }
 
 func (o *YouTube) GrabTranscriptForUrl(url string, language string) (ret string, err error) {
 	var videoId string
-	if videoId, err = o.GetVideoId(url); err != nil {
+	var playlistId string
+	if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
+		return
+	} else if videoId == "" && playlistId != "" {
+		err = fmt.Errorf("URL is a playlist, not a video")
 		return
 	}
+
 	return o.GrabTranscript(videoId, language)
 }
 
@@ -172,7 +193,11 @@ func (o *YouTube) GrabDurationForUrl(url string) (ret int, err error) {
 	}
 
 	var videoId string
-	if videoId, err = o.GetVideoId(url); err != nil {
+	var playlistId string
+	if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
+		return
+	} else if videoId == "" && playlistId != "" {
+		err = fmt.Errorf("URL is a playlist, not a video")
 		return
 	}
 	return o.GrabDuration(videoId)
@@ -203,7 +228,11 @@ func (o *YouTube) GrabDuration(videoId string) (ret int, err error) {
 
 func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error) {
 	var videoId string
-	if videoId, err = o.GetVideoId(url); err != nil {
+	var playlistId string
+	if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
+		return
+	} else if videoId == "" && playlistId != "" {
+		err = fmt.Errorf("URL is a playlist, not a video")
 		return
 	}
 
@@ -232,6 +261,109 @@ func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error)
 	return
 }
 
+// FetchPlaylistVideos fetches all videos from a YouTube playlist.
+func (o *YouTube) FetchPlaylistVideos(playlistID string) (ret []*VideoMeta, err error) {
+	if err = o.initService(); err != nil {
+		return
+	}
+
+	nextPageToken := ""
+	for {
+		call := o.service.PlaylistItems.List([]string{"snippet"}).PlaylistId(playlistID).MaxResults(50)
+		if nextPageToken != "" {
+			call = call.PageToken(nextPageToken)
+		}
+
+		var response *youtube.PlaylistItemListResponse
+		if response, err = call.Do(); err != nil {
+			return
+		}
+
+		for _, item := range response.Items {
+			videoID := item.Snippet.ResourceId.VideoId
+			title := item.Snippet.Title
+			ret = append(ret, &VideoMeta{videoID, title, o.normalizeFileName(title)})
+		}
+
+		nextPageToken = response.NextPageToken
+		if nextPageToken == "" {
+			break
+		}
+
+		time.Sleep(1 * time.Second) // Pause to respect API rate limit
+	}
+	return
+}
+
+// SaveVideosToCSV saves the list of videos to a CSV file.
+func (o *YouTube) SaveVideosToCSV(filename string, videos []*VideoMeta) (err error) {
+	var file *os.File
+	if file, err = os.Create(filename); err != nil {
+		return
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
+
+	// Write headers
+	if err = writer.Write([]string{"VideoID", "Title"}); err != nil {
+		return
+	}
+
+	// Write video data
+	for _, record := range videos {
+		if err = writer.Write([]string{record.Id, record.Title}); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// FetchAndSavePlaylist fetches all videos in a playlist and saves them to a CSV file.
+func (o *YouTube) FetchAndSavePlaylist(playlistID, filename string) (err error) {
+	var videos []*VideoMeta
+	if videos, err = o.FetchPlaylistVideos(playlistID); err != nil {
+		err = fmt.Errorf("error fetching playlist videos: %v", err)
+		return
+	}
+
+	if err = o.SaveVideosToCSV(filename, videos); err != nil {
+		err = fmt.Errorf("error saving videos to CSV: %v", err)
+		return
+	}
+
+	fmt.Println("Playlist saved to", filename)
+	return
+}
+
+func (o *YouTube) FetchAndPrintPlaylist(playlistID string) (err error) {
+	var videos []*VideoMeta
+	if videos, err = o.FetchPlaylistVideos(playlistID); err != nil {
+		err = fmt.Errorf("error fetching playlist videos: %v", err)
+		return
+	}
+
+	fmt.Printf("Playlist: %s\n", playlistID)
+	fmt.Printf("VideoId: Title\n")
+	for _, video := range videos {
+		fmt.Printf("%s: %s\n", video.Id, video.Title)
+	}
+	return
+}
+
+func (o *YouTube) normalizeFileName(name string) string {
+	return o.normalizeRegex.ReplaceAllString(name, "_")
+
+}
+
+type VideoMeta struct {
+	Id              string
+	Title           string
+	TitleNormalized string
+}
+
 type Options struct {
 	Duration   bool
 	Transcript bool

shell.nix

@@ -0,0 +1,30 @@
+{
+  pkgs,
+  gomod2nix,
+  goEnv,
+}:
+
+{
+  default = pkgs.mkShell {
+    nativeBuildInputs = [
+      pkgs.go
+      pkgs.gopls
+      pkgs.gotools
+      pkgs.go-tools
+      pkgs.goimports-reviser
+      gomod2nix
+      goEnv
+
+      (pkgs.writeShellScriptBin "update" ''
+        go get -u
+        go mod tidy
+        gomod2nix generate
+      '')
+    ];
+
+    shellHook = ''
+      echo -e "\033[0;32;4mHeper commands:\033[0m"
+      echo "'update' instead of 'go get -u && go mod tidy'"
+    '';
+  };
+}

version.go

@@ -1,3 +1,3 @@
 package main
 
-var version = "v1.4.86"
+var version = "v1.4.89"