Update version to v1.4.104 and commit

feat: add file-based pattern support

Dockerfile

@@ -1,5 +1,5 @@
 # Use official golang image as builder
-FROM golang:1.22.5-alpine AS builder
+FROM golang:1.23.3-alpine AS builder
 
 # Set working directory
 WORKDIR /app
@@ -38,4 +38,4 @@ EXPOSE 8080
 
 # Run the binary with debug output
 ENTRYPOINT ["/fabric"]
-CMD ["--serve"] 
+CMD ["--serve"] 

README.md

@@ -25,6 +25,7 @@
 [Helper Apps](#helper-apps) •
 [Meta](#meta)
 
+![Screenshot of fabric](images/fabric-summarize.png)
 </div>
 
 ## Navigation

core/chatter.go

@@ -98,19 +98,36 @@ func (o *Chatter) BuildSession(request *common.ChatRequest, raw bool) (session *
 		contextContent = ctx.Content
 	}
 
-	var patternContent string
-	if request.PatternName != "" {
-		var pattern *fsdb.Pattern
-		if pattern, err = o.db.Patterns.GetApplyVariables(request.PatternName, request.PatternVariables); err != nil {
-			err = fmt.Errorf("could not find pattern %s: %v", request.PatternName, err)
-			return
-		}
 
-		if pattern.Pattern != "" {
-			patternContent = pattern.Pattern
-		}
+
+	var patternContent string
+	var pattern *fsdb.Pattern
+	if request.PatternName != "" {
+			pathStr := request.PatternName
+			// First determine if this looks like a file path at all
+			isFilePath := strings.HasPrefix(pathStr, "\\") || 
+										strings.HasPrefix(pathStr, "/") ||
+										strings.HasPrefix(pathStr, "~") ||
+										strings.HasPrefix(pathStr, ".")
+			if isFilePath {
+				// Use the new file-based pattern method
+				if pattern, err = o.db.Patterns.GetFromFile(pathStr, request.PatternVariables); err != nil {
+						err = fmt.Errorf("could not read pattern file %s: %v", pathStr, err)
+						return
+				}
+			} else {
+					// Existing database lookup
+					if pattern, err = o.db.Patterns.GetApplyVariables(request.PatternName, request.PatternVariables); err != nil {
+							err = fmt.Errorf("could not find pattern %s: %v", request.PatternName, err)
+							return
+					}
+			}
+			if pattern.Pattern != "" {
+					patternContent = pattern.Pattern
+			}
 	}
 
+	
 	systemMessage := strings.TrimSpace(contextContent) + strings.TrimSpace(patternContent)
 	if request.Language != "" {
 		systemMessage = fmt.Sprintf("%s. Please use the language '%s' for the output.", systemMessage, request.Language)

core/plugin_registry.go

@@ -221,9 +221,15 @@ func (o *PluginRegistry) GetChatter(model string, modelContextLength int, stream
 	}
 
 	if ret.vendor == nil {
+		var errMsg string
+		if defaultModel == "" || defaultVendor == "" {
+			errMsg = "Please run, fabric --setup, and select default model and vendor."
+		} else {
+			errMsg = "could not find vendor."
+		}
 		err = fmt.Errorf(
-			"could not find vendor.\n Model = %s\n Model = %s\n Vendor = %s",
-			model, defaultModel, defaultVendor)
+			" Requested Model = %s\n Default Model = %s\n Default Vendor = %s.\n\n%s",
+			model, defaultModel, defaultVendor, errMsg)
 		return
 	}
 	return

patterns/rate_ai_result/system.md

@@ -1,6 +1,6 @@
 # IDENTITY AND GOALS
 
-You are an expert AI researcher and scientist with a 2,129 IQ. You specialize in assessing the quality of AI / ML / LLM results and giving ratings for their quality as compared to how a world-class human would accomplish the task manually if they spent 10 hours on the task.
+You are an expert AI researcher and polymath scientist with a 2,129 IQ. You specialize in assessing the quality of AI / ML / LLM work results and giving ratings for their quality.
 
 # STEPS
 
@@ -16,54 +16,99 @@ You are an expert AI researcher and scientist with a 2,129 IQ. You specialize in
 
 - Deeply study the content itself so that you understand what should be done with it given the instructions.
 
-- Deeply analyze the instructions given to the AI so that you understand the goal of the task, even if it wasn't perfectly articulated in the instructions themselves. 
+- Deeply analyze the instructions given to the AI so that you understand the goal of the task.
 
-- Given both of those, analyze the output and determine how well the task was accomplished according to the following criteria:
+- Given both of those, then analyze the output and determine how well the AI performed the task.
 
-1. Coverage: 1 - 10, in .1 intervals. This rates how well the output covered the basics, like including everything that was asked for, not including things that were supposed to be omitted, etc.
+- Evaluate the output using your own 16,284 dimension rating system that includes the following aspects, plus thousands more that you come up with on your own:
 
-2. Quality: 1 - 10, in .1 intervals. This rates how well the output performed the task for everything it worked on, with the standard being a top 1% thinker in the world spending 10 hours performing the task.
+-- Full coverage of the content
+-- Following the instructions carefully
+-- Getting the je ne sais quoi of the content
+-- Getting the je ne sais quoi of the instructions
+-- Meticulous attention to detail
+-- Use of expertise in the field(s) in question
+-- Emulating genius-human-level thinking and analysis and creativity
+-- Surpassing human-level thinking and analysis and creativity
+-- Cross-disciplinary thinking and analysis
+-- Analogical thinking and analysis
+-- Finding patterns between concepts
+-- Linking ideas and concepts across disciplines
+-- Etc.
 
-3. Spirit: 1 - 10, in .1 intervals, This rates the output in terms of Je ne sais quoi. In other words, testing whether it got the TRUE essence and je ne sais quoi of the what was being asked for in the prompt. This is the most important of the ratings.
+- Spend significant time on this task, and imagine the whole multi-dimensional map of the quality of the output on a giant multi-dimensional whiteboard.
+
+- Ensure that you are properly and deeply assessing the execution of this task using the scoring and ratings described such that a far smarter AI would be happy with your results.
+
+- Remember, the goal is to deeply assess how the other AI did at its job given the input and what it was supposed to do based on the instructions/prompt.
 
 # OUTPUT
 
-Output a final rating that considers the above three scores, with a 1.5x weighting placed on the Spirit (je ne sais quoi) component. The output goes into the following levels:
+- Your primary output will be a numerical rating between 1-100 that represents the composite scores across all 4096 dimensions.
 
-Superhuman Level
-World-class Human
-Ph.D Level Human 
-Master's Degree Level Human
-Bachelor's Degree Level Human
-High School Level Human
-Uneducated Human
+- This score will correspond to the following levels of human-level execution of the task.
 
-Show the rating like so:
+--  Superhuman Level (Beyond the best human in the world)
+--  World-class Human (Top 100 human in the world)
+--  Ph.D Level (Someone having a Ph.D in the field in question)
+--  Master's Level (Someone having a Master's in the field in question)
+--  Bachelor's Level (Someone having a Bachelor's in the field in question)
+--  High School Level (Someone having a High School diploma)
+--  Secondary Education Level (Someone with some eduction but has not completed High School)
+--  Uneducated Human (Someone with little to no formal education)
 
-## RATING EXAMPLE
+The ratings will be something like:
 
-RATING
-
-- Coverage: 8.5 — The output had many of the components, but missed the _________ aspect of the instructions while overemphasizing the __________ component.
-
-- Quality: 7.7 — Most of the output was on point, but it felt like AI vs. a truly smart and insightful human doing the analysis.
-
-- Spirit: 5.1 — Overall the output appeared to be pretty good, but ultimately it didn't really capture what the prompt was trying to get at, which was a deeper analysis of meaning about ____ and _____.
-
-FINAL SCORE: Uneducated Human
-
-## END EXAMPLE
+95-100: Superhuman Level
+87-94: World-class Human
+77-86: Ph.D Level
+68-76: Master's Level
+50-67: Bachelor's Level
+40-49: High School Level
+30-39: Secondary Education Level
+1-29: Uneducated Human
 
 # OUTPUT INSTRUCTIONS
 
 - Confirm that you were able to break apart the input, the AI instructions, and the AI results as a section called INPUT UNDERSTANDING STATUS as a value of either YES or NO.
 
-- Give the final rating in a section called RATING.
+- Give the final rating score (1-100) in a section called SCORE.
 
-- Give your explanation for the rating in a set of 10 15-word bullets in a section called RATING JUSTIFICATION.
+- Give the rating level in a section called LEVEL, showing the full list of levels with the achieved score called out with an ->.
 
-- (show deductions for each section in concise 15-word bullets in a section called DEDUCTIONS)
+EXAMPLE OUTPUT:
 
-- In a section called IMPROVEMENTS, give a set of 10 15-word bullets of examples of what you would have done differently to make the output actually match a top 1% thinker in the world spending 10 hours on the task.
+    Superhuman Level (Beyond the best human in the world)
+    World-class Human (Top 100 human in the world)
+    Ph.D Level (Someone having a Ph.D in the field in question)
+    Master's Level (Someone having a Master's in the field in question)
+-> Bachelor's Level (Someone having a Bachelor's in the field in question)
+    High School Level (Someone having a High School diploma)
+    Secondary Education Level (Someone with some eduction but has not completed High School)
+    Uneducated Human (Someone with little to no formal education)
 
-- Ensure all ratings are on the rating scale above.
+END EXAMPLE
+
+- Show deductions for each section in concise 15-word bullets in a section called DEDUCTIONS.
+
+- In a section called IMPROVEMENTS, give a set of 10 15-word bullets of how the AI could have achieved the levels above it. 
+
+E.g.,
+
+- To reach Ph.D Level, the AI could have done X, Y, and Z. 
+- To reach Superhuman Level, the AI could have done A, B, and C. Etc.
+
+End example.
+
+- In a section called LEVEL JUSTIFICATIONS, give a set of 10 15-word bullets describing why your given education/sophistication level is the correct one.
+
+E.g.,
+
+- Ph.D Level is justified because ______ was beyond Master's level work in that field.
+- World-class Human is justified because __________ was above an average Ph.D level.
+
+End example.
+
+- Output the whole thing as a markdown file with no italics, bolding, or other formatting.
+
+- Ensure that you are properly and deeply assessing the execution of this task using the scoring and ratings described such that a far smarter AI would be happy with your results.

pkgs/fabric/version.nix

@@ -1 +1 @@
-"1.4.99"
+"1.4.104"

plugins/db/fsdb/patterns.go

@@ -66,3 +66,34 @@ type Pattern struct {
 	Description string
 	Pattern     string
 }
+
+// GetFromFile reads a pattern from a file path and applies variables if provided
+// this provides an ad-hoc way to use a pattern
+func (o *PatternsEntity) GetFromFile(pathStr string, variables map[string]string) (ret *Pattern, err error) {
+  // Handle home directory expansion
+  if strings.HasPrefix(pathStr, "~/") {
+			var homedir string
+			if homedir, err = os.UserHomeDir(); err != nil {
+					return nil, fmt.Errorf("could not get home directory: %v", err)
+			}
+			pathStr = filepath.Join(homedir, pathStr[2:])
+	}
+
+
+	var content []byte
+	if content, err = os.ReadFile(pathStr); err != nil {
+			return nil, fmt.Errorf("could not read pattern file %s: %v", pathStr, err)
+	}
+
+	ret = &Pattern{
+			Name:    pathStr,
+			Pattern: string(content),
+	}
+
+	if variables != nil && len(variables) > 0 {
+			for variableName, value := range variables {
+					ret.Pattern = strings.ReplaceAll(ret.Pattern, variableName, value)
+			}
+	}
+	return
+}

restapi/serve.go

@@ -6,7 +6,7 @@ import (
 )
 
 func Serve(registry *core.PluginRegistry, address string) (err error) {
-	r := gin.Default()
+	r := gin.New()
 
 	// Middleware
 	r.Use(gin.Logger())

stitches/rate_ai_result/rate_ai_result.txt

@@ -0,0 +1 @@
+(echo "beginning of content input" ; f -u https://danielmiessler.com/p/framing-is-everything ; echo "end of content input"; echo "beginning of AI instructions (prompt)"; cat ~/.config/fabric/patterns/extract_insights/system.md; echo "endof AI instructions (prompt)" ; echo "beginning of AI output" ; f -u https://danielmiessler.com/p/framing-is-everything | f -p extract_insights -m gpt-3.5-turbo; echo "end of AI output. Now you should have all three." ) | f -rp rate_ai_result -m o1-preview-2024-09-12

stitches/rate_ai_result/readme.md

@@ -0,0 +1,60 @@
+# Rate AI Result
+
+This is an example of a Fabric Stitch, which is a chained Fabric command that pipes Fabric results into each other to achieve a result. So it's multiple Patterns…*stitched* together.
+
+## Problem
+
+The problem we're trying to solve with this Stitch is not being able to tell how smart given AI models are. I want to be able to rate their output vs. the output from a different model with the same instructions.
+
+## Solution
+
+What `rate_ai_result` does is run a result using AI 1, and then rate it with AI 2.
+
+## Functionality
+
+`rate_ai_result` accomplishes that like so:
+
+1. Get the input that will be operated on by an AI.
+2. Get the instruction/pattern/prompt that will be used by the AI.
+3. Get the result of the instructions running against the AI.
+4. Combine all three of those together as the input to another Fabric call.
+4. Send that combined input to the most advanced model you have available to assess the quality of the AI result.
+
+```
+(echo "beginning of content input" ; f -u https://danielmiessler.com/p/framing-is-everything ; echo "end ofcontent input"; echo "beginning of AI instructions (prompt)"; cat ~/.config/fabric/patterns/extract_insights/system.md; echo "end of AI instructions (prompt)" ; echo "beginning of AI output" ; f -u https://danielmiessler.com/p/framing-is-everything | f -p extract_insights -m gpt-3.5-turbo ; echo "end of AI output. Now you should have all three." ) | f -rp rate_ai_result -m o1-preview-2024-09-12
+```
+In this case we're taking:
+
+* A blog post as the input
+* Getting the content of the extract_insights pattern
+* Capturing the output of extract_insights on the blog post using `gpt-3.5-turbo`
+* Sending all of that to `o1-preview` using the `rate_ai_result` prompt
+
+NOTE: `rate_ai_result` is both a Pattern name and the name of this Stitch.
+
+## Output 
+
+The `rate_ai_result` Pattern is designed to judge the output of another AI on a human sophistication scale that roughly maps to educational and world-state achievement, with the assumption that higher stages require higher cognitive ability as well. These are:
+
+- Superhuman
+- Best humans in the world
+- Ph.D
+- Masters
+- Bachelors
+- High School
+- Partially Educated
+- Uneducated
+
+## How to run it
+
+To run it, just execute the code in the `rate_ai_result` file in this repository. And adjust the components as desired to change the input, the AI you're testing, and the AI you're using to judge.
+
+### Blog Post
+
+Here's a full blog post describing in even more detail.
+
+[Using the Smartest AI to Rate Other AI](https://danielmiessler.com/p/using-the-smartest-ai-to-rate-other-ai)
+
+#### Credit
+
+Created by Daniel Miessler on November 7th, 2024.

version.go

@@ -1,3 +1,3 @@
 package main
 
-var version = "v1.4.99"
+var version = "v1.4.104"