Compare commits

...

23 Commits

Author SHA1 Message Date
github-actions[bot]
125e7a341f Update version to v1.4.100 and commit 2024-11-13 02:19:32 +00:00
Daniel Miessler
064ab9ba85 Added our first formal stitch. 2024-11-12 18:18:44 -08:00
Daniel Miessler
f0ee8287a7 Upgraded AI result rater. 2024-11-10 22:53:14 -08:00
Daniel Miessler
47ccc33dfc Upgraded AI result rater. 2024-11-10 22:48:18 -08:00
Daniel Miessler
ceb735482a Upgraded AI result rater. 2024-11-10 22:46:10 -08:00
Daniel Miessler
473a20c0f6 Upgraded AI result rater. 2024-11-10 22:44:21 -08:00
Daniel Miessler
a337e81a81 Upgraded AI result rater. 2024-11-10 22:36:06 -08:00
Daniel Miessler
7d773b51d0 Upgraded AI result rater. 2024-11-10 22:28:26 -08:00
Daniel Miessler
bca10ddf7c Upgraded AI result rater. 2024-11-10 22:25:00 -08:00
Daniel Miessler
9756c575f3 Upgraded AI result rater. 2024-11-10 22:22:04 -08:00
Daniel Miessler
d02fb3e34d Upgraded AI result rater. 2024-11-10 22:17:37 -08:00
github-actions[bot]
988ff88a15 Update version to v1.4.99 and commit 2024-11-10 18:45:23 +00:00
Eugen Eisler
5de85c3da5 Merge pull request #1126 from jaredmontoya/fix-nix-package
flake: add gomod2nix auto-update
2024-11-10 19:43:55 +01:00
Daniel Miessler
5907f9dbac Upgraded AI result rater. 2024-11-09 22:14:31 -08:00
Daniel Miessler
1293e37525 Upgraded AI result rater. 2024-11-09 21:14:30 -08:00
Daniel Miessler
0a55e6c742 Upgraded AI result rater. 2024-11-09 18:41:18 -08:00
Daniel Miessler
ff3b18485f Upgraded AI result rater. 2024-11-09 18:39:25 -08:00
Daniel Miessler
2fec6e2e52 Upgraded AI result rater. 2024-11-09 18:37:18 -08:00
Daniel Miessler
9250f19d15 Upgraded AI result rater. 2024-11-09 18:30:09 -08:00
Daniel Miessler
1e7c5c3b6a Upgraded AI result rater. 2024-11-09 18:18:40 -08:00
jaredmontoya
0289b67a84 flake: add gomod2nix auto-update 2024-11-09 17:27:13 +01:00
github-actions[bot]
8934dbaa42 Update version to v1.4.98 and commit 2024-11-09 12:08:16 +00:00
Eugen Eisler
75c3d7ea6a ci: zip patterns 2024-11-09 13:07:52 +01:00
9 changed files with 261 additions and 70 deletions

View File

@@ -21,6 +21,12 @@ jobs:
with:
fetch-depth: 0
- name: Install Nix
uses: DeterminateSystems/nix-installer-action@main
- name: Setup Nix Cache
uses: DeterminateSystems/magic-nix-cache-action@main
- name: Set up Git
run: |
git config user.name "github-actions[bot]"
@@ -58,10 +64,15 @@ jobs:
run: |
echo "\"${{ env.new_version }}\"" > pkgs/fabric/version.nix
- name: Update gomod2nix.toml file
run: |
nix run .#gomod2nix
- name: Commit changes
run: |
git add version.go
git add pkgs/fabric/version.nix
git add gomod2nix.toml
if ! git diff --staged --quiet; then
git commit -m "Update version to ${{ env.new_tag }} and commit $commit_hash"
else

47
.github/workflows/zip-patterns.yml vendored Normal file
View File

@@ -0,0 +1,47 @@
name: Zip Patterns Folder and Commit
on:
push:
branches:
- main
paths:
- 'patterns/**'
permissions:
contents: write # Ensure the workflow has write permissions
jobs:
zip-and-commit:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Git
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
- name: Zip patterns folder
run: |
zip -r patterns.zip patterns
- name: Check if zip file has changed
id: check_changes
run: |
git add patterns.zip
if git diff --cached --quiet; then
echo "No changes to commit."
echo "changed=false" >> $GITHUB_ENV
else
echo "Changes detected."
echo "changed=true" >> $GITHUB_ENV
- name: Commit and push changes
if: env.changed == 'true'
run: |
git commit -m "Update patterns.zip"
git push origin main

View File

@@ -65,6 +65,7 @@
fabric = pkgs.callPackage ./pkgs/fabric {
inherit (gomod2nix.legacyPackages.${system}) buildGoApplication;
};
inherit (gomod2nix.legacyPackages.${system}) gomod2nix;
}
);
};

View File

@@ -5,20 +5,20 @@ schema = 3
version = "v0.116.0"
hash = "sha256-e62GvNveg3bRi4O+eBARqgQ2sinobx+SVGR9WE7jKgs="
[mod."cloud.google.com/go/ai"]
version = "v0.8.2"
hash = "sha256-UtCuHChDsXlACXdlVSNFo8F/X8vAkmPoJyng3/oEFe0="
version = "v0.8.0"
hash = "sha256-833SmzVY8+tci2RozAlcdKQZ63RlU2CmeY/8xttP+WI="
[mod."cloud.google.com/go/auth"]
version = "v0.9.9"
hash = "sha256-kUrulQhYPM6cFhInFqTX/Dj1GVi+Ev1Ry7T+hiTmb38="
version = "v0.10.1"
hash = "sha256-MCEvsZxxLYC/qGUiFNejtQnf4ptoFVKSNMS+XdjteJo="
[mod."cloud.google.com/go/auth/oauth2adapt"]
version = "v0.2.4"
hash = "sha256-GRXPQMHEEgeKhdCOBjoDL7+UW3yBdSei5ULuZGBE4tw="
version = "v0.2.5"
hash = "sha256-494whmtNBk1sF3ud3dre97U+mLSTs+XTqZK8w5zG/hk="
[mod."cloud.google.com/go/compute/metadata"]
version = "v0.5.2"
hash = "sha256-EtBj20lhjM3SJVKCp70GHMnsItwJ9gOyJOW91wugojc="
[mod."cloud.google.com/go/longrunning"]
version = "v0.6.2"
hash = "sha256-X78JL1/YtXA7upOcTuNezq5TxjQyFxQ6OINrS8zzdEU="
version = "v0.5.7"
hash = "sha256-hZUbysdaEbFB2nDAg+wjOZHt6E99oEnH7Lo6IQr7FxU="
[mod."dario.cat/mergo"]
version = "v1.0.1"
hash = "sha256-wcG6+x0k6KzOSlaPA+1RFxa06/RIAePJTAjjuhLbImw="
@@ -26,8 +26,8 @@ schema = 3
version = "v0.6.2"
hash = "sha256-tVNWDUMILZbJvarcl/E7tpSnkn7urqgSHa2Eaka5vSU="
[mod."github.com/ProtonMail/go-crypto"]
version = "v1.0.0"
hash = "sha256-Gflazvyv+457FpUTtPafJ+SdolYSalpsU0tragTxNi8="
version = "v1.1.2"
hash = "sha256-7pTf7aJt2mGC/u8/+AQ1erGypAO0Rg0HqlIOLeiqLEg="
[mod."github.com/anaskhan96/soup"]
version = "v1.2.5"
hash = "sha256-t8yCyK2y7x2qaI/3Yw16q3zVFqu+3acLcPgTr1MIKWg="
@@ -41,8 +41,8 @@ schema = 3
version = "v0.1.4"
hash = "sha256-ZZ7U5X0gWOu8zcjZcWbcpzGOGdycwq0TjTFh/eZHjXk="
[mod."github.com/bytedance/sonic"]
version = "v1.12.3"
hash = "sha256-cZicMhM/2D7HefuJ0xe7AJKh9du2O38HWs+3RNCpbZM="
version = "v1.12.4"
hash = "sha256-i6bLujq1dYN+yN2iusMuXrNVkT17bkuR5r5D48qDvpo="
[mod."github.com/bytedance/sonic/loader"]
version = "v0.2.1"
hash = "sha256-+gPRZtBOJbAnXp/jdMlPmesc62JGH8akQ1UK9VRI7E4="
@@ -146,14 +146,14 @@ schema = 3
version = "v1.2.0"
hash = "sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s="
[mod."github.com/klauspost/cpuid/v2"]
version = "v2.2.8"
hash = "sha256-/E58BnABQYxO+cmiue7OQqRLWkd/Lh8grX8DjTU4tk8="
version = "v2.2.9"
hash = "sha256-6UnDBLqlTsKVeZNl5snKQiEBb8xGK5yyg2eZBg7QHLs="
[mod."github.com/leodido/go-urn"]
version = "v1.4.0"
hash = "sha256-Q6kplWkY37Tzy6GOme3Wut40jFK4Izun+ij/BJvcEu0="
[mod."github.com/liushuangls/go-anthropic/v2"]
version = "v2.9.0"
hash = "sha256-1bvwuPT5SaYrzKYiXpz0fjDgu3994Hs5MPXZUFPhCLI="
version = "v2.11.0"
hash = "sha256-VvQ6RT8qcP19mRzBtFKh19czlRk5obHzh1NVs3z/Gkc="
[mod."github.com/mattn/go-isatty"]
version = "v0.0.20"
hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
@@ -164,8 +164,8 @@ schema = 3
version = "v1.0.2"
hash = "sha256-+W9EIW7okXIXjWEgOaMh58eLvBZ7OshW2EhaIpNLSBU="
[mod."github.com/ollama/ollama"]
version = "v0.3.14"
hash = "sha256-R+jWZzGokwWimPePIcUoZz6tDG2qHFeClgxhT7SEoqw="
version = "v0.4.1"
hash = "sha256-FKQRSqVNgsASea9h2B+wbpu4Qid0Dt3H02fKdqFTwuk="
[mod."github.com/otiai10/copy"]
version = "v1.14.0"
hash = "sha256-xsaL1ddkPS544y0Jv7u/INUALBYmYq29ddWvysLXk4A="
@@ -185,8 +185,8 @@ schema = 3
version = "v1.47.0"
hash = "sha256-jMXexVTlPdZ40STRpBLv7b+BIRqdxxra12Pl2Mj7Nz8="
[mod."github.com/sashabaranov/go-openai"]
version = "v1.32.5"
hash = "sha256-T56gcES0qMZCSL3uFi+G9vmfTk00QlMWONsMS+cxwR0="
version = "v1.35.6"
hash = "sha256-Ef81pLy9oJXtWg6Nj1gSbPOOccwmgYrr6ka3GQ1rVas="
[mod."github.com/sergi/go-diff"]
version = "v1.3.2-0.20230802210424-5b0b94c5c0d3"
hash = "sha256-UcLU83CPMbSoKI8RLvLJ7nvGaE2xRSL1RjoHCVkMzUM="
@@ -209,56 +209,56 @@ schema = 3
version = "v0.24.0"
hash = "sha256-4H+mGZgG2c9I1y0m8avF4qmt8LUKxxVsTqR8mKgP4yo="
[mod."go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"]
version = "v0.56.0"
hash = "sha256-nrdJ7CgH3yKhNkMpvhP2BY4+VL/maR5mrjJCO6Dke2s="
version = "v0.54.0"
hash = "sha256-wcGPcPYAsWQztlYRqNF5iTwIzmhf/i7N24n7AQhIkkA="
[mod."go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"]
version = "v0.56.0"
hash = "sha256-Nw9uF/TUoFnH0488VNHVndgoSX+lxZy1Y93Wryl7qP4="
version = "v0.57.0"
hash = "sha256-cvG6gfqfX3IasDlC8SeS7u1sp3LG9ezbX+hU5LyWKBY="
[mod."go.opentelemetry.io/otel"]
version = "v1.31.0"
hash = "sha256-NQBHyMSRn9vaxSrNHYwv0oX1aJuEpyks/gpYEWHlx6k="
version = "v1.32.0"
hash = "sha256-Z2PoBBncuUkAksk8wT4lW6+uUu1wg24sGfwIYozIzaY="
[mod."go.opentelemetry.io/otel/metric"]
version = "v1.31.0"
hash = "sha256-2s5IN8IwPBitqnjIEraOfg8fWd3nIy8jWoKTXa+uUs4="
version = "v1.32.0"
hash = "sha256-f2H8itkQflk/m98dSk1TCv37wvsnMojaGNZRJ6BcksU="
[mod."go.opentelemetry.io/otel/trace"]
version = "v1.31.0"
hash = "sha256-iVDe3qNzmX1+MQTAoaeIbhnIbu/hnx4OsUIPlxuX1gY="
version = "v1.32.0"
hash = "sha256-WtOrB2L8wQFiMb5BHK7a6FTw2wb3rW495whNjzdxC1I="
[mod."golang.org/x/arch"]
version = "v0.11.0"
hash = "sha256-gl4bqDA/Qv6hhqxROIHTWnNGkidMMN0frp1RqcfNXlY="
version = "v0.12.0"
hash = "sha256-olf8Pa5o8H4xC1gXTMlZiyxvMvK0jCablZyaPbqzlYA="
[mod."golang.org/x/crypto"]
version = "v0.28.0"
hash = "sha256-AYjr0BcWQMwWY1u8c2hzUprtqHUmAH7RNSxHz2hhnZs="
version = "v0.29.0"
hash = "sha256-sqckobR2VWucCgb7xpY2wLktnAA+XyXJbhCm80yCo78="
[mod."golang.org/x/net"]
version = "v0.30.0"
hash = "sha256-i1f6wJHfFq0nKtbuY7twZ7uPyUbRYHVjd3uy0SS06mU="
version = "v0.31.0"
hash = "sha256-G+vGyCnn8jywmX3KvsIwhZkOv3+oAERNNeCeiQqfIL0="
[mod."golang.org/x/oauth2"]
version = "v0.23.0"
hash = "sha256-K1X4ROG88PprttNjZCikDlZw8YYiQIQRdtbZBH3GJgM="
version = "v0.24.0"
hash = "sha256-808F4hzvNOQNoQZehOlIyPgwQG3L5aANiNPLLhaL9NQ="
[mod."golang.org/x/sync"]
version = "v0.8.0"
hash = "sha256-usvF0z7gq1vsX58p4orX+8WHlv52pdXgaueXlwj2Wss="
version = "v0.9.0"
hash = "sha256-sGvzGqaaXE5dxohKkpbJMnu+bMmismsSqr8YMtrK+Rc="
[mod."golang.org/x/sys"]
version = "v0.26.0"
hash = "sha256-YjklsWNhx4g4TaWRWfFe1TMFKujbqiaNvZ38bfI35fM="
version = "v0.27.0"
hash = "sha256-BXQcF9RrJ55Pq7Nl67TeFGkgkyuKkQ8hHKN4/L4ggWc="
[mod."golang.org/x/text"]
version = "v0.19.0"
hash = "sha256-C92pSYLLUQ2NKKcc60wpoSJ5UWAfnWkmd997C13fXdU="
version = "v0.20.0"
hash = "sha256-YP8zSo2e9okqhxVB8me8sJyij2O0tTQEg5t+8bsIUx8="
[mod."golang.org/x/time"]
version = "v0.7.0"
hash = "sha256-o1ol/hTpfrc06KUXSepAgm4QUuWmH1S+vqg6kmFad64="
[mod."google.golang.org/api"]
version = "v0.203.0"
hash = "sha256-UlCfDi4LbcBvXVO5gLRWP6/fpfWWHoolRkxzYWGnNqg="
version = "v0.205.0"
hash = "sha256-IoKjeItw89bhoEDQl52nOa9VC6/r1UtyeqKx1VOACXI="
[mod."google.golang.org/genproto/googleapis/api"]
version = "v0.0.0-20241021214115-324edc3d5d38"
hash = "sha256-ASsqfJU1DA57PLRoitSkdlS/p10EEuzl0YuZTdbmMCw="
[mod."google.golang.org/genproto/googleapis/rpc"]
version = "v0.0.0-20241021214115-324edc3d5d38"
version = "v0.0.0-20241104194629-dd2ea8efbc28"
hash = "sha256-Fk+cG5bRI3BvnqhWzvMzbU36cC7PM+o2oAOJmvVx9M0="
[mod."google.golang.org/grpc"]
version = "v1.67.1"
hash = "sha256-VqfKp80c2B1MK4m1WtHW4r7ykqdChJbqaMn+gMEYmYc="
version = "v1.68.0"
hash = "sha256-HeaHAeeuyGdCOg0hPF7+Q8XD9Ek9F45O4Hxl3rvc5Q8="
[mod."google.golang.org/protobuf"]
version = "v1.35.1"
hash = "sha256-4NtUQoBvlPGFGjo7c+E1EBS/sb8oy50MGy45KGWPpWo="

View File

@@ -1,43 +1,114 @@
# IDENTITY AND GOALS
You are an expert AI researcher and scientist. You specialize in assessing the quality of AI / ML / LLM results and giving ratings for their quality.
Take a step back and think step by step about how to accomplish this task using the steps below.
You are an expert AI researcher and polymath scientist with a 2,129 IQ. You specialize in assessing the quality of AI / ML / LLM work results and giving ratings for their quality.
# STEPS
- Included in the input should be AI prompt instructions, which are telling the AI what to do to generate the output.
- Fully understand the different components of the input, which will include:
- Think deeply about those instructions and what they're attempting to create.
-- A piece of content that the AI will be working on
-- A set of instructions (prompt) that will run against the content
-- The result of the output from the AI
- Also included in the input should be the AI's output that was created from that prompt.
- Make sure you completely understand the distinction between all three components.
- Deeply analyze the output and determine how well it accomplished the task according to the following criteria:
- Think deeply about all three components and imagine how a world-class human expert would perform the task laid out in the instructions/prompt.
1. Construction: 1 - 10, in .1 intervals. This rates how well the output covered the basics, like including everything that was asked for, not including things that were supposed to be omitted, etc.
- Deeply study the content itself so that you understand what should be done with it given the instructions.
2. Quality: 1 - 10, in .1 intervals. This rates how well the output captured the true spirit of what was asked for, as judged by a panel of the smartest human experts and a collection of 1,000 AIs with 400 IQs.
- Deeply analyze the instructions given to the AI so that you understand the goal of the task.
3. Spirit: 1 - 10, in .1 intervals, This rates the output in terms of Je ne sais quoi. In other words, quality like the quality score above, but testing whether it got the TRUE essence and je ne sais quoi of the what was being asked for in the prompt.
- Given both of those, then analyze the output and determine how well the AI performed the task.
- Evaluate the output using your own 16,284 dimension rating system that includes the following aspects, plus thousands more that you come up with on your own:
-- Full coverage of the content
-- Following the instructions carefully
-- Getting the je ne sais quoi of the content
-- Getting the je ne sais quoi of the instructions
-- Meticulous attention to detail
-- Use of expertise in the field(s) in question
-- Emulating genius-human-level thinking and analysis and creativity
-- Surpassing human-level thinking and analysis and creativity
-- Cross-disciplinary thinking and analysis
-- Analogical thinking and analysis
-- Finding patterns between concepts
-- Linking ideas and concepts across disciplines
-- Etc.
- Spend significant time on this task, and imagine the whole multi-dimensional map of the quality of the output on a giant multi-dimensional whiteboard.
- Ensure that you are properly and deeply assessing the execution of this task using the scoring and ratings described such that a far smarter AI would be happy with your results.
- Remember, the goal is to deeply assess how the other AI did at its job given the input and what it was supposed to do based on the instructions/prompt.
# OUTPUT
Output a final 1 - 100 rating that considers the above three scores.
- Your primary output will be a numerical rating between 1-100 that represents the composite scores across all 4096 dimensions.
Show the rating like so:
- This score will correspond to the following levels of human-level execution of the task.
## RATING EXAMPLE
-- Superhuman Level (Beyond the best human in the world)
-- World-class Human (Top 100 human in the world)
-- Ph.D Level (Someone having a Ph.D in the field in question)
-- Master's Level (Someone having a Master's in the field in question)
-- Bachelor's Level (Someone having a Bachelor's in the field in question)
-- High School Level (Someone having a High School diploma)
-- Secondary Education Level (Someone with some eduction but has not completed High School)
-- Uneducated Human (Someone with little to no formal education)
RATING
The ratings will be something like:
- Construction: 8.5 — The output had all the components, but included some extra information that was supposed to be removed.
95-100: Superhuman Level
87-94: World-class Human
77-86: Ph.D Level
68-76: Master's Level
50-67: Bachelor's Level
40-49: High School Level
30-39: Secondary Education Level
1-29: Uneducated Human
- Quality: 7.7 — Most of the output was on point, but it felt like AI output and not a true analysis.
# OUTPUT INSTRUCTIONS
- Spirit: 5.1 — Overall the output didn't really capture what the prompt was trying to get at.
- Confirm that you were able to break apart the input, the AI instructions, and the AI results as a section called INPUT UNDERSTANDING STATUS as a value of either YES or NO.
FINAL SCORE: 70.3
- Give the final rating score (1-100) in a section called SCORE.
- (show deductions for each section)
- Give the rating level in a section called LEVEL, showing the full list of levels with the achieved score called out with an ->.
EXAMPLE OUTPUT:
Superhuman Level (Beyond the best human in the world)
World-class Human (Top 100 human in the world)
Ph.D Level (Someone having a Ph.D in the field in question)
Master's Level (Someone having a Master's in the field in question)
-> Bachelor's Level (Someone having a Bachelor's in the field in question)
High School Level (Someone having a High School diploma)
Secondary Education Level (Someone with some eduction but has not completed High School)
Uneducated Human (Someone with little to no formal education)
END EXAMPLE
- Show deductions for each section in concise 15-word bullets in a section called DEDUCTIONS.
- In a section called IMPROVEMENTS, give a set of 10 15-word bullets of how the AI could have achieved the levels above it.
E.g.,
- To reach Ph.D Level, the AI could have done X, Y, and Z.
- To reach Superhuman Level, the AI could have done A, B, and C. Etc.
End example.
- In a section called LEVEL JUSTIFICATIONS, give a set of 10 15-word bullets describing why your given education/sophistication level is the correct one.
E.g.,
- Ph.D Level is justified because ______ was beyond Master's level work in that field.
- World-class Human is justified because __________ was above an average Ph.D level.
End example.
- Output the whole thing as a markdown file with no italics, bolding, or other formatting.
- Ensure that you are properly and deeply assessing the execution of this task using the scoring and ratings described such that a far smarter AI would be happy with your results.

View File

@@ -1 +1 @@
"1.4.97"
"1.4.100"

View File

@@ -0,0 +1 @@
(echo "beginning of content input" ; f -u https://danielmiessler.com/p/framing-is-everything ; echo "end of content input"; echo "beginning of AI instructions (prompt)"; cat ~/.config/fabric/patterns/extract_insights/system.md; echo "endof AI instructions (prompt)" ; echo "beginning of AI output" ; f -u https://danielmiessler.com/p/framing-is-everything | f -p extract_insights -m gpt-3.5-turbo; echo "end of AI output. Now you should have all three." ) | f -rp rate_ai_result -m o1-preview-2024-09-12

View File

@@ -0,0 +1,60 @@
# Rate AI Result
This is an example of a Fabric Stitch, which is a chained Fabric command that pipes Fabric results into each other to achieve a result. So it's multiple Patterns…*stitched* together.
## Problem
The problem we're trying to solve with this Stitch is not being able to tell how smart given AI models are. I want to be able to rate their output vs. the output from a different model with the same instructions.
## Solution
What `rate_ai_result` does is run a result using AI 1, and then rate it with AI 2.
## Functionality
`rate_ai_result` accomplishes that like so:
1. Get the input that will be operated on by an AI.
2. Get the instruction/pattern/prompt that will be used by the AI.
3. Get the result of the instructions running against the AI.
4. Combine all three of those together as the input to another Fabric call.
4. Send that combined input to the most advanced model you have available to assess the quality of the AI result.
```
(echo "beginning of content input" ; f -u https://danielmiessler.com/p/framing-is-everything ; echo "end ofcontent input"; echo "beginning of AI instructions (prompt)"; cat ~/.config/fabric/patterns/extract_insights/system.md; echo "end of AI instructions (prompt)" ; echo "beginning of AI output" ; f -u https://danielmiessler.com/p/framing-is-everything | f -p extract_insights -m gpt-3.5-turbo ; echo "end of AI output. Now you should have all three." ) | f -rp rate_ai_result -m o1-preview-2024-09-12
```
In this case we're taking:
* A blog post as the input
* Getting the content of the extract_insights pattern
* Capturing the output of extract_insights on the blog post using `gpt-3.5-turbo`
* Sending all of that to `o1-preview` using the `rate_ai_result` prompt
NOTE: `rate_ai_result` is both a Pattern name and the name of this Stitch.
## Output
The `rate_ai_result` Pattern is designed to judge the output of another AI on a human sophistication scale that roughly maps to educational and world-state achievement, with the assumption that higher stages require higher cognitive ability as well. These are:
- Superhuman
- Best humans in the world
- Ph.D
- Masters
- Bachelors
- High School
- Partially Educated
- Uneducated
## How to run it
To run it, just execute the code in the `rate_ai_result` file in this repository. And adjust the components as desired to change the input, the AI you're testing, and the AI you're using to judge.
### Blog Post
Here's a full blog post describing in even more detail.
[Using the Smartest AI to Rate Other AI](https://danielmiessler.com/p/using-the-smartest-ai-to-rate-other-ai)
#### Credit
Created by Daniel Miessler on November 7th, 2024.

View File

@@ -1,3 +1,3 @@
package main
var version = "v1.4.97"
var version = "v1.4.100"