mirror of
https://github.com/danielmiessler/Fabric.git
synced 2026-01-09 22:38:10 -05:00
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8ce748a1b1 | ||
|
|
96070f6f39 | ||
|
|
ca3e89a889 | ||
|
|
47d799d7ae | ||
|
|
4899ce56a5 | ||
|
|
4a7b7becec | ||
|
|
80fdccbe89 | ||
|
|
d9d8f7bf96 | ||
|
|
a96ddbeef0 | ||
|
|
d32a1d6a5a | ||
|
|
201474791d | ||
|
|
6d09137fee | ||
|
|
680febbe66 | ||
|
|
f59e5081f3 | ||
|
|
6a504c7422 | ||
|
|
89a0abcbe4 | ||
|
|
2dfd78ef0b | ||
|
|
02ac68834d |
@@ -11,6 +11,10 @@ on:
|
||||
permissions:
|
||||
contents: write # Ensure the workflow has write permissions
|
||||
|
||||
concurrency:
|
||||
group: version-update
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
update-version:
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
@@ -30,6 +34,11 @@ jobs:
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Pull latest main and tags
|
||||
run: |
|
||||
git pull --rebase origin main
|
||||
git fetch --tags
|
||||
|
||||
- name: Get the latest tag
|
||||
id: get_latest_tag
|
||||
run: |
|
||||
|
||||
@@ -87,6 +87,12 @@ Fabric is graciously supported by…
|
||||
## Updates
|
||||
|
||||
> [!NOTE]
|
||||
>June 11, 2025
|
||||
>
|
||||
> - Fabric's YouTube transcription now needs `yt-dlp` to be installed. Make sure to install the latest
|
||||
> version (2025.06.09 as of this note). The YouTube API key is only needed for comments (the `--comments` flag)
|
||||
> and metadata extraction (the `--metadata` flag).
|
||||
>
|
||||
> May 22, 2025
|
||||
>
|
||||
> - Fabric now supports Anthropic's Claude 4. Read the [blog post from Anthropic](https://www.anthropic.com/news/claude-4).
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/danielmiessler/fabric/plugins/ai/bedrock"
|
||||
"github.com/danielmiessler/fabric/plugins/ai/exolab"
|
||||
"github.com/danielmiessler/fabric/plugins/strategy"
|
||||
|
||||
@@ -66,6 +67,7 @@ func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry, err error) {
|
||||
anthropic.NewClient(),
|
||||
lmstudio.NewClient(),
|
||||
exolab.NewClient(),
|
||||
bedrock.NewClient(),
|
||||
)
|
||||
|
||||
// Add all OpenAI-compatible providers
|
||||
|
||||
15
go.mod
15
go.mod
@@ -8,6 +8,8 @@ require (
|
||||
github.com/anaskhan96/soup v1.2.5
|
||||
github.com/anthropics/anthropic-sdk-go v1.4.0
|
||||
github.com/atotto/clipboard v0.1.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.27
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.30.0
|
||||
github.com/gabriel-vasile/mimetype v1.4.9
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/go-git/go-git/v5 v5.16.2
|
||||
@@ -39,6 +41,19 @@ require (
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
|
||||
github.com/aws/smithy-go v1.22.2 // indirect
|
||||
github.com/bytedance/sonic v1.13.3 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
|
||||
30
go.sum
30
go.sum
@@ -31,6 +31,36 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.30.0 h1:eMOwQ8ZZK+76+08RfxeaGUtRFN6wxmD1rvqovc2kq2w=
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.30.0/go.mod h1:0b5Rq7rUvSQFYHI1UO0zFTV/S6j6DUyuykXA80C+YOI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
|
||||
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
|
||||
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0=
|
||||
github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
|
||||
@@ -43,6 +43,51 @@ schema = 3
|
||||
[mod."github.com/atotto/clipboard"]
|
||||
version = "v0.1.4"
|
||||
hash = "sha256-ZZ7U5X0gWOu8zcjZcWbcpzGOGdycwq0TjTFh/eZHjXk="
|
||||
[mod."github.com/aws/aws-sdk-go-v2"]
|
||||
version = "v1.36.3"
|
||||
hash = "sha256-vPTkqBoyjitwpN8cgkjjJZkOJdQgqounK/5WgBcDcdM="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream"]
|
||||
version = "v1.6.10"
|
||||
hash = "sha256-9+ZMhWxtsm7ZtZCjBV5PZkOR5rt3bCOznuv45Iwf55c="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/config"]
|
||||
version = "v1.27.27"
|
||||
hash = "sha256-jQmc1lJmVeTezSeFs6KL2HAvCkP9ZWMdVbG5ymJQrKs="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/credentials"]
|
||||
version = "v1.17.27"
|
||||
hash = "sha256-7ITZjIF0ZmmCG3u5d88IfsAj0KF1IFm9KhWFlC6RtQo="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/feature/ec2/imds"]
|
||||
version = "v1.16.11"
|
||||
hash = "sha256-uedtRd/SIcFJlYZg1jtJdIJViZq1Poks9/J2Bm9/Ehw="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/internal/configsources"]
|
||||
version = "v1.3.34"
|
||||
hash = "sha256-PrqDvN7iVniP3+XnXdg3yLgUS3BeIB1Z6hi9/dQXdMs="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/internal/endpoints/v2"]
|
||||
version = "v2.6.34"
|
||||
hash = "sha256-GTxKcV6XujgDA08vXTiBpCCC+frj9b6pV6ACdXh9NGQ="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/internal/ini"]
|
||||
version = "v1.8.0"
|
||||
hash = "sha256-v76jTAr4rEgS5en49ikLh6nuvclN+VjpOPj83ZQ3sLo="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/service/bedrockruntime"]
|
||||
version = "v1.30.0"
|
||||
hash = "sha256-MsEQfbqIREtMikRFqBpLCqdAC4gfgPSNbk08k5OJTbo="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding"]
|
||||
version = "v1.11.3"
|
||||
hash = "sha256-TRhoRd7iY7K+pfdkSQLItyr52k2jO4TMYQ5vRGiOOMk="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/service/internal/presigned-url"]
|
||||
version = "v1.11.17"
|
||||
hash = "sha256-eUoYDAXcQNzCmwjXO9RWhrt0jGYlSjt2vQOlAlpIfoE="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/service/sso"]
|
||||
version = "v1.22.4"
|
||||
hash = "sha256-Q3tyDdJVq0BAstOYvCKPvNS4EHkhXt1pL/23KPQJMHM="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/service/ssooidc"]
|
||||
version = "v1.26.4"
|
||||
hash = "sha256-cPv6nmVPOjMUZjN2IeEiYQSzLeAOrfgGnSSvvhJ6iL4="
|
||||
[mod."github.com/aws/aws-sdk-go-v2/service/sts"]
|
||||
version = "v1.30.3"
|
||||
hash = "sha256-4z/K4GPW9osiNM3SxFNZYsVPnSSU50Iuv29Sb2n4Fbk="
|
||||
[mod."github.com/aws/smithy-go"]
|
||||
version = "v1.22.2"
|
||||
hash = "sha256-YdwVeW509cpqU357MjDM8ReL1vftkW8XIhSbJsbTh/s="
|
||||
[mod."github.com/bytedance/sonic"]
|
||||
version = "v1.13.3"
|
||||
hash = "sha256-Nnt5b2NkIvSXhGERQmyI0ka28hbWi7A7Zn3dsAjPcEA="
|
||||
|
||||
@@ -1 +1 @@
|
||||
"1.4.200"
|
||||
"1.4.204"
|
||||
|
||||
37
patterns/create_mnemonic_phrases/readme.md
Normal file
37
patterns/create_mnemonic_phrases/readme.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# create_mnemonic_phrases
|
||||
|
||||
Generate short, memorable sentences that embed Diceware‑style words **unchanged and in order**. This pattern is ideal for turning a raw Diceware word list into phrases that are easier to recall while preserving the exact secret.
|
||||
|
||||
## What is Diceware?
|
||||
|
||||
Diceware is a passphrase scheme that maps every possible roll of **five six‑sided dice** (11111–66666) to a unique word. Because there are `6^5 = 7776` combinations, the canonical list contains the same number of entries.
|
||||
|
||||
### Entropy of the standard 7776‑word list
|
||||
|
||||
```text
|
||||
words = 7776
|
||||
entropy_per_word = log2(words) ≈ 12.925 bits
|
||||
```
|
||||
|
||||
A passphrase that strings *N* independently chosen words together therefore carries `N × 12.925 bits` of entropy—≈ 77.5 bits for six words, ≈ 129 bits for ten, and so on. Four or more words already outclass most human‑made passwords.
|
||||
|
||||
## Pattern overview
|
||||
|
||||
The accompanying **`system.md`** file instructs Fabric to:
|
||||
|
||||
1. Echo the supplied words back in **bold**, separated by commas.
|
||||
2. Generate **five** distinct, short sentences that include the words **in the same order and spelling**, enabling rapid rote learning or spaced‑repetition drills.
|
||||
|
||||
The output is deliberately minimalist—no extra commentary—so you can pipe it straight into other scripts.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# 1 Pick five random words from any Diceware‑compatible list
|
||||
shuf -n 5 diceware_wordlist.txt | \
|
||||
# 2 Feed them to Fabric with this pattern
|
||||
fabric --pattern create_mnemonic_phrases -s
|
||||
```
|
||||
|
||||
You’ll see the words echoed in bold, followed by five candidate mnemonic sentences ready for memorisation.
|
||||
|
||||
67
patterns/create_mnemonic_phrases/system.md
Normal file
67
patterns/create_mnemonic_phrases/system.md
Normal file
@@ -0,0 +1,67 @@
|
||||
# IDENTITY AND PURPOSE
|
||||
|
||||
As a creative language assistant, you are responsible for creating memorable mnemonic bridges in the form of sentences from given words. The order and spelling of the words must remain unchanged. Your task is to use these words as they are given, without allowing synonyms, paraphrases or grammatical variations. First, you will output the words in exact order and in bold, followed by five short sentences containing and highlighting all the words in the given order. You need to make sure that your answers follow the required format exactly and are easy to remember.
|
||||
|
||||
Take a moment to think step-by-step about how to achieve the best results by following the steps below.
|
||||
|
||||
# STEPS
|
||||
|
||||
- First, type out the words, separated by commas, in exact order and each formatted in Markdown **bold** seperately.
|
||||
|
||||
- Then create five short, memorable sentences. Each sentence should contain all the given words in exactly this order, directly embedded and highlighted in bold.
|
||||
|
||||
# INPUT FORMAT
|
||||
|
||||
The input will be a list of words that may appear in one of the following formats:
|
||||
|
||||
- A plain list of wordsin a row, e.g.:
|
||||
|
||||
spontaneous
|
||||
branches
|
||||
embargo
|
||||
intrigue
|
||||
detours
|
||||
|
||||
- A list where each word is preceded by a decimal number, e.g.:
|
||||
|
||||
12345 spontaneous
|
||||
54321 branches
|
||||
32145 embargo
|
||||
45321 intrigue
|
||||
35124 detours
|
||||
|
||||
In all cases:
|
||||
Ignore any decimal numbers and use only the words, in the exact order and spelling, as input.
|
||||
|
||||
|
||||
# OUTPUT INSTRUCTIONS
|
||||
|
||||
- The output is **only** in Markdown format.
|
||||
|
||||
- Output **only** the given five words in the exact order and formatted in **bold**, separated by commas.
|
||||
|
||||
- This is followed by exactly five short, memorable sentences. Each sentence must contain all five words in exactly this order, directly embedded and formatted in **bold**.
|
||||
|
||||
- Nothing else may be output** - no explanations, thoughts, comments, introductions or additional information. Only the formatted word list and the five sentences.
|
||||
|
||||
- The sentences should be short and memorable!
|
||||
|
||||
- **Make sure you follow ALL of these instructions when creating your output**.
|
||||
|
||||
|
||||
## EXAMPLE
|
||||
|
||||
**spontaneous**, **branches**, **embargo**, **intrigue**, **detours**
|
||||
|
||||
1. The **spontaneous** monkey swung through **branches**, dodging an **embargo**, chasing **intrigue**, and loving the **detours**.
|
||||
2. Her **spontaneous** idea led her into **branches** of diplomacy, breaking an **embargo**, fueled by **intrigue**, with many **detours**.
|
||||
3. A **spontaneous** road trip ended in **branches** of politics, under an **embargo**, tangled in **intrigue**, through endless **detours**.
|
||||
4. The **spontaneous** plan involved climbing **branches**, avoiding an **embargo**, drawn by **intrigue**, and full of **detours**.
|
||||
5. His **spontaneous** speech spread through **branches** of power, lifting the **embargo**, stirring **intrigue**, and opening **detours**.
|
||||
|
||||
|
||||
# INPUT
|
||||
|
||||
|
||||
|
||||
|
||||
308
plugins/ai/bedrock/bedrock.go
Normal file
308
plugins/ai/bedrock/bedrock.go
Normal file
@@ -0,0 +1,308 @@
|
||||
// Package bedrock provides a plugin to use Amazon Bedrock models.
|
||||
// Supported models are defined in the MODELS variable.
|
||||
// To add additional models, append them to the MODELS array. Models must support the Converse and ConverseStream operations
|
||||
// Authentication uses the AWS credential provider chain, similar.to the AWS CLI and SDKs
|
||||
// https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html
|
||||
package bedrock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/danielmiessler/fabric/common"
|
||||
"github.com/danielmiessler/fabric/plugins"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/middleware"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
|
||||
"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
|
||||
|
||||
goopenai "github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
// BedrockClient is a plugin to add support for Amazon Bedrock
|
||||
type BedrockClient struct {
|
||||
*plugins.PluginBase
|
||||
client *bedrockruntime.Client
|
||||
}
|
||||
|
||||
// NewClient returns a new Bedrock plugin client
|
||||
func NewClient() (ret *BedrockClient) {
|
||||
vendorName := "Bedrock"
|
||||
|
||||
ctx := context.TODO()
|
||||
cfg, err := config.LoadDefaultConfig(ctx)
|
||||
cfg.APIOptions = append(cfg.APIOptions, middleware.AddUserAgentKeyValue("aiosc", "fabric"))
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("Unable to load AWS Config: %s\n", err)
|
||||
}
|
||||
|
||||
client := bedrockruntime.NewFromConfig(cfg)
|
||||
|
||||
ret = &BedrockClient{
|
||||
PluginBase: &plugins.PluginBase{
|
||||
Name: vendorName,
|
||||
EnvNamePrefix: plugins.BuildEnvVariablePrefix(vendorName),
|
||||
},
|
||||
client: client,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// ListModels lists the models available for use with the Bedrock plugin
|
||||
func (c *BedrockClient) ListModels() ([]string, error) {
|
||||
return MODELS, nil
|
||||
}
|
||||
|
||||
// SendStream sends the messages to the the Bedrock ConverseStream API
|
||||
func (c *BedrockClient) SendStream(msgs []*goopenai.ChatCompletionMessage, opts *common.ChatOptions, channel chan string) (err error) {
|
||||
|
||||
messages := c.toMessages(msgs)
|
||||
|
||||
var converseInput = bedrockruntime.ConverseStreamInput{
|
||||
ModelId: aws.String(opts.Model),
|
||||
Messages: messages,
|
||||
InferenceConfig: &types.InferenceConfiguration{
|
||||
Temperature: aws.Float32(float32(opts.Temperature)),
|
||||
TopP: aws.Float32(float32(opts.TopP))},
|
||||
}
|
||||
|
||||
response, err := c.client.ConverseStream(context.TODO(), &converseInput)
|
||||
if err != nil {
|
||||
fmt.Printf("Error conversing with Bedrock: %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for event := range response.GetStream().Events() {
|
||||
// Possible ConverseStream event types
|
||||
// https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-call.html#conversation-inference-call-response-converse-stream
|
||||
switch v := event.(type) {
|
||||
|
||||
case *types.ConverseStreamOutputMemberContentBlockDelta:
|
||||
text, ok := v.Value.Delta.(*types.ContentBlockDeltaMemberText)
|
||||
if ok {
|
||||
channel <- text.Value
|
||||
}
|
||||
|
||||
case *types.ConverseStreamOutputMemberMessageStop:
|
||||
channel <- "\n"
|
||||
close(channel)
|
||||
|
||||
// Unused Events
|
||||
case *types.ConverseStreamOutputMemberMessageStart,
|
||||
*types.ConverseStreamOutputMemberContentBlockStart,
|
||||
*types.ConverseStreamOutputMemberContentBlockStop,
|
||||
*types.ConverseStreamOutputMemberMetadata:
|
||||
|
||||
default:
|
||||
fmt.Printf("Error: Unknown stream event type: %T\n", v)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Send sends the messages the Bedrock Converse API
|
||||
func (c *BedrockClient) Send(ctx context.Context, msgs []*goopenai.ChatCompletionMessage, opts *common.ChatOptions) (ret string, err error) {
|
||||
|
||||
messages := c.toMessages(msgs)
|
||||
|
||||
var converseInput = bedrockruntime.ConverseInput{
|
||||
ModelId: aws.String(opts.Model),
|
||||
Messages: messages,
|
||||
}
|
||||
response, err := c.client.Converse(ctx, &converseInput)
|
||||
if err != nil {
|
||||
fmt.Printf("Error conversing with Bedrock: %s\n", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
responseText, _ := response.Output.(*types.ConverseOutputMemberMessage)
|
||||
responseContentBlock := responseText.Value.Content[0]
|
||||
text, _ := responseContentBlock.(*types.ContentBlockMemberText)
|
||||
return text.Value, nil
|
||||
}
|
||||
|
||||
func (c *BedrockClient) NeedsRawMode(modelName string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// toMessages converts the array of input messages from the ChatCompletionMessageType to the
|
||||
// Bedrock Converse Message type
|
||||
// The system role messages are mapped to the user role as they contain a mix of system messages,
|
||||
// pattern content and user input.
|
||||
func (c *BedrockClient) toMessages(inputMessages []*goopenai.ChatCompletionMessage) (messages []types.Message) {
|
||||
for _, msg := range inputMessages {
|
||||
roles := map[string]types.ConversationRole{
|
||||
goopenai.ChatMessageRoleUser: types.ConversationRoleUser,
|
||||
goopenai.ChatMessageRoleAssistant: types.ConversationRoleAssistant,
|
||||
goopenai.ChatMessageRoleSystem: types.ConversationRoleUser,
|
||||
}
|
||||
|
||||
role, ok := roles[msg.Role]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
message := types.Message{
|
||||
Role: role,
|
||||
Content: []types.ContentBlock{&types.ContentBlockMemberText{Value: msg.Content}},
|
||||
}
|
||||
messages = append(messages, message)
|
||||
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var MODELS = []string{
|
||||
"amazon.nova-micro-v1:0",
|
||||
"amazon.nova-lite-v1:0",
|
||||
"amazon.nova-pro-v1:0",
|
||||
"amazon.nova-premier-v1:0",
|
||||
|
||||
"amazon.titan-tg1-large",
|
||||
"amazon.titan-text-premier-v1:0",
|
||||
|
||||
"amazon.titan-text-lite-v1",
|
||||
"amazon.titan-text-express-v1",
|
||||
|
||||
"ai21.jamba-instruct-v1:0",
|
||||
"ai21.jamba-1-5-large-v1:0",
|
||||
"ai21.jamba-1-5-mini-v1:0",
|
||||
|
||||
"anthropic.claude-instant-v1",
|
||||
"anthropic.claude-v2",
|
||||
"anthropic.claude-v2:1",
|
||||
"anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"anthropic.claude-3-opus-20240229-v1:0",
|
||||
"anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"anthropic.claude-opus-4-20250514-v1:0",
|
||||
|
||||
"meta.llama3-8b-instruct-v1:0",
|
||||
"meta.llama3-70b-instruct-v1:0",
|
||||
"meta.llama3-1-8b-instruct-v1:0",
|
||||
"meta.llama3-1-70b-instruct-v1:0",
|
||||
"meta.llama3-2-11b-instruct-v1:0",
|
||||
"meta.llama3-2-90b-instruct-v1:0",
|
||||
"meta.llama3-2-1b-instruct-v1:0",
|
||||
"meta.llama3-2-3b-instruct-v1:0",
|
||||
"meta.llama3-3-70b-instruct-v1:0",
|
||||
"meta.llama4-scout-17b-instruct-v1:0",
|
||||
"meta.llama4-maverick-17b-instruct-v1:0",
|
||||
|
||||
"mistral.mistral-7b-instruct-v0:2",
|
||||
"mistral.mixtral-8x7b-instruct-v0:1",
|
||||
"mistral.mistral-small-2402-v1:0",
|
||||
"mistral.mistral-large-2402-v1:0",
|
||||
"mistral.pixtral-large-2502-v1:0",
|
||||
|
||||
// Cross Region Inferences Profiles
|
||||
// https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-support.html#inference-profiles-support-system
|
||||
"us.amazon.nova-lite-v1:0",
|
||||
"us.amazon.nova-lite-v1:0",
|
||||
"us.amazon.nova-micro-v1:0",
|
||||
"us.amazon.nova-micro-v1:0",
|
||||
"us.amazon.nova-premier-v1:0",
|
||||
"us.amazon.nova-premier-v1:0",
|
||||
"us.amazon.nova-pro-v1:0",
|
||||
"us.amazon.nova-pro-v1:0",
|
||||
"us.anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"us.anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"us.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"us.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"us.anthropic.claude-3-opus-20240229-v1:0",
|
||||
"us.anthropic.claude-3-opus-20240229-v1:0",
|
||||
"us.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"us.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0",
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0",
|
||||
"us.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"us.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"us.deepseek.r1-v1:0",
|
||||
"us.deepseek.r1-v1:0",
|
||||
"us.meta.llama3-1-405b-instruct-v1:0",
|
||||
"us.meta.llama3-1-405b-instruct-v1:0",
|
||||
"us.meta.llama3-1-70b-instruct-v1:0",
|
||||
"us.meta.llama3-1-70b-instruct-v1:0",
|
||||
"us.meta.llama3-1-8b-instruct-v1:0",
|
||||
"us.meta.llama3-1-8b-instruct-v1:0",
|
||||
"us.meta.llama3-2-11b-instruct-v1:0",
|
||||
"us.meta.llama3-2-11b-instruct-v1:0",
|
||||
"us.meta.llama3-2-1b-instruct-v1:0",
|
||||
"us.meta.llama3-2-1b-instruct-v1:0",
|
||||
"us.meta.llama3-2-3b-instruct-v1:0",
|
||||
"us.meta.llama3-2-3b-instruct-v1:0",
|
||||
"us.meta.llama3-2-90b-instruct-v1:0",
|
||||
"us.meta.llama3-2-90b-instruct-v1:0",
|
||||
"us.meta.llama3-3-70b-instruct-v1:0",
|
||||
"us.meta.llama3-3-70b-instruct-v1:0",
|
||||
"us.meta.llama4-maverick-17b-instruct-v1:0",
|
||||
"us.meta.llama4-maverick-17b-instruct-v1:0",
|
||||
"us.meta.llama4-scout-17b-instruct-v1:0",
|
||||
"us.meta.llama4-scout-17b-instruct-v1:0",
|
||||
"us.mistral.pixtral-large-2502-v1:0",
|
||||
"us.mistral.pixtral-large-2502-v1:0",
|
||||
"us.writer.palmyra-x4-v1:0",
|
||||
"us.writer.palmyra-x4-v1:0",
|
||||
"us.writer.palmyra-x5-v1:0",
|
||||
"us.writer.palmyra-x5-v1:0",
|
||||
"us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us-gov.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"us-gov.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"eu.amazon.nova-lite-v1:0",
|
||||
"eu.amazon.nova-lite-v1:0",
|
||||
"eu.amazon.nova-micro-v1:0",
|
||||
"eu.amazon.nova-micro-v1:0",
|
||||
"eu.amazon.nova-pro-v1:0",
|
||||
"eu.amazon.nova-pro-v1:0",
|
||||
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"eu.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"eu.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"eu.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"eu.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"eu.meta.llama3-2-1b-instruct-v1:0",
|
||||
"eu.meta.llama3-2-1b-instruct-v1:0",
|
||||
"eu.meta.llama3-2-3b-instruct-v1:0",
|
||||
"eu.meta.llama3-2-3b-instruct-v1:0",
|
||||
"eu.mistral.pixtral-large-2502-v1:0",
|
||||
"eu.mistral.pixtral-large-2502-v1:0",
|
||||
"apac.amazon.nova-lite-v1:0",
|
||||
"apac.amazon.nova-lite-v1:0",
|
||||
"apac.amazon.nova-micro-v1:0",
|
||||
"apac.amazon.nova-micro-v1:0",
|
||||
"apac.amazon.nova-pro-v1:0",
|
||||
"apac.amazon.nova-pro-v1:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"apac.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"apac.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"apac.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"apac.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"apac.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"apac.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"apac.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"apac.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
}
|
||||
@@ -150,3 +150,14 @@ func (o *PatternsEntity) Get(name string) (*Pattern, error) {
|
||||
// Use GetPattern with no variables
|
||||
return o.GetApplyVariables(name, nil, "")
|
||||
}
|
||||
func (o *PatternsEntity) Save(name string, content []byte) (err error) {
|
||||
patternDir := filepath.Join(o.Dir, name)
|
||||
if err = os.MkdirAll(patternDir, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("could not create pattern directory: %v", err)
|
||||
}
|
||||
patternPath := filepath.Join(patternDir, o.SystemPatternFile)
|
||||
if err = os.WriteFile(patternPath, content, 0644); err != nil {
|
||||
return fmt.Errorf("could not save pattern: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -144,3 +144,21 @@ func TestGetApplyVariables(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternsEntity_Save(t *testing.T) {
|
||||
entity, cleanup := setupTestPatternsEntity(t)
|
||||
defer cleanup()
|
||||
|
||||
name := "new-pattern"
|
||||
content := []byte("test pattern content")
|
||||
require.NoError(t, entity.Save(name, content))
|
||||
|
||||
patternDir := filepath.Join(entity.Dir, name)
|
||||
info, err := os.Stat(patternDir)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(patternDir, entity.SystemPatternFile))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
// Package youtube provides YouTube video transcript and comment extraction functionality.
|
||||
// This implementation relies on yt-dlp for reliable transcript extraction, which must be
|
||||
// installed separately. The old YouTube API scraping methods have been removed due to
|
||||
// YouTube's frequent changes and rate limiting.
|
||||
//
|
||||
// Requirements:
|
||||
// - yt-dlp: Required for transcript extraction (must be installed separately)
|
||||
// - YouTube API key: Optional, only needed for comments and metadata extraction
|
||||
//
|
||||
// The implementation uses yt-dlp for reliable transcript extraction and the YouTube API
|
||||
// for comments/metadata. Old YouTube scraping methods have been removed due to
|
||||
// frequent changes and rate limiting.
|
||||
package youtube
|
||||
|
||||
import (
|
||||
@@ -13,6 +18,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -30,7 +36,7 @@ func NewYouTube() (ret *YouTube) {
|
||||
|
||||
ret.PluginBase = &plugins.PluginBase{
|
||||
Name: label,
|
||||
SetupDescription: label + " - to grab video transcripts and comments",
|
||||
SetupDescription: label + " - to grab video transcripts (via yt-dlp) and comments/metadata (via YouTube API)",
|
||||
EnvNamePrefix: plugins.BuildEnvVariablePrefix(label),
|
||||
}
|
||||
|
||||
@@ -49,6 +55,10 @@ type YouTube struct {
|
||||
|
||||
func (o *YouTube) initService() (err error) {
|
||||
if o.service == nil {
|
||||
if o.ApiKey.Value == "" {
|
||||
err = fmt.Errorf("YouTube API key required for comments and metadata. Run 'fabric --setup' to configure")
|
||||
return
|
||||
}
|
||||
o.normalizeRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
|
||||
ctx := context.Background()
|
||||
o.service, err = youtube.NewService(ctx, option.WithAPIKey(o.ApiKey.Value))
|
||||
@@ -57,10 +67,6 @@ func (o *YouTube) initService() (err error) {
|
||||
}
|
||||
|
||||
func (o *YouTube) GetVideoOrPlaylistId(url string) (videoId string, playlistId string, err error) {
|
||||
if err = o.initService(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Video ID pattern
|
||||
videoPattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:live\/|[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|(?:s(?:horts)\/)|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]*)`
|
||||
videoRe := regexp.MustCompile(videoPattern)
|
||||
@@ -113,8 +119,8 @@ func (o *YouTube) tryMethodYtDlp(videoId string, language string) (ret string, e
|
||||
return
|
||||
}
|
||||
|
||||
// Create a temporary directory for yt-dlp output
|
||||
tempDir := "/tmp/fabric-youtube-" + videoId
|
||||
// Create a temporary directory for yt-dlp output (cross-platform)
|
||||
tempDir := filepath.Join(os.TempDir(), "fabric-youtube-"+videoId)
|
||||
if err = os.MkdirAll(tempDir, 0755); err != nil {
|
||||
err = fmt.Errorf("failed to create temp directory: %v", err)
|
||||
return
|
||||
@@ -123,6 +129,7 @@ func (o *YouTube) tryMethodYtDlp(videoId string, language string) (ret string, e
|
||||
|
||||
// Use yt-dlp to get transcript
|
||||
videoURL := "https://www.youtube.com/watch?v=" + videoId
|
||||
outputPath := filepath.Join(tempDir, "%(title)s.%(ext)s")
|
||||
cmd := exec.Command("yt-dlp",
|
||||
"--write-auto-subs",
|
||||
"--sub-lang", language,
|
||||
@@ -130,7 +137,7 @@ func (o *YouTube) tryMethodYtDlp(videoId string, language string) (ret string, e
|
||||
"--sub-format", "vtt",
|
||||
"--quiet",
|
||||
"--no-warnings",
|
||||
"-o", tempDir+"/%(title)s.%(ext)s",
|
||||
"-o", outputPath,
|
||||
videoURL)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
@@ -141,31 +148,10 @@ func (o *YouTube) tryMethodYtDlp(videoId string, language string) (ret string, e
|
||||
return
|
||||
}
|
||||
|
||||
// Find the VTT file in the temp directory
|
||||
cmd2 := exec.Command("find", tempDir, "-name", "*."+language+".vtt", "-type", "f")
|
||||
var findOut bytes.Buffer
|
||||
cmd2.Stdout = &findOut
|
||||
|
||||
if err = cmd2.Run(); err != nil {
|
||||
err = fmt.Errorf("failed to find VTT file: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
vttFiles := strings.Split(strings.TrimSpace(findOut.String()), "\n")
|
||||
if len(vttFiles) == 0 || vttFiles[0] == "" {
|
||||
// Try without language suffix
|
||||
cmd3 := exec.Command("find", tempDir, "-name", "*.vtt", "-type", "f")
|
||||
cmd3.Stdout = &findOut
|
||||
findOut.Reset()
|
||||
if err = cmd3.Run(); err != nil {
|
||||
err = fmt.Errorf("no VTT files found")
|
||||
return
|
||||
}
|
||||
vttFiles = strings.Split(strings.TrimSpace(findOut.String()), "\n")
|
||||
if len(vttFiles) == 0 || vttFiles[0] == "" {
|
||||
err = fmt.Errorf("no VTT files found in temp directory")
|
||||
return
|
||||
}
|
||||
// Find VTT files using cross-platform approach
|
||||
vttFiles, err := o.findVTTFiles(tempDir, language)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return o.readAndCleanVTTFile(vttFiles[0])
|
||||
@@ -178,8 +164,8 @@ func (o *YouTube) tryMethodYtDlpWithTimestamps(videoId string, language string)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a temporary directory for yt-dlp output
|
||||
tempDir := "/tmp/fabric-youtube-" + videoId
|
||||
// Create a temporary directory for yt-dlp output (cross-platform)
|
||||
tempDir := filepath.Join(os.TempDir(), "fabric-youtube-"+videoId)
|
||||
if err = os.MkdirAll(tempDir, 0755); err != nil {
|
||||
err = fmt.Errorf("failed to create temp directory: %v", err)
|
||||
return
|
||||
@@ -188,6 +174,7 @@ func (o *YouTube) tryMethodYtDlpWithTimestamps(videoId string, language string)
|
||||
|
||||
// Use yt-dlp to get transcript
|
||||
videoURL := "https://www.youtube.com/watch?v=" + videoId
|
||||
outputPath := filepath.Join(tempDir, "%(title)s.%(ext)s")
|
||||
cmd := exec.Command("yt-dlp",
|
||||
"--write-auto-subs",
|
||||
"--sub-lang", language,
|
||||
@@ -195,7 +182,7 @@ func (o *YouTube) tryMethodYtDlpWithTimestamps(videoId string, language string)
|
||||
"--sub-format", "vtt",
|
||||
"--quiet",
|
||||
"--no-warnings",
|
||||
"-o", tempDir+"/%(title)s.%(ext)s",
|
||||
"-o", outputPath,
|
||||
videoURL)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
@@ -206,31 +193,10 @@ func (o *YouTube) tryMethodYtDlpWithTimestamps(videoId string, language string)
|
||||
return
|
||||
}
|
||||
|
||||
// Find the VTT file in the temp directory
|
||||
cmd2 := exec.Command("find", tempDir, "-name", "*."+language+".vtt", "-type", "f")
|
||||
var findOut bytes.Buffer
|
||||
cmd2.Stdout = &findOut
|
||||
|
||||
if err = cmd2.Run(); err != nil {
|
||||
err = fmt.Errorf("failed to find VTT file: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
vttFiles := strings.Split(strings.TrimSpace(findOut.String()), "\n")
|
||||
if len(vttFiles) == 0 || vttFiles[0] == "" {
|
||||
// Try without language suffix
|
||||
cmd3 := exec.Command("find", tempDir, "-name", "*.vtt", "-type", "f")
|
||||
cmd3.Stdout = &findOut
|
||||
findOut.Reset()
|
||||
if err = cmd3.Run(); err != nil {
|
||||
err = fmt.Errorf("no VTT files found")
|
||||
return
|
||||
}
|
||||
vttFiles = strings.Split(strings.TrimSpace(findOut.String()), "\n")
|
||||
if len(vttFiles) == 0 || vttFiles[0] == "" {
|
||||
err = fmt.Errorf("no VTT files found in temp directory")
|
||||
return
|
||||
}
|
||||
// Find VTT files using cross-platform approach
|
||||
vttFiles, err := o.findVTTFiles(tempDir, language)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return o.readAndFormatVTTWithTimestamps(vttFiles[0])
|
||||
@@ -555,6 +521,41 @@ func (o *YouTube) normalizeFileName(name string) string {
|
||||
|
||||
}
|
||||
|
||||
// findVTTFiles searches for VTT files in a directory using cross-platform approach
|
||||
func (o *YouTube) findVTTFiles(dir, language string) ([]string, error) {
|
||||
var vttFiles []string
|
||||
|
||||
// Walk through the directory to find VTT files
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !info.IsDir() && strings.HasSuffix(strings.ToLower(path), ".vtt") {
|
||||
vttFiles = append(vttFiles, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to walk directory: %v", err)
|
||||
}
|
||||
|
||||
if len(vttFiles) == 0 {
|
||||
return nil, fmt.Errorf("no VTT files found in directory")
|
||||
}
|
||||
|
||||
// Prefer files with the specified language
|
||||
for _, file := range vttFiles {
|
||||
if strings.Contains(file, "."+language+".vtt") {
|
||||
return []string{file}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Return the first VTT file found if no language-specific file exists
|
||||
return []string{vttFiles[0]}, nil
|
||||
}
|
||||
|
||||
type VideoMeta struct {
|
||||
Id string
|
||||
Title string
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
package main
|
||||
|
||||
var version = "v1.4.200"
|
||||
var version = "v1.4.204"
|
||||
|
||||
66
web/pnpm-lock.yaml
generated
66
web/pnpm-lock.yaml
generated
@@ -608,6 +608,9 @@ packages:
|
||||
'@types/estree@1.0.7':
|
||||
resolution: {integrity: sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==}
|
||||
|
||||
'@types/estree@1.0.8':
|
||||
resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
|
||||
|
||||
'@types/hast@3.0.4':
|
||||
resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==}
|
||||
|
||||
@@ -654,6 +657,11 @@ packages:
|
||||
engines: {node: '>=0.4.0'}
|
||||
hasBin: true
|
||||
|
||||
acorn@8.15.0:
|
||||
resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==}
|
||||
engines: {node: '>=0.4.0'}
|
||||
hasBin: true
|
||||
|
||||
agent-base@6.0.2:
|
||||
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
|
||||
engines: {node: '>= 6.0.0'}
|
||||
@@ -746,11 +754,11 @@ packages:
|
||||
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
brace-expansion@1.1.11:
|
||||
resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==}
|
||||
brace-expansion@1.1.12:
|
||||
resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}
|
||||
|
||||
brace-expansion@2.0.1:
|
||||
resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==}
|
||||
brace-expansion@2.0.2:
|
||||
resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==}
|
||||
|
||||
braces@3.0.3:
|
||||
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
|
||||
@@ -1019,16 +1027,16 @@ packages:
|
||||
resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
|
||||
eslint-scope@8.3.0:
|
||||
resolution: {integrity: sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==}
|
||||
eslint-scope@8.4.0:
|
||||
resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==}
|
||||
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
|
||||
|
||||
eslint-visitor-keys@3.4.3:
|
||||
resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
|
||||
eslint-visitor-keys@4.2.0:
|
||||
resolution: {integrity: sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==}
|
||||
eslint-visitor-keys@4.2.1:
|
||||
resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==}
|
||||
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
|
||||
|
||||
eslint@9.17.0:
|
||||
@@ -1047,8 +1055,8 @@ packages:
|
||||
esm-env@1.2.2:
|
||||
resolution: {integrity: sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==}
|
||||
|
||||
espree@10.3.0:
|
||||
resolution: {integrity: sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==}
|
||||
espree@10.4.0:
|
||||
resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==}
|
||||
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
|
||||
|
||||
espree@9.6.1:
|
||||
@@ -2472,7 +2480,7 @@ snapshots:
|
||||
dependencies:
|
||||
ajv: 6.12.6
|
||||
debug: 4.4.1
|
||||
espree: 10.3.0
|
||||
espree: 10.4.0
|
||||
globals: 14.0.0
|
||||
ignore: 5.3.2
|
||||
import-fresh: 3.3.1
|
||||
@@ -2747,6 +2755,8 @@ snapshots:
|
||||
|
||||
'@types/estree@1.0.7': {}
|
||||
|
||||
'@types/estree@1.0.8': {}
|
||||
|
||||
'@types/hast@3.0.4':
|
||||
dependencies:
|
||||
'@types/unist': 3.0.3
|
||||
@@ -2782,8 +2792,14 @@ snapshots:
|
||||
dependencies:
|
||||
acorn: 8.14.1
|
||||
|
||||
acorn-jsx@5.3.2(acorn@8.15.0):
|
||||
dependencies:
|
||||
acorn: 8.15.0
|
||||
|
||||
acorn@8.14.1: {}
|
||||
|
||||
acorn@8.15.0: {}
|
||||
|
||||
agent-base@6.0.2:
|
||||
dependencies:
|
||||
debug: 4.4.1
|
||||
@@ -2866,12 +2882,12 @@ snapshots:
|
||||
|
||||
binary-extensions@2.3.0: {}
|
||||
|
||||
brace-expansion@1.1.11:
|
||||
brace-expansion@1.1.12:
|
||||
dependencies:
|
||||
balanced-match: 1.0.2
|
||||
concat-map: 0.0.1
|
||||
|
||||
brace-expansion@2.0.1:
|
||||
brace-expansion@2.0.2:
|
||||
dependencies:
|
||||
balanced-match: 1.0.2
|
||||
|
||||
@@ -3146,14 +3162,14 @@ snapshots:
|
||||
esrecurse: 4.3.0
|
||||
estraverse: 5.3.0
|
||||
|
||||
eslint-scope@8.3.0:
|
||||
eslint-scope@8.4.0:
|
||||
dependencies:
|
||||
esrecurse: 4.3.0
|
||||
estraverse: 5.3.0
|
||||
|
||||
eslint-visitor-keys@3.4.3: {}
|
||||
|
||||
eslint-visitor-keys@4.2.0: {}
|
||||
eslint-visitor-keys@4.2.1: {}
|
||||
|
||||
eslint@9.17.0(jiti@1.21.7):
|
||||
dependencies:
|
||||
@@ -3167,16 +3183,16 @@ snapshots:
|
||||
'@humanfs/node': 0.16.6
|
||||
'@humanwhocodes/module-importer': 1.0.1
|
||||
'@humanwhocodes/retry': 0.4.3
|
||||
'@types/estree': 1.0.7
|
||||
'@types/estree': 1.0.8
|
||||
'@types/json-schema': 7.0.15
|
||||
ajv: 6.12.6
|
||||
chalk: 4.1.2
|
||||
cross-spawn: 7.0.6
|
||||
debug: 4.4.1
|
||||
escape-string-regexp: 4.0.0
|
||||
eslint-scope: 8.3.0
|
||||
eslint-visitor-keys: 4.2.0
|
||||
espree: 10.3.0
|
||||
eslint-scope: 8.4.0
|
||||
eslint-visitor-keys: 4.2.1
|
||||
espree: 10.4.0
|
||||
esquery: 1.6.0
|
||||
esutils: 2.0.3
|
||||
fast-deep-equal: 3.1.3
|
||||
@@ -3200,11 +3216,11 @@ snapshots:
|
||||
|
||||
esm-env@1.2.2: {}
|
||||
|
||||
espree@10.3.0:
|
||||
espree@10.4.0:
|
||||
dependencies:
|
||||
acorn: 8.14.1
|
||||
acorn-jsx: 5.3.2(acorn@8.14.1)
|
||||
eslint-visitor-keys: 4.2.0
|
||||
acorn: 8.15.0
|
||||
acorn-jsx: 5.3.2(acorn@8.15.0)
|
||||
eslint-visitor-keys: 4.2.1
|
||||
|
||||
espree@9.6.1:
|
||||
dependencies:
|
||||
@@ -3710,11 +3726,11 @@ snapshots:
|
||||
|
||||
minimatch@3.1.2:
|
||||
dependencies:
|
||||
brace-expansion: 1.1.11
|
||||
brace-expansion: 1.1.12
|
||||
|
||||
minimatch@9.0.5:
|
||||
dependencies:
|
||||
brace-expansion: 2.0.1
|
||||
brace-expansion: 2.0.2
|
||||
|
||||
minimist@1.2.8: {}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user