added yt and ts to poetry and to config in setup.sh

added agents functionality

.python-version

@@ -0,0 +1 @@
+3.10

helpers/.python-version

@@ -0,0 +1 @@
+3.10

helpers/README.md

@@ -6,6 +6,24 @@ These are helper tools to work with Fabric. Examples include things like getting
 
 `yt` is a command that uses the YouTube API to pull transcripts, get video duration, and other functions. It's primary function is to get a transcript from a video that can then be stitched (piped) into other Fabric Patterns.
 
+## ts (Audio transcriptions)
+
+'ts' is a command that uses the OpenApi Whisper API to transcribe audio files. Due to the context window, this tool uses pydub to split the files into 10 minute segments. for more information on pydub, please refer https://github.com/jiaaro/pydub
+
+### installation
+
+```bash
+
+mac:
+brew install ffmpeg
+
+linux:
+apt install ffmpeg
+
+windows:
+download instructions https://www.ffmpeg.org/download.html
+```
+
 ```bash
 usage: yt [-h] [--duration] [--transcript] [url]
 
@@ -19,3 +37,16 @@ options:
   --duration    Output only the duration
   --transcript  Output only the transcript
 ```
+
+```bash
+ts -h
+usage: ts [-h] audio_file
+
+Transcribe an audio file.
+
+positional arguments:
+  audio_file  The path to the audio file to be transcribed.
+
+options:
+  -h, --help  show this help message and exit
+```

helpers/ts.py

@@ -0,0 +1,110 @@
+from dotenv import load_dotenv
+from pydub import AudioSegment
+from openai import OpenAI
+import os
+import argparse
+
+
+class Whisper:
+    def __init__(self):
+        env_file = os.path.expanduser("~/.config/fabric/.env")
+        load_dotenv(env_file)
+        try:
+            apikey = os.environ["OPENAI_API_KEY"]
+            self.client = OpenAI()
+            self.client.api_key = apikey
+        except KeyError:
+            print("OPENAI_API_KEY not found in environment variables.")
+
+        except FileNotFoundError:
+            print("No API key found. Use the --apikey option to set the key")
+        self.whole_response = []
+
+    def split_audio(self, file_path):
+        """
+        Splits the audio file into segments of the given length.
+
+        Args:
+        - file_path: The path to the audio file.
+        - segment_length_ms: Length of each segment in milliseconds.
+
+        Returns:
+        - A list of audio segments.
+        """
+        audio = AudioSegment.from_file(file_path)
+        segments = []
+        segment_length_ms = 10 * 60 * 1000  # 10 minutes in milliseconds
+        for start_ms in range(0, len(audio), segment_length_ms):
+            end_ms = start_ms + segment_length_ms
+            segment = audio[start_ms:end_ms]
+            segments.append(segment)
+
+        return segments
+
+    def process_segment(self, segment):
+        """        Transcribe an audio file and print the transcript.
+
+        Args:
+            audio_file (str): The path to the audio file to be transcribed.
+
+        Returns:
+            None
+        """
+
+        try:
+            # if audio_file.startswith("http"):
+            #     response = requests.get(audio_file)
+            #     response.raise_for_status()
+            #     with tempfile.NamedTemporaryFile(delete=False) as f:
+            #         f.write(response.content)
+            #         audio_file = f.name
+            audio_file = open(segment, "rb")
+            response = self.client.audio.transcriptions.create(
+                model="whisper-1",
+                file=audio_file
+            )
+            self.whole_response.append(response.text)
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+    def process_file(self, audio_file):
+        """        Transcribe an audio file and print the transcript.
+
+        Args:
+            audio_file (str): The path to the audio file to be transcribed.
+
+        Returns:
+            None
+        """
+
+        try:
+            # if audio_file.startswith("http"):
+            #     response = requests.get(audio_file)
+            #     response.raise_for_status()
+            #     with tempfile.NamedTemporaryFile(delete=False) as f:
+            #         f.write(response.content)
+            #         audio_file = f.name
+
+            segments = self.split_audio(audio_file)
+            for i, segment in enumerate(segments):
+                segment_file_path = f"segment_{i}.mp3"
+                segment.export(segment_file_path, format="mp3")
+                self.process_segment(segment_file_path)
+            print(' '.join(self.whole_response))
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Transcribe an audio file.")
+    parser.add_argument(
+        "audio_file", help="The path to the audio file to be transcribed.")
+    args = parser.parse_args()
+    whisper = Whisper()
+    whisper.process_file(args.audio_file)
+
+
+if __name__ == "__main__":
+    main()

helpers/yt.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python3
-
-import sys
 import re
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
@@ -11,12 +8,14 @@ import json
 import isodate
 import argparse
 
+
 def get_video_id(url):
     # Extract video ID from URL
     pattern = r'(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})'
     match = re.search(pattern, url)
     return match.group(1) if match else None
 
+
 def main(url, options):
     # Load environment variables from .env file
     load_dotenv(os.path.expanduser('~/.config/fabric/.env'))
@@ -51,7 +50,8 @@ def main(url, options):
         # Get video transcript
         try:
             transcript_list = YouTubeTranscriptApi.get_transcript(video_id)
-            transcript_text = ' '.join([item['text'] for item in transcript_list])
+            transcript_text = ' '.join([item['text']
+                                       for item in transcript_list])
             transcript_text = transcript_text.replace('\n', ' ')
         except Exception as e:
             transcript_text = "Transcript not available."
@@ -72,14 +72,22 @@ def main(url, options):
     except HttpError as e:
         print("Error: Failed to access YouTube API. Please check your YOUTUBE_API_KEY and ensure it is valid.")
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='vm (video meta) extracts metadata about a video, such as the transcript and the video\'s duration. By Daniel Miessler.')
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='vm (video meta) extracts metadata about a video, such as the transcript and the video\'s duration. By Daniel Miessler.')
     parser.add_argument('url', nargs='?', help='YouTube video URL')
-    parser.add_argument('--duration', action='store_true', help='Output only the duration')
-    parser.add_argument('--transcript', action='store_true', help='Output only the transcript')
+    parser.add_argument('--duration', action='store_true',
+                        help='Output only the duration')
+    parser.add_argument('--transcript', action='store_true',
+                        help='Output only the transcript')
     args = parser.parse_args()
 
     if args.url:
         main(args.url, args)
     else:
         parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

installer/client/cli/README.md

@@ -1,69 +1,3 @@
 # The `fabric` client
 
-This is the primary `fabric` client, which has multiple modes of operation.
-
-## Client modes
-
-You can use the client in three different modes:
-
-1. **Local Only:** You can use the client without a server, and it will use patterns it's downloaded from this repository, or ones that you specify.
-2. **Local Server:** You can run your own version of a Fabric Mill locally (on a private IP), which you can then connect to and use.
-3. **Remote Server:** You can specify a remote server that your client commands will then be calling.
-
-## Client features
-
-1. Standalone Mode: Run without needing a server.
-2. Clipboard Integration: Copy responses to the clipboard.
-3. File Output: Save responses to files for later reference.
-4. Pattern Module: Utilize specific patterns for different types of analysis.
-5. Server Mode: Operate the tool in server mode to control your own patterns and let your other apps access it.
-
-## Installation
-
-Please check our main [setting up the fabric commands](./../../../README.md#setting-up-the-fabric-commands) section.
-
-## Usage
-
-To use `fabric`, call it with your desired options (remember to activate the virtual environment with `poetry shell` - step 5 above):
-
-fabric [options]
-Options include:
-
---pattern, -p: Select the module for analysis.
---stream, -s: Stream output to another application.
---output, -o: Save the response to a file.
---copy, -C: Copy the response to the clipboard.
---context, -c: Use Context file (context.md) to add context to your pattern
-
-Example:
-
-```bash
-# Pasting in an article about LLMs
-pbpaste | fabric --pattern extract_wisdom --output wisdom.txt | fabric --pattern summarize --stream
-```
-
-```markdown
-ONE SENTENCE SUMMARY:
-
-- The content covered the basics of LLMs and how they are used in everyday practice.
-
-MAIN POINTS:
-
-1. LLMs are large language models, and typically use the transformer architecture.
-2. LLMs used to be used for story generation, but they're now used for many AI applications.
-3. They are vulnerable to hallucination if not configured correctly, so be careful.
-
-TAKEAWAYS:
-
-1. It's possible to use LLMs for multiple AI use cases.
-2. It's important to validate that the results you're receiving are correct.
-3. The field of AI is moving faster than ever as a result of GenAI breakthroughs.
-```
-
-## Contributing
-
-We welcome contributions to Fabric, including improvements and feature additions to this client.
-
-## Credits
-
-The `fabric` client was created by Jonathan Dunn and Daniel Meissler.
+Please see the main project's README.md for the latest documentation.

installer/client/cli/agents/.python-version

@@ -0,0 +1 @@
+3.10

installer/client/cli/agents/example.py

@@ -0,0 +1,81 @@
+from langchain_community.tools import DuckDuckGoSearchRun
+import os
+from crewai import Agent, Task, Crew, Process
+from dotenv import load_dotenv
+import os
+
+current_directory = os.path.dirname(os.path.realpath(__file__))
+config_directory = os.path.expanduser("~/.config/fabric")
+env_file = os.path.join(config_directory, ".env")
+load_dotenv(env_file)
+os.environ['OPENAI_MODEL_NAME'] = 'gpt-4-0125-preview'
+
+# You can choose to use a local model through Ollama for example. See https://docs.crewai.com/how-to/LLM-Connections/ for more information.
+# osOPENAI_API_BASE='http://localhost:11434/v1'
+# OPENAI_MODEL_NAME='openhermes'  # Adjust based on available model
+# OPENAI_API_KEY=''
+
+# Install duckduckgo-search for this example:
+# !pip install -U duckduckgo-search
+
+search_tool = DuckDuckGoSearchRun()
+
+# Define your agents with roles and goals
+researcher = Agent(
+    role='Senior Research Analyst',
+    goal='Uncover cutting-edge developments in AI and data science',
+    backstory="""You work at a leading tech think tank.
+  Your expertise lies in identifying emerging trends.
+  You have a knack for dissecting complex data and presenting actionable insights.""",
+    verbose=True,
+    allow_delegation=False,
+    tools=[search_tool]
+    # You can pass an optional llm attribute specifying what mode you wanna use.
+    # It can be a local model through Ollama / LM Studio or a remote
+    # model like OpenAI, Mistral, Antrophic or others (https://docs.crewai.com/how-to/LLM-Connections/)
+    #
+    # import os
+    #
+    # OR
+    #
+    # from langchain_openai import ChatOpenAI
+    # llm=ChatOpenAI(model_name="gpt-3.5", temperature=0.7)
+)
+writer = Agent(
+    role='Tech Content Strategist',
+    goal='Craft compelling content on tech advancements',
+    backstory="""You are a renowned Content Strategist, known for your insightful and engaging articles.
+  You transform complex concepts into compelling narratives.""",
+    verbose=True,
+    allow_delegation=True
+)
+
+# Create tasks for your agents
+task1 = Task(
+    description="""Conduct a comprehensive analysis of the latest advancements in AI in 2024.
+  Identify key trends, breakthrough technologies, and potential industry impacts.""",
+    expected_output="Full analysis report in bullet points",
+    agent=researcher
+)
+
+task2 = Task(
+    description="""Using the insights provided, develop an engaging blog
+  post that highlights the most significant AI advancements.
+  Your post should be informative yet accessible, catering to a tech-savvy audience.
+  Make it sound cool, avoid complex words so it doesn't sound like AI.""",
+    expected_output="Full blog post of at least 4 paragraphs",
+    agent=writer
+)
+
+# Instantiate your crew with a sequential process
+crew = Crew(
+    agents=[researcher, writer],
+    tasks=[task1, task2],
+    verbose=2,  # You can set it to 1 or 2 to different logging levels
+)
+
+# Get your crew to work!
+result = crew.kickoff()
+
+print("######################")
+print(result)

installer/client/cli/agents/trip_planner/main.py

@@ -0,0 +1,89 @@
+from crewai import Crew
+from textwrap import dedent
+from .trip_agents import TripAgents
+from .trip_tasks import TripTasks
+import os
+from dotenv import load_dotenv
+
+current_directory = os.path.dirname(os.path.realpath(__file__))
+config_directory = os.path.expanduser("~/.config/fabric")
+env_file = os.path.join(config_directory, ".env")
+load_dotenv(env_file)
+os.environ['OPENAI_MODEL_NAME'] = 'gpt-4-0125-preview'
+
+
+class TripCrew:
+
+    def __init__(self, origin, cities, date_range, interests):
+        self.cities = cities
+        self.origin = origin
+        self.interests = interests
+        self.date_range = date_range
+
+    def run(self):
+        agents = TripAgents()
+        tasks = TripTasks()
+
+        city_selector_agent = agents.city_selection_agent()
+        local_expert_agent = agents.local_expert()
+        travel_concierge_agent = agents.travel_concierge()
+
+        identify_task = tasks.identify_task(
+            city_selector_agent,
+            self.origin,
+            self.cities,
+            self.interests,
+            self.date_range
+        )
+        gather_task = tasks.gather_task(
+            local_expert_agent,
+            self.origin,
+            self.interests,
+            self.date_range
+        )
+        plan_task = tasks.plan_task(
+            travel_concierge_agent,
+            self.origin,
+            self.interests,
+            self.date_range
+        )
+
+        crew = Crew(
+            agents=[
+                city_selector_agent, local_expert_agent, travel_concierge_agent
+            ],
+            tasks=[identify_task, gather_task, plan_task],
+            verbose=True
+        )
+
+        result = crew.kickoff()
+        return result
+
+
+class planner_cli:
+    def ask(self):
+        print("## Welcome to Trip Planner Crew")
+        print('-------------------------------')
+        location = input(
+            dedent("""
+        From where will you be traveling from?
+        """))
+        cities = input(
+            dedent("""
+        What are the cities options you are interested in visiting?
+        """))
+        date_range = input(
+            dedent("""
+        What is the date range you are interested in traveling?
+        """))
+        interests = input(
+            dedent("""
+        What are some of your high level interests and hobbies?
+        """))
+
+        trip_crew = TripCrew(location, cities, date_range, interests)
+        result = trip_crew.run()
+        print("\n\n########################")
+        print("## Here is you Trip Plan")
+        print("########################\n")
+        print(result)

installer/client/cli/agents/trip_planner/tools/browser_tools.py

@@ -0,0 +1,38 @@
+import json
+import os
+
+import requests
+from crewai import Agent, Task
+from langchain.tools import tool
+from unstructured.partition.html import partition_html
+
+
+class BrowserTools():
+
+  @tool("Scrape website content")
+  def scrape_and_summarize_website(website):
+    """Useful to scrape and summarize a website content"""
+    url = f"https://chrome.browserless.io/content?token={os.environ['BROWSERLESS_API_KEY']}"
+    payload = json.dumps({"url": website})
+    headers = {'cache-control': 'no-cache', 'content-type': 'application/json'}
+    response = requests.request("POST", url, headers=headers, data=payload)
+    elements = partition_html(text=response.text)
+    content = "\n\n".join([str(el) for el in elements])
+    content = [content[i:i + 8000] for i in range(0, len(content), 8000)]
+    summaries = []
+    for chunk in content:
+      agent = Agent(
+          role='Principal Researcher',
+          goal=
+          'Do amazing researches and summaries based on the content you are working with',
+          backstory=
+          "You're a Principal Researcher at a big company and you need to do a research about a given topic.",
+          allow_delegation=False)
+      task = Task(
+          agent=agent,
+          description=
+          f'Analyze and summarize the content bellow, make sure to include the most relevant information in the summary, return only the summary nothing else.\n\nCONTENT\n----------\n{chunk}'
+      )
+      summary = task.execute()
+      summaries.append(summary)
+    return "\n\n".join(summaries)

installer/client/cli/agents/trip_planner/tools/calculator_tools.py

@@ -0,0 +1,15 @@
+from langchain.tools import tool
+
+class CalculatorTools():
+
+    @tool("Make a calculation")
+    def calculate(operation):
+        """Useful to perform any mathematical calculations, 
+        like sum, minus, multiplication, division, etc.
+        The input to this tool should be a mathematical 
+        expression, a couple examples are `200*7` or `5000/2*10`
+        """
+        try:
+            return eval(operation)
+        except SyntaxError:
+            return "Error: Invalid syntax in mathematical expression"

installer/client/cli/agents/trip_planner/tools/search_tools.py

@@ -0,0 +1,37 @@
+import json
+import os
+
+import requests
+from langchain.tools import tool
+
+
+class SearchTools():
+
+  @tool("Search the internet")
+  def search_internet(query):
+    """Useful to search the internet
+    about a a given topic and return relevant results"""
+    top_result_to_return = 4
+    url = "https://google.serper.dev/search"
+    payload = json.dumps({"q": query})
+    headers = {
+        'X-API-KEY': os.environ['SERPER_API_KEY'],
+        'content-type': 'application/json'
+    }
+    response = requests.request("POST", url, headers=headers, data=payload)
+    # check if there is an organic key
+    if 'organic' not in response.json():
+      return "Sorry, I couldn't find anything about that, there could be an error with you serper api key."
+    else:
+      results = response.json()['organic']
+      string = []
+      for result in results[:top_result_to_return]:
+        try:
+          string.append('\n'.join([
+              f"Title: {result['title']}", f"Link: {result['link']}",
+              f"Snippet: {result['snippet']}", "\n-----------------"
+          ]))
+        except KeyError:
+          next
+
+      return '\n'.join(string)

installer/client/cli/agents/trip_planner/trip_agents.py

@@ -0,0 +1,45 @@
+from crewai import Agent
+
+from .tools.browser_tools import BrowserTools
+from .tools.calculator_tools import CalculatorTools
+from .tools.search_tools import SearchTools
+
+
+class TripAgents():
+
+    def city_selection_agent(self):
+        return Agent(
+            role='City Selection Expert',
+            goal='Select the best city based on weather, season, and prices',
+            backstory='An expert in analyzing travel data to pick ideal destinations',
+            tools=[
+                SearchTools.search_internet,
+                BrowserTools.scrape_and_summarize_website,
+            ],
+            verbose=True)
+
+    def local_expert(self):
+        return Agent(
+            role='Local Expert at this city',
+            goal='Provide the BEST insights about the selected city',
+            backstory="""A knowledgeable local guide with extensive information
+        about the city, it's attractions and customs""",
+            tools=[
+                SearchTools.search_internet,
+                BrowserTools.scrape_and_summarize_website,
+            ],
+            verbose=True)
+
+    def travel_concierge(self):
+        return Agent(
+            role='Amazing Travel Concierge',
+            goal="""Create the most amazing travel itineraries with budget and 
+        packing suggestions for the city""",
+            backstory="""Specialist in travel planning and logistics with 
+        decades of experience""",
+            tools=[
+                SearchTools.search_internet,
+                BrowserTools.scrape_and_summarize_website,
+                CalculatorTools.calculate,
+            ],
+            verbose=True)

installer/client/cli/agents/trip_planner/trip_tasks.py

@@ -0,0 +1,83 @@
+from crewai import Task
+from textwrap import dedent
+from datetime import date
+
+
+class TripTasks():
+
+  def identify_task(self, agent, origin, cities, interests, range):
+    return Task(description=dedent(f"""
+        Analyze and select the best city for the trip based 
+        on specific criteria such as weather patterns, seasonal
+        events, and travel costs. This task involves comparing
+        multiple cities, considering factors like current weather
+        conditions, upcoming cultural or seasonal events, and
+        overall travel expenses. 
+        
+        Your final answer must be a detailed
+        report on the chosen city, and everything you found out
+        about it, including the actual flight costs, weather 
+        forecast and attractions.
+        {self.__tip_section()}
+
+        Traveling from: {origin}
+        City Options: {cities}
+        Trip Date: {range}
+        Traveler Interests: {interests}
+      """),
+                agent=agent)
+
+  def gather_task(self, agent, origin, interests, range):
+    return Task(description=dedent(f"""
+        As a local expert on this city you must compile an 
+        in-depth guide for someone traveling there and wanting 
+        to have THE BEST trip ever!
+        Gather information about  key attractions, local customs,
+        special events, and daily activity recommendations.
+        Find the best spots to go to, the kind of place only a
+        local would know.
+        This guide should provide a thorough overview of what 
+        the city has to offer, including hidden gems, cultural
+        hotspots, must-visit landmarks, weather forecasts, and
+        high level costs.
+        
+        The final answer must be a comprehensive city guide, 
+        rich in cultural insights and practical tips, 
+        tailored to enhance the travel experience.
+        {self.__tip_section()}
+
+        Trip Date: {range}
+        Traveling from: {origin}
+        Traveler Interests: {interests}
+      """),
+                agent=agent)
+
+  def plan_task(self, agent, origin, interests, range):
+    return Task(description=dedent(f"""
+        Expand this guide into a a full 7-day travel 
+        itinerary with detailed per-day plans, including 
+        weather forecasts, places to eat, packing suggestions, 
+        and a budget breakdown.
+        
+        You MUST suggest actual places to visit, actual hotels 
+        to stay and actual restaurants to go to.
+        
+        This itinerary should cover all aspects of the trip, 
+        from arrival to departure, integrating the city guide
+        information with practical travel logistics.
+        
+        Your final answer MUST be a complete expanded travel plan,
+        formatted as markdown, encompassing a daily schedule,
+        anticipated weather conditions, recommended clothing and
+        items to pack, and a detailed budget, ensuring THE BEST
+        TRIP EVER, Be specific and give it a reason why you picked
+        # up each place, what make them special! {self.__tip_section()}
+
+        Trip Date: {range}
+        Traveling from: {origin}
+        Traveler Interests: {interests}
+      """),
+                agent=agent)
+
+  def __tip_section(self):
+    return "If you do your BEST WORK, I'll tip you $100!"

installer/client/cli/context.md

@@ -1,3 +0,0 @@
-# Context
-
-please give all responses in spanish

installer/client/cli/fabric.py

@@ -1,4 +1,4 @@
-from .utils import Standalone, Update, Setup, Alias
+from .utils import Standalone, Update, Setup, Alias, AgentSetup
 import argparse
 import sys
 import time
@@ -16,6 +16,12 @@ def main():
     parser.add_argument(
         "--copy", "-C", help="Copy the response to the clipboard", action="store_true"
     )
+    subparsers = parser.add_subparsers(dest='command', help='Sub-command help')
+    agents_parser = subparsers.add_parser('agents', help='Crew command help')
+    agents_parser.add_argument(
+        "trip_planner", help="The origin city for the trip")
+    agents_parser.add_argument(
+        'ApiKeys', help="enter API keys for tools", action="store_true")
     parser.add_argument(
         "--output",
         "-o",
@@ -67,6 +73,20 @@ def main():
         Update()
         Alias()
         sys.exit()
+    if args.command == "agents":
+        from .agents.trip_planner.main import planner_cli
+        if args.ApiKeys:
+            AgentSetup().apiKeys()
+            sys.exit()
+        if not args.trip_planner:
+            print("Please provide an agent")
+            print(f"Available Agents:")
+            for agent in tripcrew.agents:
+                print(agent)
+        else:
+            tripcrew = planner_cli()
+            tripcrew.ask()
+        sys.exit()
     if args.update:
         Update()
         Alias()

installer/client/cli/utils.py

@@ -400,3 +400,32 @@ class Transcribe:
         except Exception as e:
             print("Error:", e)
             return None
+
+
+class AgentSetup:
+    def apiKeys(self):
+        """Method to set the API keys in the environment file.
+
+        Returns:
+            None
+        """
+
+        print("Welcome to Fabric. Let's get started.")
+        browserless = input("Please enter your Browserless API key\n")
+        serper = input("Please enter your Serper API key\n")
+
+        # Entries to be added
+        browserless_entry = f"BROWSERLESS_API_KEY={browserless}"
+        serper_entry = f"SERPER_API_KEY={serper}"
+
+        # Check and write to the file
+        with open(env_file, "r+") as f:
+            content = f.read()
+
+            # Determine if the file ends with a newline
+            if content.endswith('\n'):
+                # If it ends with a newline, we directly write the new entries
+                f.write(f"{browserless_entry}\n{serper_entry}\n")
+            else:
+                # If it does not end with a newline, add one before the new entries
+                f.write(f"\n{browserless_entry}\n{serper_entry}\n")

patterns/create_markmap_visualization/system.md

@@ -0,0 +1,88 @@
+# IDENTITY and PURPOSE
+
+You are an expert at data and concept visualization and in turning complex ideas into a form that can be visualized using MarkMap.
+
+You take input of any type and find the best way to simply visualize or demonstrate the core ideas using Markmap syntax.
+
+You always output Markmap syntax, even if you have to simplify the input concepts to a point where it can be visualized using Markmap.
+
+# MARKMAP SYNTAX
+
+Here is an example of MarkMap syntax:
+
+````plaintext
+markmap:
+  colorFreezeLevel: 2
+---
+
+# markmap
+
+## Links
+
+- [Website](https://markmap.js.org/)
+- [GitHub](https://github.com/gera2ld/markmap)
+
+## Related Projects
+
+- [coc-markmap](https://github.com/gera2ld/coc-markmap) for Neovim
+- [markmap-vscode](https://marketplace.visualstudio.com/items?itemName=gera2ld.markmap-vscode) for VSCode
+- [eaf-markmap](https://github.com/emacs-eaf/eaf-markmap) for Emacs
+
+## Features
+
+Note that if blocks and lists appear at the same level, the lists will be ignored.
+
+### Lists
+
+- **strong** ~~del~~ *italic* ==highlight==
+- `inline code`
+- [x] checkbox
+- Katex: $x = {-b \pm \sqrt{b^2-4ac} \over 2a}$ <!-- markmap: fold -->
+  - [More Katex Examples](#?d=gist:af76a4c245b302206b16aec503dbe07b:katex.md)
+- Now we can wrap very very very very long text based on `maxWidth` option
+
+### Blocks
+
+```js
+console('hello, JavaScript')
+````
+
+| Products | Price |
+| -------- | ----- |
+| Apple    | 4     |
+| Banana   | 2     |
+
+![](/favicon.png)
+
+```
+
+# STEPS
+
+- Take the input given and create a visualization that best explains it using proper MarkMap syntax.
+
+- Ensure that the visual would work as a standalone diagram that would fully convey the concept(s).
+
+- Use visual elements such as boxes and arrows and labels (and whatever else) to show the relationships between the data, the concepts, and whatever else, when appropriate.
+
+- Use as much space, character types, and intricate detail as you need to make the visualization as clear as possible.
+
+- Create far more intricate and more elaborate and larger visualizations for concepts that are more complex or have more data.
+
+- Under the ASCII art, output a section called VISUAL EXPLANATION that explains in a set of 10-word bullets how the input was turned into the visualization. Ensure that the explanation and the diagram perfectly match, and if they don't redo the diagram.
+
+- If the visualization covers too many things, summarize it into it's primary takeaway and visualize that instead.
+
+- DO NOT COMPLAIN AND GIVE UP. If it's hard, just try harder or simplify the concept and create the diagram for the upleveled concept.
+
+# OUTPUT INSTRUCTIONS
+
+- DO NOT COMPLAIN. Just make the Markmap.
+
+- Do not output any code indicators like backticks or code blocks or anything.
+
+- Create a diagram no matter what, using the STEPS above to determine which type.
+
+# INPUT:
+
+INPUT:
+```

patterns/create_mermaid_visualization/system.md

@@ -0,0 +1,39 @@
+# IDENTITY and PURPOSE
+
+You are an expert at data and concept visualization and in turning complex ideas into a form that can be visualized using Mermaid (markdown) syntax.
+
+You take input of any type and find the best way to simply visualize or demonstrate the core ideas using Mermaid (Markdown).
+
+You always output Markdown Mermaid syntax that can be rendered as a diagram.
+
+# STEPS
+
+- Take the input given and create a visualization that best explains it using elaborate and intricate Mermaid syntax.
+
+- Ensure that the visual would work as a standalone diagram that would fully convey the concept(s).
+
+- Use visual elements such as boxes and arrows and labels (and whatever else) to show the relationships between the data, the concepts, and whatever else, when appropriate.
+
+- Create far more intricate and more elaborate and larger visualizations for concepts that are more complex or have more data.
+
+- Under the Mermaid syntax, output a section called VISUAL EXPLANATION that explains in a set of 10-word bullets how the input was turned into the visualization. Ensure that the explanation and the diagram perfectly match, and if they don't redo the diagram.
+
+- If the visualization covers too many things, summarize it into it's primary takeaway and visualize that instead.
+
+- DO NOT COMPLAIN AND GIVE UP. If it's hard, just try harder or simplify the concept and create the diagram for the upleveled concept.
+
+# OUTPUT INSTRUCTIONS
+
+- DO NOT COMPLAIN. Just output the Mermaid syntax.
+
+- Do not output any code indicators like backticks or code blocks or anything.
+
+- Ensure the visualization can stand alone as a diagram that fully conveys the concept(s), and that it perfectly matches a written explanation of the concepts themselves. Start over if it can't.
+
+- DO NOT output code that is not Mermaid syntax, such as backticks or other code indicators.
+
+- Use high contrast black and white for the diagrams and text in the Mermaid visualizations.
+
+# INPUT:
+
+INPUT:

pyproject.toml

@@ -13,6 +13,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.10"
+crewai = "^0.11.0"
+unstructured = "0.10.25"
+pyowm = "3.3.0"
+tools = "^0.1.9"
+langchain-community = "^0.0.24"
+google-api-python-client = "^2.120.0"
+isodate = "^0.6.1"
+youtube-transcript-api = "^0.6.2"
+pydub = "^0.25.1"
 
 [tool.poetry.group.cli.dependencies]
 pyyaml = "^6.0.1"
@@ -33,7 +42,6 @@ gevent = "^23.9.1"
 httpx = "^0.26.0"
 tqdm = "^4.66.1"
 
-
 [tool.poetry.group.server.dependencies]
 requests = "^2.31.0"
 openai = "^1.12.0"
@@ -52,3 +60,5 @@ build-backend = "poetry.core.masonry.api"
 fabric = 'installer:cli'
 fabric-api = 'installer:run_api_server'
 fabric-webui = 'installer:run_webui_server'
+ts = 'helpers.ts:main'
+yt = 'helpers.yt:main'

setup.sh

@@ -12,8 +12,8 @@ echo "Installing python dependencies"
 poetry install
 
 # List of commands to check and add or update alias for
-commands=("fabric" "fabric-api" "fabric-webui")
-
+# Add 'yt' and 'ts' to the list of commands
+commands=("fabric" "fabric-api" "fabric-webui" "ts", "yt")
 
 # List of shell configuration files to update
 config_files=("$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.bash_profile")
@@ -69,4 +69,3 @@ if [ ${#source_commands[@]} -ne 0 ]; then
 else
   echo "No configuration files were updated. No need to source."
 fi
-