added support for local llms

chat_utils.py

@@ -47,10 +47,10 @@ def results_from_db(db:FAISS, question, num_results=10):
     return results
 
 
-def rerank_fuzzy_matching(question, results, num_results=4):
+def rerank_fuzzy_matching(question, results, num_results=5):
     filtered_question = filter_stopwords(question)
     if filtered_question == '':
-        return results[-4:]
+        return results[-5:]
     scores_and_results = []
     for result in results:
         score = fuzz.partial_ratio(question, result.page_content)
@@ -74,7 +74,12 @@ def qa_from_db(question, db, llm_name):
     llm = create_llm(llm_name)
     results = results_from_db(db, question)
     reranked_results = rerank_fuzzy_matching(question, results)
-    message = f'{chat_prompt} ---------- Context: {reranked_results} -------- User Question: {question} ---------- Response:'
+    if type(llm_name) != str:
+        reranked_results = reranked_results[:2]
+        message = f'Answer the user question based on the context. Question: {question} Context: {reranked_results} Answer:'
+    else:
+        message = f'{chat_prompt} ---------- Context: {reranked_results} -------- User Question: {question} ---------- Response:'
+    print(message)
     output = llm(message)
     return output
 

local_app.py

@@ -0,0 +1,95 @@
+import os
+
+import streamlit as st
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+
+from streamlit_chat import message as st_message
+
+import pandas as pd
+
+from local_chat_utils import load_db_from_file_and_create_if_not_exists_local
+from streamlit_app_utils import generate_answer
+
+from langchain.llms import GPT4All, LlamaCpp
+
+from dotenv import load_dotenv
+
+
+
+load_dotenv('test.env')
+
+model_type = os.getenv('MODEL_TYPE')
+model_path = os.getenv('MODEL_PATH')
+print(model_path)
+
+#Model is initialized here. Configure it with your parameters and the path to your model.
+
+loading = st.spinner('Initializing LLM')
+with st.spinner('Initializing LLM...'):
+    if 'llm' not in st.session_state:
+        with st.spinner('Loading LLM...'):
+            if model_type == 'LlamaCpp':
+                llm = LlamaCpp(model_path=model_path, n_ctx=1000)
+                st.session_state.llm = llm
+            elif model_type == 'GPT4All':
+                llm = GPT4All(model=model_path, backend='gptj', n_ctx=1000)
+                st.session_state.llm = llm
+            else:
+                st.warning('Invalid model type. GPT4ALL or LlamaCpp supported - make sure you specify in your env file.')
+
+
+def chat():
+    st.title('Chat')
+    if 'text_input' not in st.session_state:
+        st.session_state.text_input = ''
+    directory = 'documents'
+    files = os.listdir(directory)
+    files = [file for file in files if file.endswith('.txt') or file.endswith('.pdf')]
+    selected_file = st.selectbox('Select a file', files)
+    st.write('You selected: ' + selected_file)
+    selected_file_path = os.path.join(directory, selected_file)
+
+    if st.button('Load file (first time might take a second...) pressing this button will reset the chat history'):
+        db = load_db_from_file_and_create_if_not_exists_local(selected_file_path, 'hkunlp/instructor-base')
+        st.session_state.db = db
+        st.session_state.history = []
+
+    user_input = st.text_input('Enter your question', key='text_input')
+
+    if st.button('Ask') and 'db' in st.session_state:
+        answer = generate_answer(st.session_state.db, st.session_state.llm)
+
+
+    if 'history' not in st.session_state:
+        st.session_state.history = []
+    for i, chat in enumerate(st.session_state.history):
+        st_message(**chat, key=str(i))
+
+
+
+
+def documents():
+    st.title('Documents')
+    st.markdown('Documents are stored in the documents folder in the project directory.')
+    directory = 'documents'
+    files = os.listdir(directory)
+    files = [file for file in files if file.endswith('.txt') or file.endswith('.pdf')]
+    if files:
+        files_df = pd.DataFrame(files, columns=['File Name'], index=range(1, len(files) + 1))
+        st.dataframe(files_df, width=1000)
+    else:
+        st.write('No documents found in documents folder. Add some documents first!')
+
+
+
+PAGES = {
+    "Chat": chat,
+    "Documents": documents,
+}
+
+st.sidebar.title("Navigation")
+selection = st.sidebar.radio("Go to", list(PAGES.keys()))
+st.sidebar.markdown(' [Contact author](mailto:ethanujohnston@gmail.com)')
+st.sidebar.markdown(' [Github](https://github.com/e-johnstonn/docsummarizer)')
+page = PAGES[selection]
+page()

local_chat_utils.py

@@ -0,0 +1,53 @@
+import os
+
+from langchain import FAISS
+from langchain.embeddings import HuggingFaceInstructEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from summary_utils import doc_loader, remove_special_tokens
+
+import streamlit as st
+
+
+def create_and_save_local(file_path, model_name):
+    name = os.path.split(file_path)[1].split('.')[0]
+    embeddings = HuggingFaceInstructEmbeddings(model_name=model_name)
+    doc = doc_loader(file_path)
+    splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    split_docs = splitter.split_documents(doc)
+    processed_split_docs = remove_special_tokens(split_docs)
+    db = FAISS.from_documents(processed_split_docs, embeddings)
+    db.save_local(folder_path='local_embeddings', index_name=name)
+
+
+def load_local_embeddings(file_path, model_name):
+    name = os.path.split(file_path)[1].split('.')[0]
+    embeddings = HuggingFaceInstructEmbeddings(model_name=model_name)
+    db = FAISS.load_local(folder_path='local_embeddings', index_name=name, embeddings=embeddings)
+    return db
+
+
+
+def load_db_from_file_and_create_if_not_exists_local(file_path, model_name):
+    with st.spinner('Loading chat embeddings...'):
+        try:
+            db = load_local_embeddings(file_path, model_name)
+            print('success')
+        except RuntimeError:
+            print('not found')
+            create_and_save_local(file_path, model_name)
+            db = load_local_embeddings(file_path, model_name)
+    if db:
+        st.success('Loaded successfully! Start a chat below.')
+    else:
+        st.warning('Something went wrong... failed to load chat embeddings.')
+    return db
+
+
+
+
+
+
+
+
+

my_prompts.py

@@ -47,12 +47,9 @@ Format in HTML. Text should be divided into paragraphs. Paragraphs should be ind
 """
 
 chat_prompt = """
-You will be provided some context from a document. 
-Based on this context, answer the user question. 
-Do not make anything up. Only answer based on the given context. 
-You can quote from the context, but do not say 'based on the context'. 
-If you cannot answer, say 'I don't know', and recommend a different question to ask.
-Answer to the best of your ability. 
-
+You will be provided some context from a document.
+Based on this context, answer the user question.
+Only answer based on the given context.
+If you cannot answer, say 'I don't know' and recommend a different question.
 
 """

summaries/sparksofagi_summary.txt

@@ -1,7 +1,19 @@
-Key Takeaways:
-- The article discusses the capabilities and limitations of GPT-4, a language model developed by OpenAI.
-- GPT-4 exhibits traits of intelligence such as reasoning, planning, and learning from experience, but still has limitations in certain tasks.
-- The model can generate and manipulate images, music, and language, and can perform mathematical reasoning.
-- The article also discusses the importance of explainability and theory of mind in communication with humans.
+<h3>Key Takeaways:</h3>
+<ul>
+    <li>GPT-4 exhibits advanced capabilities in natural language processing, mathematical reasoning, image and music generation, and theory of mind.</li>
+    <li>While GPT-4 outperforms previous models like ChatGPT, it still has limitations and is not perfect at what it does.</li>
+    <li>GPT-4 can generate output-consistent and process-consistent explanations, but may still produce nonsensical or wrong outputs.</li>
+    <li>The AI can provide useful suggestions for data visualization and even implement them in Python.</li>
+</ul>
 
-The article explores the advancements in AI research, particularly in natural language processing achieved by large language models (LLMs), and focuses on GPT-4, a language model developed by OpenAI. The article reports on evidence that GPT-4 exhibits many traits of intelligence, including reasoning, planning, and the ability to learn from experience, and with these capabilities at or above human-level. The article provides examples of GPT-4's capabilities in language, vision, coding, and mathematics, and compares its performance to previous LLMs. However, the article also highlights the limitations of GPT-4, particularly in certain tasks such as fact-checking and managing large numbers or complicated expressions. The article also discusses the importance of explainability and theory of mind in communication with humans, and evaluates GPT-4's ability to generate output-consistent and process-consistent explanations. Additionally, the article provides examples of how GPT-4 can be used in practical scenarios, such as visualizing a dataset for a Hollywood producer and communicating with family members about the Covid-19 vaccine. Overall, the article suggests that while GPT-4 represents progress towards AGI, it still has limitations and does not come close to being able to do anything that a human can do.
+<p>&emsp;&emsp;The document discusses the capabilities and limitations of GPT-4, a large language model developed by OpenAI. GPT-4 exhibits advanced abilities in natural language processing, allowing it to interact with APIs for calendar and email functions, answer questions, and coordinate events. It also demonstrates improved mathematical reasoning compared to previous models like ChatGPT, although it still struggles with large numbers and complicated expressions.</p>
+
+<p>&emsp;&emsp;GPT-4 can generate and manipulate images and music, successfully combining objects with letters of the alphabet and generating 2D and 3D images according to detailed instructions. However, it does not seem to understand harmony in music generation. The model's ability to generate code from prompts could be combined with existing image synthesis models to produce higher-quality images.</p>
+
+<p>&emsp;&emsp;The document highlights GPT-4's progress towards artificial general intelligence (AGI), with its capabilities at or above human-level in various tasks. However, it is not perfect and does not come close to being able to do anything that a human can do. GPT-4 performs better than GPT-3 in categories related to people and places, but fact-checking may require inputs from an external corpus. The model also occasionally states that neither answer is correct, and its rationale for decisions may differ from human experts.</p>
+
+<p>&emsp;&emsp;GPT-4 demonstrates advanced theory of mind and is remarkably good at generating reasonable and coherent explanations. The quality of explanations can be evaluated by checking output consistency and process consistency. The model is good at generating output-consistent explanations, even when the output is nonsensical or wrong. Experiments have been conducted to test the process consistency of GPT-4's explanations.</p>
+
+<p>&emsp;&emsp;In a text-based game scenario, GPT-4 is able to navigate through different rooms and interact with objects. Additionally, the AI can provide useful suggestions for data visualization, such as using a network graph to represent IMDb dataset information, and even implement the visualization in Python using networkx, pandas, and plotly libraries.</p>
+
+<p>&emsp;&emsp;Overall, GPT-4 represents significant progress in AI research, particularly in natural language processing, mathematical reasoning, image and music generation, and theory of mind. However, it still has limitations and is not perfect at what it does, indicating that there is still room for improvement in the pursuit of AGI.</p>