Skip to content

Commit

Permalink
Moved to the new lollms vector database
Browse files Browse the repository at this point in the history
  • Loading branch information
@ParisNeo
ParisNeo committed Sep 1, 2024
1 parent 275c203 commit 54cbcd6
Show file tree
Hide file tree
Showing 13 changed files with 368 additions and 289 deletions.
10 changes: 8 additions & 2 deletions app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ def check_and_install_package(package: str, version: str):
ASCIIColors.red(f"Error checking/installing {package}: {str(e)}")

packages: List[Tuple[str, str]] = [
("lollmsvectordb", "0.8.3"),
("lollmsvectordb", "1.0.2"),
("freedom_search", "0.1.9"),
("scrapemaster", "0.1.6"),
("scrapemaster", "0.2.0"),
("lollms_client", "0.6.2")
]

Expand Down Expand Up @@ -211,6 +211,12 @@ def get_ip_addresses():

sio = socketio.AsyncServer(async_mode="asgi", cors_allowed_origins=allowed_origins, ping_timeout=1200, ping_interval=30) # Enable CORS for selected origins


# A simple fix for v 11.0 to 12 alpha
if config.rag_vectorizer=="bert":
config.rag_vectorizer="semantic"
config.save_config()

LOLLMSWebUI.build_instance(config=config, lollms_paths=lollms_paths, args=args, sio=sio)
lollmsElfServer:LOLLMSWebUI = LOLLMSWebUI.get_instance()
lollmsElfServer.verbose = True
Expand Down
7 changes: 4 additions & 3 deletions configs/config.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 135
version: 136
binding_name: null
model_name: null
model_variant: null
Expand Down Expand Up @@ -269,8 +269,8 @@ audio_silenceTimer: 5000

# Data vectorization
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
rag_vectorizer: bert # possible values bert, tfidf, word2vec
rag_vectorizer_model: bert-base-nli-mean-tokens # The model name if applicable
rag_vectorizer: semantic # possible values semantic, tfidf, openai
rag_vectorizer_model: sentence-transformers/bert-base-nli-mean-tokens # The model name if applicable
rag_vectorizer_parameters: null # Parameters of the model in json format
rag_chunk_size: 512 # number of tokens per chunk
rag_overlap: 0 # number of tokens of overlap
Expand All @@ -285,6 +285,7 @@ rag_min_nb_tokens_in_chunk: 10 #this removed any useless junk ith less than x to
rag_max_n_hops: 3 #We set the maximum number of hop in multi hops rag

rag_deactivate: false # if you have a large context model, you can activate this to use your document as a whole
rag_vectorizer_openai_key: "" # The open ai key (if not provided, this will use the environment varaible OPENAI_API_KEY)

contextual_summary: false #If activated this will completely replace the rag and instead will use contextual summary

Expand Down
2 changes: 1 addition & 1 deletion lollms_core
Submodule lollms_core updated 17 files
+4 −3 configs/config.yaml
+2 −2 elf_docker_cfg/personal/configs/lollms_elf_config.yaml
+2 −2 elf_test_cfg/personal/configs/lollms_elf_config.yaml
+2 −2 elf_test_cfg/personal/configs/lollms_elf_local_config.yaml
+13 −15 lollms/app.py
+4 −3 lollms/configs/config.yaml
+26 −9 lollms/databases/discussions_database.py
+3 −3 lollms/databases/skills_database.py
+2 −2 lollms/functions/prompting/image_gen_prompts.py
+2 −2 lollms/functions/prompting/system_prompts.py
+7 −7 lollms/internet.py
+25 −15 lollms/personality.py
+2 −2 lollms/server/configs/config.yaml
+22 −19 lollms/server/endpoints/lollms_file_system.py
+13 −2 lollms/server/endpoints/lollms_rag.py
+2 −2 lollms/services/tti/dalle/lollms_dalle.py
+2 −2 personal_data/configs/lollms_discord_local_config.yaml
2 changes: 1 addition & 1 deletion lollms_webui.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def terminate_thread(thread):
else:
ASCIIColors.yellow("Canceled successfully")# The current version of the webui

lollms_webui_version="11 code name: Wonder"
lollms_webui_version="12 (alpha) code name: Strawberry"



Expand Down
8 changes: 0 additions & 8 deletions web/dist/assets/index-41e49800.css
View file

This file was deleted.

416 changes: 208 additions & 208 deletions web/dist/assets/index-0ea89304.js → web/dist/assets/index-6460d57e.js
View file

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions web/dist/assets/index-a719ae7b.css
View file

Large diffs are not rendered by default.

Binary file added web/dist/assets/strawberry-20d9a7ba.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 2 additions & 2 deletions web/dist/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LoLLMS WebUI</title>
<script type="module" crossorigin src="/assets/index-0ea89304.js"></script>
<link rel="stylesheet" href="/assets/index-41e49800.css">
<script type="module" crossorigin src="/assets/index-6460d57e.js"></script>
<link rel="stylesheet" href="/assets/index-a719ae7b.css">
</head>
<body>
<div id="app"></div>
Expand Down
Binary file modified web/src/assets/strawberry.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
123 changes: 73 additions & 50 deletions web/src/views/DiscussionsView.vue
View file
Original file line number Diff line number Diff line change
@@ -1,60 +1,64 @@
<template>
<transition name="fade-and-fly">
<div v-if="!isReady" class="fixed top-0 left-0 w-screen h-screen flex items-center justify-center bg-gradient-to-br from-blue-100 to-purple-100 dark:from-blue-900 dark:to-purple-900 overflow-hidden">
<!-- Falling stars -->
<div class="absolute inset-0 pointer-events-none overflow-hidden">
<div v-for="n in 50" :key="n" class="absolute animate-fall"
:style="{
left: `${Math.random() * 100}%`,
top: `-20px`,
animationDuration: `${3 + Math.random() * 7}s`,
animationDelay: `${Math.random() * 5}s`
}">
<svg class="w-2 h-2 text-yellow-300" fill="currentColor" viewBox="0 0 20 20">
<path d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.175 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81h3.461a1 1 0 00.951-.69l1.07-3.292z" />
</svg>
</div>
<transition name="fade-and-fly">
<div v-if="!isReady" class="fixed top-0 left-0 w-screen h-screen flex items-center justify-center bg-gradient-to-br from-blue-100 to-purple-100 dark:from-blue-900 dark:to-purple-900 overflow-hidden">
<!-- Falling strawberries -->
<div class="absolute inset-0 pointer-events-none overflow-hidden">
<div v-for="n in 50" :key="n" class="absolute animate-fall animate-giggle"
:style="{
left: `${Math.random() * 100}%`,
top: `-20px`,
animationDuration: `${3 + Math.random() * 7}s`,
animationDelay: `${Math.random() * 5}s`
}">
<img
src="@/assets/strawberry.png"
alt="Falling Strawberry"
class="w-6 h-6"
/>
</div>
</div>

<div class="flex flex-col items-center text-center max-w-4xl w-full px-4 relative z-10">
<div class="mb-8 w-full">
<h1 class="text-5xl md:text-6xl font-bold text-transparent bg-clip-text bg-gradient-to-r from-blue-600 to-purple-600 dark:from-blue-400 dark:to-purple-400 mb-2 animate-glow">
LoLLMS
</h1>
<p class="text-2xl text-gray-600 dark:text-gray-300 italic">
One tool to rule them all
</p>
<p class="text-xl text-gray-500 dark:text-gray-400 mb-6">
by ParisNeo
<div class="flex flex-col items-center text-center max-w-4xl w-full px-4 relative z-10">
<div class="mb-8 w-full">
<h1 class="text-5xl md:text-6xl font-bold text-transparent bg-clip-text bg-gradient-to-r from-blue-600 to-purple-600 dark:from-blue-400 dark:to-purple-400 mb-2 animate-glow">
LoLLMS
</h1>
<p class="text-2xl text-gray-600 dark:text-gray-300 italic">
One tool to rule them all
</p>
<p class="text-xl text-gray-500 dark:text-gray-400 mb-6">
by ParisNeo
</p>
<p class="bottom-0 text-2xl text-gray-600 dark:text-gray-300 italic">
{{ version_info }}
</p>

<div class="w-full h-24 relative overflow-hidden bg-gradient-to-r from-blue-200 to-purple-200 dark:from-blue-800 dark:to-purple-800 rounded-full shadow-lg">
<img
class="w-24 h-24 animate-glow rounded-full absolute top-0 transition-all duration-300 ease-linear"
:style="{ left: `calc(${loading_progress}% - 3rem)` }"
title="LoLLMS WebUI"
src="@/assets/strawberry.png"
alt="Strawberry Logo"
>
</div>
</div>

<div class="w-full max-w-2xl">
<div role="status" class="w-full">
<p class="text-xl text-gray-700 dark:text-gray-300">
{{ loading_infos }}...
</p>
<p class="bottom-0 text-2xl text-gray-600 dark:text-gray-300 italic">
{{ version_info }}
<p class="text-2xl font-bold text-blue-600 dark:text-blue-400 mt-2">
{{ Math.round(loading_progress) }}%
</p>

<div class="w-full h-24 relative overflow-hidden bg-gradient-to-r from-blue-200 to-purple-200 dark:from-blue-800 dark:to-purple-800 rounded-full shadow-lg">
<img
class="w-24 h-24 animate-glow rounded-full absolute top-0 transition-all duration-300 ease-linear"
:style="{ left: `calc(${loading_progress}% - 3rem)` }"
title="LoLLMS WebUI"
src="@/assets/logo.png"
alt="Logo"
>
</div>
</div>

<div class="w-full max-w-2xl">
<div role="status" class="w-full">
<p class="text-xl text-gray-700 dark:text-gray-300">
{{ loading_infos }}...
</p>
<p class="text-2xl font-bold text-blue-600 dark:text-blue-400 mt-2">
{{ Math.round(loading_progress) }}%
</p>
</div>
</div>
</div>
</div>
</div>
</transition>
</div>
</transition>



<button v-if="isReady" @click="togglePanel" class="absolute top-2 left-2 p-3 bg-white bg-opacity-0 cursor-pointer transition-all duration-300 hover:scale-110 hover:bg-opacity-20 hover:shadow-xl group">
<div v-show="leftPanelCollapsed" ><i data-feather='chevron-right'></i></div>
Expand Down Expand Up @@ -378,6 +382,25 @@


<style scoped>
@keyframes giggle {
0%, 100% {
transform: translateX(0) rotate(0deg) scale(1);
}
25% {
transform: translateX(-5px) rotate(-10deg) scale(1.05);
}
50% {
transform: translateX(5px) rotate(10deg) scale(0.95);
}
75% {
transform: translateX(-5px) rotate(-10deg) scale(1.05);
}
}
.animate-giggle {
animation: giggle 1.5s infinite ease-in-out;
}
.custom-scrollbar {
scrollbar-width: thin;
scrollbar-color: rgba(155, 155, 155, 0.5) transparent;
Expand Down
75 changes: 62 additions & 13 deletions web/src/views/SettingsView.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1134,15 +1134,15 @@
</td>
<td>
<select
id="rag_vectorizer"
required
v-model="configFile.rag_vectorizer"
@change="settingsChanged=true"
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
id="rag_vectorizer"
required
v-model="configFile.rag_vectorizer"
@change="settingsChanged=true"
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
<option value="bert">Bert Vectorizer</option>
<option value="semantic">Semantic Vectorizer</option>
<option value="tfidf">TFIDF Vectorizer</option>
<option value="word2vec">Word2Vec Vectorizer</option>
<option value="openai">OpenAI Vectorizer</option>
</select>
</td>
</tr>
Expand All @@ -1152,16 +1152,65 @@
</td>
<td>
<select
id="rag_vectorizer_model"
id="rag_vectorizer_model"
required
v-model="configFile.rag_vectorizer_model"
@change="settingsChanged=true"
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
:disabled="configFile.rag_vectorizer === 'tfidf'"
>
<!-- Semantic Vectorizer Models -->
<option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/bert-base-nli-mean-tokens">sentence-transformers/bert-base-nli-mean-tokens</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="bert-base-uncased">bert-base-uncased</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="bert-base-multilingual-uncased">bert-base-multilingual-uncased</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="bert-large-uncased">bert-large-uncased</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="bert-large-uncased-whole-word-masking-finetuned-squad">bert-large-uncased-whole-word-masking-finetuned-squad</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="distilbert-base-uncased">distilbert-base-uncased</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="roberta-base">roberta-base</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="roberta-large">roberta-large</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="xlm-roberta-base">xlm-roberta-base</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="xlm-roberta-large">xlm-roberta-large</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="albert-base-v2">albert-base-v2</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="albert-large-v2">albert-large-v2</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="albert-xlarge-v2">albert-xlarge-v2</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="albert-xxlarge-v2">albert-xxlarge-v2</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-MiniLM-L6-v2">sentence-transformers/all-MiniLM-L6-v2</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-MiniLM-L12-v2">sentence-transformers/all-MiniLM-L12-v2</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-distilroberta-v1">sentence-transformers/all-distilroberta-v1</option>
<option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-mpnet-base-v2">sentence-transformers/all-mpnet-base-v2</option>

<!-- OpenAI Vectorizer Models -->
<option v-if="configFile.rag_vectorizer === 'openai'" value="text-embedding-ada-002">text-embedding-ada-002</option>
<option v-if="configFile.rag_vectorizer === 'openai'" value="text-embedding-babbage-001">text-embedding-babbage-001</option>
<option v-if="configFile.rag_vectorizer === 'openai'" value="text-embedding-curie-001">text-embedding-curie-001</option>
<option v-if="configFile.rag_vectorizer === 'openai'" value="text-embedding-davinci-001">text-embedding-davinci-001</option>

<!-- Disabled Option for TFIDF -->
<option v-if="configFile.rag_vectorizer === 'tfidf'" disabled>No models available for TFIDF</option>
</select>
</td>
</tr>



<tr>
<td style="min-width: 200px;">
<label for="rag_vectorizer_openai_key" class="text-sm font-bold" style="margin-right: 1rem;">Open AI key for open ai embedding method (if not provided I'll use OPENAI_API_KEY environment variable):</label>
</td>
<td>
<div class="flex flex-row">
<input
type="text"
id="rag_vectorizer_openai_key"
required
v-model="configFile.rag_vectorizer_model"
v-model="configFile.rag_vectorizer_openai_key"
@change="settingsChanged=true"
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
class="mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
<option value="bert-base-nli-mean-tokens">bert-base-nli-mean-tokens</option>
</select>
</div>
</td>
</tr>
</tr>

<tr>
<td style="min-width: 200px;">
<label for="rag_chunk_size" class="text-sm font-bold" style="margin-right: 1rem;">RAG chunk size:</label>
Expand Down
2 changes: 1 addition & 1 deletion zoos/personalities_zoo
Submodule personalities_zoo updated 1 files
+8 −8 science/scientific_bibiography_maker/scripts/processor.py

0 comments on commit 54cbcd6

Please sign in to comment.