Merge pull request #292 from n4ze3m/next

v1.10.0

app/ui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.9.5",
+  "version": "1.10.0",
   "type": "module",
   "scripts": {
     "dev": "vite",

app/ui/src/@types/bot.ts

@@ -22,6 +22,7 @@ export type BotSettings = {
     inactivityTimeout: number;
     autoResetSession: boolean;
     autoSyncDataSources: boolean;
+    internetSearchEnabled: boolean;
   };
   chatModel: {
     label: string;

app/ui/src/components/Bot/DS/DsTable.tsx

@@ -18,13 +18,15 @@ import { YoutubeIcon } from "../../Icons/YoutubeIcon";
 import { ApiIcon } from "../../Icons/ApiIcon";
 export const DsTable = ({
   data,
+  searchNode
 }: {
   data: {
     id: string;
     type: string;
     content: string;
     status: string;
   }[];
+  searchNode: React.ReactNode;
 }) => {
   const statusColor = (status: string) => {
     switch (status.toLowerCase()) {
@@ -134,6 +136,8 @@ export const DsTable = ({
       <div className="mt-8 flex flex-col">
         <div className="-my-2 -mx-4 overflow-x-auto  sm:-mx-6 lg:-mx-8">
           <div className="inline-block min-w-full p-2 align-middle md:px-6 lg:px-8">
+          {searchNode}
+
             <div className="overflow-hidden bg-white ring-1 ring-black ring-opacity-5 rounded-lg dark:bg-[#262626]">
               {data.length === 0 && (
                 <Empty description="No data sources found." className="m-8" />

app/ui/src/components/Bot/Playground/Message.tsx

@@ -10,6 +10,7 @@ import React from "react";
 import { removeUUID } from "../../../utils/filename";
 import { useSpeechSynthesis } from "../../../hooks/useSpeechSynthesis";
 import { useElevenLabsTTS } from "../../../hooks/useElevenLabsTTS";
+import { Collapse } from "antd";
 
 type Props = Message & {
   onSourceClick(source: any): void;
@@ -55,24 +56,41 @@ export const PlaygroundMessage = (props: Props) => {
               <Markdown message={props.message} />
             </div>
 
-            {props.isBot && (
-              <div className="mt-3 flex flex-wrap gap-2">
-                {props?.sources?.map((source, index) => (
-                  <button
-                    key={index}
-                    onClick={props.onSourceClick.bind(null, source)}
-                    className="inline-flex cursor-pointer transition-shadow duration-300 ease-in-out hover:shadow-lg  items-center rounded-md bg-gray-100 p-1 text-xs text-gray-800 border border-gray-300 dark:bg-gray-800 dark:border-gray-700 dark:text-gray-100 opacity-80 hover:opacity-100"
-                  >
-                    <span className="text-xs">
-                      {removeUUID(
-                        `${
-                          source?.metadata?.path || source?.metadata?.source
-                        }`.replace("./uploads/", "")
-                      )}
-                    </span>
-                  </button>
-                ))}
-              </div>
+            {props.isBot && props?.sources && props?.sources?.length > 0 && (
+              <Collapse
+                className="mt-6"
+                ghost
+                items={[
+                  {
+                    key: "1",
+                    label: (
+                      <div className="italic text-gray-500 dark:text-gray-400">
+                        Bot sources
+                      </div>
+                    ),
+                    children: (
+                      <div className="mt-3 flex flex-wrap gap-2">
+                        {props?.sources?.map((source, index) => (
+                          <button
+                            key={index}
+                            onClick={props.onSourceClick.bind(null, source)}
+                            className="inline-flex cursor-pointer transition-shadow !line-clamp-1 duration-300 ease-in-out hover:shadow-lg  items-center rounded-md bg-gray-200 p-1 text-xs text-gray-800 border border-gray-300 dark:bg-gray-800 dark:border-gray-700 dark:text-gray-100 opacity-80 hover:opacity-100 "
+                          >
+                            <span className="text-xs">
+                              {removeUUID(
+                                `${
+                                  source?.metadata?.path ||
+                                  source?.metadata?.source
+                                }`.replace("./uploads/", "")
+                              )}
+                            </span>
+                          </button>
+                        ))}
+                      </div>
+                    ),
+                  },
+                ]}
+              />
             )}
           </div>
 

app/ui/src/components/Bot/Settings/SettingsBody.tsx

@@ -169,6 +169,7 @@ export const SettingsBody: React.FC<BotSettings> = ({
             autoResetSession: data.autoResetSession,
             inactivityTimeout: data.inactivityTimeout,
             autoSyncDataSources: data.autoSyncDataSources,
+            internetSearchEnabled: data.internetSearchEnabled,
           }}
           form={form}
           requiredMark={false}
@@ -226,7 +227,24 @@ export const SettingsBody: React.FC<BotSettings> = ({
                     options={chatModel}
                   />
                 </Form.Item>
-
+                <Form.Item
+                  label={"Embedding Method"}
+                  name="embedding"
+                  help={
+                    <>
+                      <p className="text-xs text-gray-500 dark:text-gray-400">
+                        If you change the embedding method, make sure to
+                        re-fetch the data source or choose a model with the same
+                        dimensions
+                      </p>
+                    </>
+                  }
+                >
+                  <Select
+                    placeholder="Select an embedding method"
+                    options={embeddingModel}
+                  />
+                </Form.Item>
                 <Form.Item
                   hasFeedback={!isStreamingSupported(currentModel)}
                   help={
@@ -266,23 +284,14 @@ export const SettingsBody: React.FC<BotSettings> = ({
                   />
                 </Form.Item>
 
+
                 <Form.Item
-                  label={"Embedding Method"}
-                  name="embedding"
-                  help={
-                    <>
-                      <p className="text-xs text-gray-500 dark:text-gray-400">
-                        If you change the embedding method, make sure to
-                        re-fetch the data source or choose a model with the same
-                        dimensions
-                      </p>
-                    </>
-                  }
+                  name="internetSearchEnabled"
+                  label="Enable internet search"
+                  valuePropName="checked"
+                  help="This is experimental and may not work as expected."
                 >
-                  <Select
-                    placeholder="Select an embedding method"
-                    options={embeddingModel}
-                  />
+                  <Switch />
                 </Form.Item>
 
                 <Form.Item
@@ -455,7 +464,7 @@ export const SettingsBody: React.FC<BotSettings> = ({
                   />
                 </Form.Item>
 
-              <Form.Item
+                <Form.Item
                   name="autoSyncDataSources"
                   label="Auto Sync Data Source(s)"
                   tooltip="This will automatically re-fetch the URL-based data sources at a certain interval."

app/ui/src/index.css

@@ -59,3 +59,7 @@ markdown-processing {
   border: none !important;
   box-shadow: none !important;
 }
+
+.ant-collapse-header {
+  padding:  0 !important;
+}

app/ui/src/routes/bot/ds.tsx

@@ -4,19 +4,27 @@ import React from "react";
 import { SkeletonLoading } from "../../components/Common/SkeletonLoading";
 import { DsTable } from "../../components/Bot/DS/DsTable";
 import api from "../../services/api";
-import { Pagination } from "antd";
+import { Input, Pagination } from "antd";
 
 export default function BotDSRoot() {
   const param = useParams<{ id: string }>();
   const navigate = useNavigate();
   const [page, setPage] = React.useState(1);
   const [limit, setLimit] = React.useState(10);
+  const [search, setSearch] = React.useState<string | undefined>(undefined);
+  const [searchValue, setSearchValue] = React.useState<string | undefined>(
+    undefined
+  );
 
   const { data: botData, status } = useQuery(
-    ["getBotDS", param.id, page, limit],
+    ["getBotDS", param.id, page, limit, searchValue],
     async () => {
       const response = await api.get(
-        `/bot/${param.id}/source?page=${page}&limit=${limit}`
+        `/bot/${param.id}/source?page=${page}&limit=${limit}${
+          searchValue && searchValue.trim().length > 0
+            ? `&search=${searchValue}`
+            : ""
+        }`
       );
       return response.data as {
         data: {
@@ -47,7 +55,20 @@ export default function BotDSRoot() {
       {status === "loading" && <SkeletonLoading />}
       {status === "success" && (
         <div className="px-4 sm:px-6 lg:px-8">
-          <DsTable data={botData.data} />
+          <DsTable
+            data={botData.data}
+            searchNode={
+              <div className="flex mb-6 justify-end sm:justify-center md:justify-end">
+                <Input.Search
+                  value={search}
+                  onSearch={(value) => setSearchValue(value)}
+                  onChange={(e) => setSearch(e.target.value)}
+                  placeholder="Search"
+                  style={{ width: 300 }}
+                />
+              </div>
+            }
+          />
           {botData.total >= 10 && (
             <div className="my-3 flex items-center justify-end">
               <Pagination

docker/docker-compose.yml

@@ -10,11 +10,13 @@ services:
     environment:
       DATABASE_URL: postgres://postgres:postgres@dialoqbase-pg:5432/dialoqbase?connection_limit=15&pool_timeout=0
       DB_REDIS_URL: redis://redis:6379
+      DB_SEARXNG_URL: http://searxng:8080
     env_file:
       - .env
     depends_on:
       - dialoqbase-pg
       - redis
+      - searxng
     volumes:
       - .uploads:/app/uploads
 
@@ -34,4 +36,10 @@ services:
     container_name: redis
     restart: unless-stopped
     volumes:
-      - .redis:/data
+      - .redis:/data
+
+  searxng:
+    image: searxng/searxng
+    volumes:
+      - ./searxng:/etc/searxng:rw
+    restart: unless-stopped

docker/searxng/limiter.toml

@@ -0,0 +1 @@
+#https://docs.searxng.org/admin/searx.limiter.html

docker/searxng/settings.yml

@@ -0,0 +1,78 @@
+use_default_settings: true
+
+search:
+  # Filter results. 0: None, 1: Moderate, 2: Strict
+  safe_search: 0
+  # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "yandex", "mwmbl",
+  # "seznam", "startpage", "stract", "swisscows", "qwant", "wikipedia" - leave blank to turn it off
+  # by default.
+  autocomplete: 'google'
+  # minimun characters to type before autocompleter starts
+  autocomplete_min: 4
+  # Default search language - leave blank to detect from browser information or
+  # use codes from 'languages.py'
+  default_lang: 'auto'
+  # max_page: 0  # if engine supports paging, 0 means unlimited numbers of pages
+  # Available languages
+  # languages:
+  #   - all
+  #   - en
+  #   - en-US
+  #   - de
+  #   - it-IT
+  #   - fr
+  #   - fr-BE
+  # ban time in seconds after engine errors
+  ban_time_on_fail: 5
+  # max ban time in seconds after engine errors
+  max_ban_time_on_fail: 120
+  suspended_times:
+    # Engine suspension time after error (in seconds; set to 0 to disable)
+    # For error "Access denied" and "HTTP error [402, 403]"
+    SearxEngineAccessDenied: 86400
+    # For error "CAPTCHA"
+    SearxEngineCaptcha: 86400
+    # For error "Too many request" and "HTTP error 429"
+    SearxEngineTooManyRequests: 3600
+    # Cloudflare CAPTCHA
+    cf_SearxEngineCaptcha: 1296000
+    cf_SearxEngineAccessDenied: 86400
+    # ReCAPTCHA
+    recaptcha_SearxEngineCaptcha: 604800
+
+  # remove format to deny access, use lower case.
+  # formats: [html, csv, json, rss]
+  formats:
+    - html
+    - json
+
+server:
+  # Is overwritten by ${SEARXNG_PORT} and ${SEARXNG_BIND_ADDRESS}
+  port: 8888
+  bind_address: '0.0.0.0'
+  # public URL of the instance, to ensure correct inbound links. Is overwritten
+  # by ${SEARXNG_URL}.
+  base_url: false # "http://example.com/location"
+  # rate limit the number of request on the instance, block some bots.
+  # Is overwritten by ${SEARXNG_LIMITER}
+  limiter: false
+  # enable features designed only for public instances.
+  # Is overwritten by ${SEARXNG_PUBLIC_INSTANCE}
+  public_instance: false
+
+  # If your instance owns a /etc/searxng/settings.yml file, then set the following
+  # values there.
+
+  secret_key: 'KDzXs0qvZdoZnzW7Eq4jhubjgTWayRM' # Is overwritten by ${SEARXNG_SECRET}
+  # Proxy image results through SearXNG. Is overwritten by ${SEARXNG_IMAGE_PROXY}
+  image_proxy: false
+  # 1.0 and 1.1 are supported
+  http_protocol_version: '1.0'
+  # POST queries are more secure as they don't show up in history but may cause
+  # problems when using Firefox containers
+  method: 'POST'
+  default_http_headers:
+    X-Content-Type-Options: nosniff
+    X-Download-Options: noopen
+    X-Robots-Tag: noindex, nofollow
+    Referrer-Policy: no-referrer

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.9.5",
+  "version": "1.10.0",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

server/package.json

@@ -79,6 +79,7 @@
     "jsonwebtoken": "^9.0.2",
     "langchain": "^0.1.25",
     "mammoth": "^1.6.0",
+    "ml-distance": "^4.0.1",
     "pdf-parse": "^1.1.1",
     "pdfjs-dist": "^3.7.107",
     "pubsub-js": "^1.9.4",

server/prisma/migrations/q_14_5/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "Bot" ADD COLUMN     "internetSearchEnabled" BOOLEAN DEFAULT false;
+
+-- AlterTable
+ALTER TABLE "DialoqbaseSettings" ADD COLUMN     "internetSearchEnabled" BOOLEAN DEFAULT false;

server/prisma/migrations/q_14_6/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "User" ADD COLUMN     "isSuspended" BOOLEAN NOT NULL DEFAULT false;