chore: Add internetSearchEnabled column to Bot and DialoqbaseSettings…

… tables

app/ui/src/components/Bot/DS/DsTable.tsx

@@ -18,13 +18,15 @@ import { YoutubeIcon } from "../../Icons/YoutubeIcon";
 import { ApiIcon } from "../../Icons/ApiIcon";
 export const DsTable = ({
   data,
+  searchNode
 }: {
   data: {
     id: string;
     type: string;
     content: string;
     status: string;
   }[];
+  searchNode: React.ReactNode;
 }) => {
   const statusColor = (status: string) => {
     switch (status.toLowerCase()) {
@@ -134,6 +136,8 @@ export const DsTable = ({
       <div className="mt-8 flex flex-col">
         <div className="-my-2 -mx-4 overflow-x-auto  sm:-mx-6 lg:-mx-8">
           <div className="inline-block min-w-full p-2 align-middle md:px-6 lg:px-8">
+          {searchNode}
+
             <div className="overflow-hidden bg-white ring-1 ring-black ring-opacity-5 rounded-lg dark:bg-[#262626]">
               {data.length === 0 && (
                 <Empty description="No data sources found." className="m-8" />

app/ui/src/routes/bot/ds.tsx

@@ -4,19 +4,27 @@ import React from "react";
 import { SkeletonLoading } from "../../components/Common/SkeletonLoading";
 import { DsTable } from "../../components/Bot/DS/DsTable";
 import api from "../../services/api";
-import { Pagination } from "antd";
+import { Input, Pagination } from "antd";
 
 export default function BotDSRoot() {
   const param = useParams<{ id: string }>();
   const navigate = useNavigate();
   const [page, setPage] = React.useState(1);
   const [limit, setLimit] = React.useState(10);
+  const [search, setSearch] = React.useState<string | undefined>(undefined);
+  const [searchValue, setSearchValue] = React.useState<string | undefined>(
+    undefined
+  );
 
   const { data: botData, status } = useQuery(
-    ["getBotDS", param.id, page, limit],
+    ["getBotDS", param.id, page, limit, searchValue],
     async () => {
       const response = await api.get(
-        `/bot/${param.id}/source?page=${page}&limit=${limit}`
+        `/bot/${param.id}/source?page=${page}&limit=${limit}${
+          searchValue && searchValue.trim().length > 0
+            ? `&search=${searchValue}`
+            : ""
+        }`
       );
       return response.data as {
         data: {
@@ -47,7 +55,20 @@ export default function BotDSRoot() {
       {status === "loading" && <SkeletonLoading />}
       {status === "success" && (
         <div className="px-4 sm:px-6 lg:px-8">
-          <DsTable data={botData.data} />
+          <DsTable
+            data={botData.data}
+            searchNode={
+              <div className="flex mb-6 justify-end sm:justify-center md:justify-end">
+                <Input.Search
+                  value={search}
+                  onSearch={(value) => setSearchValue(value)}
+                  onChange={(e) => setSearch(e.target.value)}
+                  placeholder="Search"
+                  style={{ width: 300 }}
+                />
+              </div>
+            }
+          />
           {botData.total >= 10 && (
             <div className="my-3 flex items-center justify-end">
               <Pagination

server/prisma/migrations/q_14_5/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "Bot" ADD COLUMN     "internetSearchEnabled" BOOLEAN DEFAULT false;
+
+-- AlterTable
+ALTER TABLE "DialoqbaseSettings" ADD COLUMN     "internetSearchEnabled" BOOLEAN DEFAULT false;

server/prisma/schema.prisma

@@ -1,6 +1,6 @@
 generator client {
   provider        = "prisma-client-js"
-  previewFeatures = ["postgresqlExtensions"]
+  previewFeatures = ["postgresqlExtensions", "fullTextSearch"]
 }
 
 datasource db {
@@ -30,7 +30,8 @@ model Bot {
   streaming                     Boolean          @default(false)
   showRef                       Boolean          @default(false)
   inactivityTimeout             Int?             @default(3600)
-  autoResetSession              Boolean          @default(false) 
+  autoResetSession              Boolean          @default(false)
+  internetSearchEnabled         Boolean?         @default(false)
   questionGeneratorPrompt       String           @default("Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question.   Chat History: {chat_history} Follow Up Input: {question} Standalone question:")
   qaPrompt                      String           @default("You are a helpful AI assistant. Use the following pieces of context to answer the question at the end. If you don't know the answer, just say you don't know. DO NOT try to make up an answer. If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context.  {context}  Question: {question} Helpful answer in markdown:")
   voice_to_text_type            String           @default("web_api")
@@ -44,7 +45,7 @@ model Bot {
   bot_api_key                   String?
   bot_model_api_key             String?
   options                       Json?            @default("{}") @db.Json
-  autoSyncDataSources           Boolean?          @default(false)
+  autoSyncDataSources           Boolean?         @default(false)
   BotAppearance                 BotAppearance[]
   document                      BotDocument[]
   BotIntegration                BotIntegration[]
@@ -96,13 +97,14 @@ model DialoqbaseSettings {
   allowUserToCreateBots        Boolean  @default(true)
   allowUserToRegister          Boolean  @default(false)
   numberOfDocuments            Int      @default(10)
-  fileUploadSizeLimit          Int?      @default(10)
+  fileUploadSizeLimit          Int?     @default(10)
   legacyMode                   Boolean  @default(false)
   hidePlayground               Boolean  @default(false)
   defaultChunkSize             Int?     @default(1000)
   defaultChunkOverlap          Int?     @default(200)
   dynamicallyFetchOllamaModels Boolean? @default(false)
   hideDefaultModels            Boolean? @default(false)
+  internetSearchEnabled        Boolean? @default(false)
   defaultChatModel             String   @default("gpt-3.5-turbo-dbase")
   defaultEmbeddingModel        String   @default("dialoqbase_eb_text-embedding-ada-002")
   ollamaURL                    String?  @default("http://host.docker.internal:11434")

server/src/handlers/api/v1/bot/bot/get.handler.ts

@@ -3,7 +3,15 @@ import { FastifyReply, FastifyRequest } from "fastify";
 import { GetBotRequestById, GetDatasourceByBotId } from "./types";
 import { getSettings } from "../../../../../utils/common";
 import { getAllOllamaModels } from "../../../../../utils/ollama";
-
+const preprocessSearchTerms = (searchTerm: string) => {
+  const tsquerySpecialChars = /[()|&:*!]/g
+  const search =  searchTerm
+    .trim()
+    .replace(tsquerySpecialChars, ' ')
+    .split(/\s+/)
+    .join(' | ')
+    return search
+}
 export const getBotByIdEmbeddingsHandler = async (
   request: FastifyRequest<GetBotRequestById>,
   reply: FastifyReply
@@ -14,7 +22,7 @@ export const getBotByIdEmbeddingsHandler = async (
   const bot = await prisma.bot.findFirst({
     where: {
       id,
-      user_id: request.user?.is_admin ? undefined : request.user?.user_id
+      user_id: request.user?.is_admin ? undefined : request.user?.user_id,
     },
   });
 
@@ -45,11 +53,12 @@ export const getDatasourceByBotId = async (
   const id = request.params.id;
   const { limit, page } = request.query;
   const skip = (page - 1) * limit;
+  const search = request.query.search;
 
   const bot = await prisma.bot.findFirst({
     where: {
       id,
-      user_id: request.user?.is_admin ? undefined : request.user?.user_id
+      user_id: request.user?.user_id,
     },
   });
 
@@ -58,35 +67,74 @@ export const getDatasourceByBotId = async (
       message: "Bot not found",
     });
   }
-
-  const sources = await prisma.botSource.findMany({
-    where: {
-      botId: id,
-      type: {
-        notIn: ["crawl", "sitemap", "zip"],
+  const [sources, totalCount] = await Promise.all([
+    prisma.botSource.findMany({
+      where: {
+        botId: id,
+        type: {
+          notIn: ["crawl", "sitemap", "zip"],
+        },
+        OR: search
+          ? [
+            {
+              content: {
+                search:  preprocessSearchTerms(search),
+              },
+            },
+            {
+              document: {
+                some: {
+                  content: {
+                    search: preprocessSearchTerms(search),
+                  },
+                },
+              },
+            },
+          ]
+          : undefined,
       },
-    },
-    orderBy: {
-      createdAt: 'desc'
-    },
-    skip,
-    take: limit,
-  });
-
-  const totalCount = await prisma.botSource.count({
-    where: {
-      botId: id,
-      type: {
-        notIn: ["crawl", "sitemap", "zip"],
+      orderBy: {
+        createdAt: "desc",
       },
-    },
-  });
+      skip,
+      take: limit,
+      include: {
+        document: true,
+      },
+    }),
+    prisma.botSource.count({
+      where: {
+        botId: id,
+        type: {
+          notIn: ["crawl", "sitemap", "zip"],
+        },
+        OR: search
+          ? [
+            {
+              content: {
+                search: preprocessSearchTerms(search),
+              },
+            },
+            {
+              document: {
+                some: {
+                  content: {
+                    search: preprocessSearchTerms(search),
+                  },
+                },
+              },
+            },
+          ]
+          : undefined,
+      },
+    }),
+  ]);
 
   return {
     data: sources,
     total: totalCount,
     next: page * limit < totalCount ? page + 1 : null,
-    prev: page > 1 ? page - 1 : null
+    prev: page > 1 ? page - 1 : null,
   };
 };
 
@@ -100,7 +148,7 @@ export const getBotByIdHandler = async (
   const bot = await prisma.bot.findFirst({
     where: {
       id,
-      user_id: request.user?.is_admin ? undefined : request.user?.user_id
+      user_id: request.user?.is_admin ? undefined : request.user?.user_id,
     },
   });
 
@@ -121,7 +169,7 @@ export const getAllBotsHandler = async (
   const prisma = request.server.prisma;
   const bots = await prisma.bot.findMany({
     where: {
-      user_id: request.user?.user_id
+      user_id: request.user?.user_id,
     },
     orderBy: {
       createdAt: "desc",
@@ -201,7 +249,7 @@ export const getCreateBotConfigHandler = async (
     embeddingModel,
     defaultChatModel: settings?.defaultChatModel,
     defaultEmbeddingModel: settings?.defaultEmbeddingModel,
-    fileUploadSizeLimit: settings?.fileUploadSizeLimit
+    fileUploadSizeLimit: settings?.fileUploadSizeLimit,
   };
 };
 
@@ -215,7 +263,7 @@ export const getBotByIdSettingsHandler = async (
   const bot = await prisma.bot.findFirst({
     where: {
       id,
-      user_id: request.user?.is_admin ? undefined : request.user?.user_id
+      user_id: request.user?.is_admin ? undefined : request.user?.user_id,
     },
   });
   if (!bot) {
@@ -291,7 +339,7 @@ export const isBotReadyHandler = async (
   const bot = await prisma.bot.findFirst({
     where: {
       id,
-      user_id: request.user?.is_admin ? undefined : request.user?.user_id
+      user_id: request.user?.is_admin ? undefined : request.user?.user_id,
     },
   });
 

server/src/handlers/api/v1/bot/bot/types.ts

@@ -150,5 +150,6 @@ export type GetDatasourceByBotId  = {
   Querystring: {
       limit: number;
       page: number;
+      search?: string
   }
 }