Merge pull request #290 from n4ze3m/next

v1.9.5

app/ui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.9.4",
+  "version": "1.9.5",
   "type": "module",
   "scripts": {
     "dev": "vite",

app/ui/src/@types/bot.ts

@@ -21,6 +21,7 @@ export type BotSettings = {
     semanticSearchSimilarityScore: string;
     inactivityTimeout: number;
     autoResetSession: boolean;
+    autoSyncDataSources: boolean;
   };
   chatModel: {
     label: string;

app/ui/src/components/Bot/Settings/SettingsBody.tsx

@@ -168,6 +168,7 @@ export const SettingsBody: React.FC<BotSettings> = ({
             semanticSearchSimilarityScore: data.semanticSearchSimilarityScore,
             autoResetSession: data.autoResetSession,
             inactivityTimeout: data.inactivityTimeout,
+            autoSyncDataSources: data.autoSyncDataSources,
           }}
           form={form}
           requiredMark={false}
@@ -453,6 +454,14 @@ export const SettingsBody: React.FC<BotSettings> = ({
                     placeholder="Enter inactivity timeout"
                   />
                 </Form.Item>
+
+              <Form.Item
+                  name="autoSyncDataSources"
+                  label="Auto Sync Data Source(s)"
+                  tooltip="This will automatically re-fetch the URL-based data sources at a certain interval."
+                >
+                  <Switch />
+                </Form.Item>
               </div>
             </div>
 

app/ui/src/components/Common/BotForm.tsx

@@ -35,7 +35,7 @@ type Props = {
   form: FormInstance<any>;
   showEmbeddingAndModels: boolean;
   newSelectedSource?: any;
-  botConfig: BotConfig
+  botConfig: BotConfig;
 };
 function classNames(...classes: string[]) {
   return classes.filter(Boolean).join(" ");
@@ -47,7 +47,7 @@ export const BotForm = ({
   setSelectedSource,
   form,
   showEmbeddingAndModels,
-  botConfig
+  botConfig,
 }: Props) => {
   const youtubeMode = Form.useWatch(["options", "youtube_mode"], form);
   const url = Form.useWatch(["content"], form);
@@ -117,7 +117,7 @@ export const BotForm = ({
             }}
           >
             <Upload.Dragger
-              accept={`.pdf,.docx,.csv,.txt,.mp3,.mp4`}
+              accept={`.pdf,.docx,.csv,.txt,.mp3,.mp4,.zip`}
               multiple={true}
               maxCount={botConfig?.fileUploadSizeLimit}
               beforeUpload={(file) => {
@@ -130,6 +130,8 @@ export const BotForm = ({
                   "audio/mp4",
                   "video/mp4",
                   "video/mpeg",
+                  "application/zip",
+                  "application/x-zip-compressed",
                 ]
                   .map((type) => type.toLowerCase())
                   .join(", ");
@@ -161,12 +163,13 @@ export const BotForm = ({
                   <InboxIcon className="h-10 w-10 text-gray-400" />
                 </p>
                 <p className="ant-upload-text">
-                  Click or drag PDF, Docx, CSV , TXT, MP3, MP4 files to this
+                  Click or drag PDF, Docx, CSV , TXT, MP3, MP4, Zip files to
+                  this
                 </p>
                 <p className="ant-upload-hint">
-                  Support is available for a single or bulk upload of up to 10
+                  {`Support is available for a single or bulk upload of up to ${botConfig?.fileUploadSizeLimit}
                   files. Please note that file upload is in beta, so if you
-                  encounter any issues, kindly report them.
+                  encounter any issues, kindly report them.`}
                 </p>
               </div>
             </Upload.Dragger>

app/ui/src/routes/settings/application.tsx

@@ -29,6 +29,7 @@ export default function SettingsApplicationRoot() {
       dynamicallyFetchOllamaModels: boolean;
       ollamaURL: string;
       fileUploadSizeLimit: number;
+      refetchDatasource: boolean;
     };
   });
 
@@ -173,9 +174,15 @@ export default function SettingsApplicationRoot() {
                     ]}
                     tooltip="Default is 10"
                   >
-
                     <InputNumber size="large" style={{ width: "100%" }} />
-
+                  </Form.Item>
+                  <Form.Item
+                    label="Refetch Data Source"
+                    name="refetchDatasource"
+                    valuePropName="checked"
+                    help="This will refetch the data source at a specific interval."
+                  >
+                    <Switch />
                   </Form.Item>
                 </div>
                 <div className="bg-gray-50 border-x border-b rounded-b-md rounded-x-md px-4 py-3 text-right sm:px-6 dark:bg-[#141414] dark:border-gray-600">

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.9.4",
+  "version": "1.9.5",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

server/package.json

@@ -61,6 +61,7 @@
     "cohere-ai": "^6.2.1",
     "concurrently": "^7.0.0",
     "copyfiles": "^2.4.1",
+    "cron": "^3.1.7",
     "d3-dsv": "2",
     "date-fns": "^3.6.0",
     "discord.js": "^14.11.0",
@@ -92,6 +93,8 @@
     "turndown": "^7.1.3",
     "unique-names-generator": "^4.7.1",
     "wavefile": "^11.0.0",
+    "yauzl": "^3.1.3",
+    "yauzl-promise": "^4.0.0",
     "yt-transcript": "^0.0.2",
     "ytdl-core": "^4.11.5"
   },
@@ -103,6 +106,8 @@
     "@types/pubsub-js": "^1.8.3",
     "@types/tap": "^15.0.5",
     "@types/turndown": "^5.0.4",
+    "@types/yauzl": "^2.10.3",
+    "@types/yauzl-promise": "^4.0.1",
     "c8": "9.0.0",
     "fastify-tsconfig": "2.0.0",
     "prisma": "^5.9.1",

server/prisma/migrations/q_14_3/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "DialoqbaseSettings" ADD COLUMN     "refetchDatasource" BOOLEAN NOT NULL DEFAULT false;

server/prisma/migrations/q_14_4/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Bot" ADD COLUMN     "autoSyncDataSources" BOOLEAN DEFAULT false;

server/prisma/schema.prisma

@@ -44,6 +44,7 @@ model Bot {
   bot_api_key                   String?
   bot_model_api_key             String?
   options                       Json?            @default("{}") @db.Json
+  autoSyncDataSources           Boolean?          @default(false)
   BotAppearance                 BotAppearance[]
   document                      BotDocument[]
   BotIntegration                BotIntegration[]
@@ -106,6 +107,7 @@ model DialoqbaseSettings {
   defaultEmbeddingModel        String   @default("dialoqbase_eb_text-embedding-ada-002")
   ollamaURL                    String?  @default("http://host.docker.internal:11434")
   usePuppeteerFetch            Boolean? @default(false)
+  refetchDatasource            Boolean  @default(false)
 }
 
 model BotIntegration {

server/src/app.ts

@@ -13,6 +13,9 @@ import swaggerUi from "@fastify/swagger-ui";
 import { pathToFileURL } from "url";
 import { Worker } from "bullmq";
 import { parseRedisUrl } from "./utils/redis";
+import { CronJob } from 'cron';
+import { processDatasourceCron } from "./cron/index";
+
 declare module "fastify" {
   interface Session {
     is_bot_allowed: boolean;
@@ -31,7 +34,11 @@ const app: FastifyPluginAsync<AppOptions> = async (
 
   void fastify.register(FastifySSEPlugin);
 
-  void fastify.register(fastifyMultipart);
+  void fastify.register(fastifyMultipart, {
+    limits: {
+      fileSize: 1 * 1024 * 1024 * 1024,
+    }
+  });
 
   void fastify.register(swagger);
 
@@ -99,8 +106,17 @@ const worker = new Worker("vector", workerUrl, {
   useWorkerThreads: workerThreads === "true",
 });
 
+const job = new CronJob(
+  process.env.DB_CRON_TIME || '0 0 0 * * *',
+  processDatasourceCron,
+  null,
+  true,
+  process.env.DB_CRON_TIMEZONE
+);
+
 process.on("SIGINT", async () => {
   await worker.close();
+  job.stop();
   process.exit();
 });
 

server/src/chain/index.ts

@@ -21,6 +21,17 @@ type RetrievalChainInput = {
   question: string;
 };
 
+const updateTemplateVariables = (template: string) => {
+  // replace template {time} with current time
+  template = template.replace("{time}", new Date().toLocaleTimeString());
+  // replace template {date} with current date
+  template = template.replace("{date}", new Date().toLocaleDateString());
+  // replace template {day} with current day
+  template = template.replace("{day}", new Date().toLocaleString('en-us', { weekday: 'long' }));
+
+  return template;
+}
+
 export function groupMessagesByConversation(messages: any[]) {
   // check if messages are in even numbers if not remove the last message
   if (messages.length % 2 !== 0) {
@@ -109,12 +120,17 @@ export const createChain = ({
   retriever,
   response_template,
 }: {
-  llm: BaseLanguageModel<any> | BaseChatModel<any> ;
+  llm: BaseLanguageModel<any> | BaseChatModel<any>;
   question_llm: BaseLanguageModel<any> | BaseChatModel<any>;
   retriever: Runnable;
   question_template: string;
   response_template: string;
 }) => {
+
+  question_template = updateTemplateVariables(question_template);
+
+  response_template = updateTemplateVariables(response_template);
+
   const retrieverChain = createRetrieverChain(
     question_llm,
     retriever,

server/src/cron/index.ts

@@ -0,0 +1,70 @@
+import { PrismaClient } from "@prisma/client";
+import { getSettings } from "../utils/common";
+import { queue } from "../queue/q";
+const prisma = new PrismaClient();
+
+async function processDatasourceCron() {
+    try {
+        await prisma.$connect();
+        const setting = await getSettings(prisma);
+
+        if (!setting.refetchDatasource) {
+            return;
+        }
+
+        console.log("[CRON] Processing datasource cron");
+
+
+        const dataSources = await prisma.botSource.findMany({
+            where: {
+                bot: {
+                    autoSyncDataSources: true
+                },
+                type: {
+                    in: [
+                        "website",
+                        "crawl",
+                        "sitemap",
+                    ]
+                }
+            },
+            include: {
+                bot: true
+            }
+        })
+
+        for (const dataSource of dataSources) {
+
+            await prisma.botDocument.deleteMany({
+                where: {
+                    botId: dataSource.botId,
+                    sourceId: dataSource.id,
+                },
+            });
+            await queue.add(
+                "process",
+                [
+                    {
+                        ...dataSource,
+                        embedding: dataSource.bot.embedding,
+                    },
+                ],
+                {
+                    jobId: dataSource.id,
+                    removeOnComplete: true,
+                    removeOnFail: true,
+                }
+            );
+        }
+
+
+        console.log("[CRON] Finished processing datasource cron");
+
+    } catch (error) {
+        console.error(error);
+    } finally {
+        await prisma.$disconnect();
+    }
+}
+
+export { processDatasourceCron }; 

server/src/handlers/api/v1/admin/type.ts

@@ -5,6 +5,7 @@ export type UpdateDialoqbaseSettingsRequest = {
     allowUserToRegister: boolean;
     usePuppeteerFetch: boolean;
     fileUploadSizeLimit: number;
+    refetchDatasource: boolean;
   };
 };
 

server/src/handlers/api/v1/bot/bot/get.handler.ts

@@ -63,7 +63,7 @@ export const getDatasourceByBotId = async (
     where: {
       botId: id,
       type: {
-        notIn: ["crawl", "sitemap"],
+        notIn: ["crawl", "sitemap", "zip"],
       },
     },
     orderBy: {
@@ -77,7 +77,7 @@ export const getDatasourceByBotId = async (
     where: {
       botId: id,
       type: {
-        notIn: ["crawl", "sitemap"],
+        notIn: ["crawl", "sitemap", "zip"],
       },
     },
   });
@@ -119,10 +119,9 @@ export const getAllBotsHandler = async (
   reply: FastifyReply
 ) => {
   const prisma = request.server.prisma;
-
   const bots = await prisma.bot.findMany({
     where: {
-      user_id: request.user?.is_admin ? undefined : request.user?.user_id
+      user_id: request.user?.user_id
     },
     orderBy: {
       createdAt: "desc",

server/src/handlers/api/v1/bot/bot/upload.handler.ts

@@ -124,7 +124,7 @@ export const createBotFileHandler = async (
       const path = `./uploads/${fileName}`;
       await fs.promises.mkdir("./uploads", { recursive: true });
       await pump(
-         file.file,
+        file.file,
         fs.createWriteStream(path) as any
       );
       const type = fileTypeFinder(file.mimetype);

server/src/plugins/bull.ts

@@ -1,7 +1,7 @@
 import fp from "fastify-plugin";
 import { FastifyPluginAsync } from "fastify";
 import { Queue } from "bullmq";
-import { parseRedisUrl } from "../utils/redis";
+import { queue } from "../queue/q";
 declare module "fastify" {
   interface FastifyInstance {
     queue: Queue<any>;
@@ -13,16 +13,6 @@ const bullPlugin: FastifyPluginAsync = fp(async (server, options) => {
   if (!redis_url) {
     throw new Error("Redis url is not defined");
   }
-  const { host, port, password } = parseRedisUrl(redis_url);
-
-  const queue = new Queue("vector", {
-    connection: {
-      host,
-      port,
-      password,
-      username: process?.env?.DB_REDIS_USERNAME,
-    },
-  });
 
   server.decorate("queue", queue);