Replace mime-types extension with handrolled check. (#154)

core/package.json

@@ -32,15 +32,13 @@
     "@dqbd/tiktoken": "^1.0.7",
     "gitignore-parser": "^0.0.2",
     "gray-matter": "^4.0.3",
-    "mime-types": "^2.1.35",
     "mustache": "^4.2.0",
     "openai": "^4.11.1",
     "temporal-polyfill": "^0.2.4",
     "vectra": "^0.6.0",
     "yaml": "^2.4.1"
   },
   "devDependencies": {
-    "@types/mime-types": "^2.1.4",
     "@types/mustache": "^4.2.5",
     "vitest": "^1.5.0"
   }

core/src/content/gitignore_fs.test.ts

@@ -3,6 +3,16 @@ import { describe, expect, it } from "vitest";
 import { GitignoreFs } from "./gitignore_fs.js";
 
 describe("gitignore fs", () => {
+  it("skips files with no extension", async () => {
+    const fs = new GitignoreFs(
+      new ObjectFileSystemAdapter({
+        "file.txt": "abc",
+        skip: "skip",
+      })
+    );
+
+    expect(await fs.readdir("/")).toEqual(["file.txt"]);
+  });
   it("reads while obeying .gitignores", async () => {
     const fs = new GitignoreFs(
       new ObjectFileSystemAdapter({

core/src/content/gitignore_fs.ts

@@ -1,6 +1,5 @@
 import { FileSystem, SEP } from "@davidsouther/jiffies/lib/cjs/fs.js";
 import { join, normalize } from "path";
-import { contentType } from "mime-types";
 import * as gitignoreParser from "gitignore-parser";
 
 export class GitignoreFs extends FileSystem {
@@ -25,9 +24,10 @@ export class GitignoreFs extends FileSystem {
       });
     }
     const paths = await this.adapter.scandir(path);
+    const ignoredNames = [".git", ".gitignore"];
     const filtered = paths.filter(
       (p) =>
-        p.name !== ".git" &&
+        !ignoredNames.includes(p.name) &&
         (p.isDirectory() || isTextExtension(p.name)) &&
         gitignores.every((g) =>
           p.isDirectory() ? g.accepts(p.name + "/") : g.accepts(p.name)
@@ -38,10 +38,29 @@ export class GitignoreFs extends FileSystem {
 }
 
 function isTextExtension(name: string) {
-  const overrides = ["go", "ts"];
-  if (overrides.includes(name.split(".").pop() || "")) {
-    return true;
+  // If theres no extension, err on the side
+  // of caution and don't include it in the context.
+  if (!name.includes(".")) {
+    return false;
   }
-  const contType = contentType(name) || "";
-  return contType.startsWith("text") || contType.startsWith("application");
+
+  // From https://github.com/bevry/binaryextensions/blob/master/source/index.ts
+  const binaryExtensions = [
+    "dds",
+    "eot",
+    "gif",
+    "ico",
+    "jar",
+    "jpeg",
+    "jpg",
+    "pdf",
+    "png",
+    "swf",
+    "tga",
+    "ttf",
+    "zip",
+  ];
+
+  const ext = name.split(".").pop() || "";
+  return !binaryExtensions.includes(ext);
 }

integ/package.json

@@ -1,12 +1,13 @@
 {
   "name": "integ",
   "version": "1.0.0",
-  "description": "",
   "main": "index.js",
+  "devDependencies": {},
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "keywords": [],
   "author": "",
-  "license": "ISC"
+  "license": "ISC",
+  "description": ""
 }