Switch package for checking filetype.

The previous package, mime-type, was too narrowly focused. The
`istextorbinary` package broadly covers a large list of file extensions.

core/package.json

@@ -32,15 +32,14 @@
     "@dqbd/tiktoken": "^1.0.7",
     "gitignore-parser": "^0.0.2",
     "gray-matter": "^4.0.3",
-    "mime-types": "^2.1.35",
+    "istextorbinary": "^9.5.0",
     "mustache": "^4.2.0",
     "openai": "^4.11.1",
     "temporal-polyfill": "^0.2.4",
     "vectra": "^0.6.0",
     "yaml": "^2.4.1"
   },
   "devDependencies": {
-    "@types/mime-types": "^2.1.4",
     "@types/mustache": "^4.2.5",
     "vitest": "^1.5.0"
   }

core/src/content/gitignore_fs.ts

@@ -1,6 +1,6 @@
 import { FileSystem, SEP } from "@davidsouther/jiffies/lib/cjs/fs.js";
 import { join, normalize } from "path";
-import { contentType } from "mime-types";
+import { isText } from "istextorbinary";
 import * as gitignoreParser from "gitignore-parser";
 
 export class GitignoreFs extends FileSystem {
@@ -25,9 +25,10 @@ export class GitignoreFs extends FileSystem {
       });
     }
     const paths = await this.adapter.scandir(path);
+    const nameFilter = [".gitignore", ".git"];
     const filtered = paths.filter(
       (p) =>
-        p.name !== ".git" &&
+        !nameFilter.includes(p.name) &&
         (p.isDirectory() || isTextExtension(p.name)) &&
         gitignores.every((g) =>
           p.isDirectory() ? g.accepts(p.name + "/") : g.accepts(p.name)
@@ -38,10 +39,5 @@ export class GitignoreFs extends FileSystem {
 }
 
 function isTextExtension(name: string) {
-  const overrides = ["go", "ts"];
-  if (overrides.includes(name.split(".").pop() || "")) {
-    return true;
-  }
-  const contType = contentType(name) || "";
-  return contType.startsWith("text") || contType.startsWith("application");
+  return isText(name);
 }