diff --git a/core/package.json b/core/package.json index 8db05a2..c2aeb6e 100644 --- a/core/package.json +++ b/core/package.json @@ -32,7 +32,7 @@ "@dqbd/tiktoken": "^1.0.7", "gitignore-parser": "^0.0.2", "gray-matter": "^4.0.3", - "mime-types": "^2.1.35", + "istextorbinary": "^9.5.0", "mustache": "^4.2.0", "openai": "^4.11.1", "temporal-polyfill": "^0.2.4", @@ -40,7 +40,6 @@ "yaml": "^2.4.1" }, "devDependencies": { - "@types/mime-types": "^2.1.4", "@types/mustache": "^4.2.5", "vitest": "^1.5.0" } diff --git a/core/src/content/gitignore_fs.ts b/core/src/content/gitignore_fs.ts index e796c8d..c679bcd 100644 --- a/core/src/content/gitignore_fs.ts +++ b/core/src/content/gitignore_fs.ts @@ -1,6 +1,6 @@ import { FileSystem, SEP } from "@davidsouther/jiffies/lib/cjs/fs.js"; import { join, normalize } from "path"; -import { contentType } from "mime-types"; +import { isText } from "istextorbinary"; import * as gitignoreParser from "gitignore-parser"; export class GitignoreFs extends FileSystem { @@ -25,9 +25,10 @@ export class GitignoreFs extends FileSystem { }); } const paths = await this.adapter.scandir(path); + const nameFilter = [".gitignore", ".git"]; const filtered = paths.filter( (p) => - p.name !== ".git" && + !nameFilter.includes(p.name) && (p.isDirectory() || isTextExtension(p.name)) && gitignores.every((g) => p.isDirectory() ? g.accepts(p.name + "/") : g.accepts(p.name) @@ -38,10 +39,5 @@ export class GitignoreFs extends FileSystem { } function isTextExtension(name: string) { - const overrides = ["go", "ts"]; - if (overrides.includes(name.split(".").pop() || "")) { - return true; - } - const contType = contentType(name) || ""; - return contType.startsWith("text") || contType.startsWith("application"); + return isText(name); } diff --git a/package-lock.json b/package-lock.json index a83900d..9642db7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,10 +14,10 @@ }, "cli": { "name": "@ailly/cli", - "version": "1.5.0", + "version": "1.5.1", "license": "ISC", "dependencies": { - "@ailly/core": "1.5.0", + "@ailly/core": "1.5.1", "@davidsouther/jiffies": "^2.2.4", "yaml": "^2.4.1" }, @@ -33,7 +33,7 @@ }, "core": { "name": "@ailly/core", - "version": "1.5.0", + "version": "1.5.1", "license": "ISC", "dependencies": { "@aws-sdk/client-bedrock-runtime": "^3.427.0", @@ -42,7 +42,7 @@ "@dqbd/tiktoken": "^1.0.7", "gitignore-parser": "^0.0.2", "gray-matter": "^4.0.3", - "mime-types": "^2.1.35", + "istextorbinary": "^9.5.0", "mustache": "^4.2.0", "openai": "^4.11.1", "temporal-polyfill": "^0.2.4", @@ -50,7 +50,6 @@ "yaml": "^2.4.1" }, "devDependencies": { - "@types/mime-types": "^2.1.4", "@types/mustache": "^4.2.5", "vitest": "^1.5.0" }, @@ -60,9 +59,9 @@ }, "extension": { "name": "ailly", - "version": "1.5.0", + "version": "1.5.1", "dependencies": { - "@ailly/core": "1.5.0", + "@ailly/core": "1.5.1", "@davidsouther/jiffies": "^2.1.3" }, "devDependencies": { @@ -3704,12 +3703,6 @@ "@types/unist": "*" } }, - "node_modules/@types/mime-types": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", - "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", - "dev": true - }, "node_modules/@types/mocha": { "version": "10.0.6", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.6.tgz", @@ -4747,6 +4740,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/binaryextensions": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-6.11.0.tgz", + "integrity": "sha512-sXnYK/Ij80TO3lcqZVV2YgfKN5QjUWIRk/XSm2J/4bd/lPko3lvk0O4ZppH6m+6hB2/GTu+ptNwVFe1xh+QLQw==", + "dependencies": { + "editions": "^6.21.0" + }, + "engines": { + "node": ">=4" + }, + "funding": { + "url": "https://bevry.me/fund" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -5777,6 +5784,20 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/editions": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/editions/-/editions-6.21.0.tgz", + "integrity": "sha512-ofkXJtn7z0urokN62DI3SBo/5xAtF0rR7tn+S/bSYV79Ka8pTajIIl+fFQ1q88DQEImymmo97M4azY3WX/nUdg==", + "dependencies": { + "version-range": "^4.13.0" + }, + "engines": { + "node": ">=4" + }, + "funding": { + "url": "https://bevry.me/fund" + } + }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -8154,6 +8175,22 @@ "node": ">=8" } }, + "node_modules/istextorbinary": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-9.5.0.tgz", + "integrity": "sha512-5mbUj3SiZXCuRf9fT3ibzbSSEWiy63gFfksmGfdOzujPjW3k+z8WvIBxcJHBoQNlaZaiyB25deviif2+osLmLw==", + "dependencies": { + "binaryextensions": "^6.11.0", + "editions": "^6.21.0", + "textextensions": "^6.11.0" + }, + "engines": { + "node": ">=4" + }, + "funding": { + "url": "https://bevry.me/fund" + } + }, "node_modules/iterator.prototype": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz", @@ -11669,6 +11706,20 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/textextensions": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-6.11.0.tgz", + "integrity": "sha512-tXJwSr9355kFJI3lbCkPpUH5cP8/M0GGy2xLO34aZCjMXBaK3SoPnZwr/oWmo1FdCnELcs4npdCIOFtq9W3ruQ==", + "dependencies": { + "editions": "^6.21.0" + }, + "engines": { + "node": ">=4" + }, + "funding": { + "url": "https://bevry.me/fund" + } + }, "node_modules/tinybench": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.8.0.tgz", @@ -12229,6 +12280,17 @@ "node": ">=12" } }, + "node_modules/version-range": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/version-range/-/version-range-4.14.0.tgz", + "integrity": "sha512-gjb0ARm9qlcBAonU4zPwkl9ecKkas+tC2CGwFfptTCWWIVTWY1YUbT2zZKsOAF1jR/tNxxyLwwG0cb42XlYcTg==", + "engines": { + "node": ">=4" + }, + "funding": { + "url": "https://bevry.me/fund" + } + }, "node_modules/vfile": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.1.tgz", @@ -13291,9 +13353,9 @@ }, "web": { "name": "@ailly/web", - "version": "1.5.0", + "version": "1.5.1", "dependencies": { - "@ailly/core": "1.5.0", + "@ailly/core": "1.5.1", "@davidsouther/jiffies": "^2.2.4", "marked-react": "^2.0.0", "react": "^18",