Merge pull request #53 from snyk/feat/revert-not-returning-error

feat: revert throwing on encountering errors

.eslintrc.json

@@ -13,6 +13,7 @@
   "rules": {
     "@typescript-eslint/semi": "error",
     "@typescript-eslint/no-use-before-define": "off",
+    "@typescript-eslint/interface-name-prefix": "off",
     "@typescript-eslint/await-thenable": "error",
     "@typescript-eslint/ban-ts-ignore": "error",
     "@typescript-eslint/no-unnecessary-type-assertion": "error",

lib/berkeleydb/database-pages.ts

@@ -1,5 +1,5 @@
 import { DATABASE_PAGE_HEADER_SIZE, HASH_INDEX_ENTRY_SIZE } from './types';
-import { RpmParserError } from '../types';
+import { ParserError } from '../types';
 
 /**
  * Extract the values from a hash index, which is stored in a Hash DB page.
@@ -16,7 +16,7 @@ export function bufferToHashIndexValues(
   // Hash table entries are always stored in pairs of 2.
   if (entries % 2 !== 0) {
     const pageNumber = page.readUInt32LE(8);
-    throw new RpmParserError('The number of entries must be a multiple of 2', {
+    throw new ParserError('The number of entries must be a multiple of 2', {
       entries,
       pageNumber,
     });

lib/berkeleydb/hash-pages.ts

@@ -1,5 +1,5 @@
 import { DATABASE_PAGE_HEADER_SIZE, HashPageType } from './types';
-import { RpmParserError } from '../types';
+import { ParserError } from '../types';
 
 /**
  * Traverse the data (overflow) pages and extract the data.
@@ -24,7 +24,7 @@ export function bufferToHashValueContent(
   // Note there may be 5 different page types of varying length, but we are interested only one.
   const pageType = page.readUInt8(pageStartOffset);
   if (pageType !== HashPageType.H_OFFPAGE) {
-    throw new RpmParserError('Unsupported page type', { pageType });
+    throw new ParserError('Unsupported page type', { pageType });
   }
 
   const startPageNumber = page.readUInt32LE(pageStartOffset + 4);

lib/berkeleydb/index.ts

@@ -3,7 +3,7 @@ import { eventLoopSpinner } from 'event-loop-spinner';
 import { bufferToHashIndexValues } from './database-pages';
 import { bufferToHashValueContent } from './hash-pages';
 import { MagicNumber, DatabasePageType, HashPageType } from './types';
-import { RpmParserError } from '../types';
+import { ParserError } from '../types';
 
 export { bufferToHashIndexValues, bufferToHashValueContent };
 
@@ -31,8 +31,13 @@ const validPageSizes: ReadonlyArray<number> = [
  * - An Overflow page -- this page contains the data. The data may span multiple pages (hence "overflow" pages).
  * @param data The contents of a BerkeleyDB database.
  */
-export async function bufferToHashDbValues(data: Buffer): Promise<Buffer[]> {
+export async function bufferToHashDbValues(
+  data: Buffer,
+): Promise<Buffer[] | never> {
+  validateBerkeleyDbMetadata(data);
+
   const pageSize = data.readUInt32LE(20);
+  validatePageSize(pageSize);
 
   const lastPageNumber = data.readUInt32LE(32);
 
@@ -80,24 +85,25 @@ export async function bufferToHashDbValues(data: Buffer): Promise<Buffer[]> {
   return result;
 }
 
+/**
+ * Exported for testing
+ */
 export function validateBerkeleyDbMetadata(data: Buffer): void | never {
   // We are only interested in Hash DB. Other types are B-Tree, Queue, Heap, etc.
   const magicNumber = data.readUInt32LE(12);
   if (magicNumber !== MagicNumber.DB_HASH) {
-    throw new RpmParserError('Unexpected database magic number', {
-      magicNumber,
-    });
+    throw new ParserError('Unexpected database magic number', { magicNumber });
   }
 
   // The first page of the database must be a Hash DB metadata page.
   const pageType = data.readUInt8(25);
   if (pageType !== DatabasePageType.P_HASHMETA) {
-    throw new RpmParserError('Unexpected page type', { pageType });
+    throw new ParserError('Unexpected page type', { pageType });
   }
 
   const encryptionAlgorithm = data.readUInt8(24);
   if (encryptionAlgorithm !== 0) {
-    throw new RpmParserError('Encrypted databases are not supported', {
+    throw new ParserError('Encrypted databases are not supported', {
       encryptionAlgorithm,
     });
   }
@@ -107,7 +113,7 @@ export function validateBerkeleyDbMetadata(data: Buffer): void | never {
   // packages on the system. We don't want to allocate too much memory.
   const entriesCount = data.readUInt32LE(88);
   if (entriesCount < 0 || entriesCount > 50_000) {
-    throw new RpmParserError('Invalid number of entries in the database', {
+    throw new ParserError('Invalid number of entries in the database', {
       entriesCount,
     });
   }
@@ -118,6 +124,6 @@ export function validateBerkeleyDbMetadata(data: Buffer): void | never {
  */
 export function validatePageSize(pageSize: number): void | never {
   if (!validPageSizes.includes(pageSize)) {
-    throw new RpmParserError('Invalid page size', { pageSize });
+    throw new ParserError('Invalid page size', { pageSize });
   }
 }

lib/index.ts

@@ -1,45 +1,52 @@
-import {
-  bufferToHashDbValues,
-  validateBerkeleyDbMetadata,
-  validatePageSize,
-} from './berkeleydb';
+import { bufferToHashDbValues } from './berkeleydb';
 import { bufferToPackageInfo } from './rpm';
 import { PackageInfo } from './rpm/types';
-import { RpmParserResponse } from './types';
+import { IParserResponse } from './types';
 
 /**
  * Get a list of packages given a Buffer that contains an RPM database in BerkeleyDB format.
  * The database is inspected as best-effort, returning all valid/readable entries.
  * @param data An RPM database in BerkeleyDB format.
+ * @deprecated Should use snyk/dep-graph. The response format is kept for backwards compatibility with snyk/kubernetes-monitor.
  */
-export async function getPackages(data: Buffer): Promise<RpmParserResponse> {
-  validateBerkeleyDbMetadata(data);
-
-  const pageSize = data.readUInt32LE(20);
-  validatePageSize(pageSize);
-
-  const berkeleyDbValues = await bufferToHashDbValues(data);
-
-  let packagesSkipped = 0;
-
-  const rpmPackageInfos = new Array<PackageInfo>();
-  for (const entry of berkeleyDbValues) {
-    const packageInfo = await bufferToPackageInfo(entry);
-    if (packageInfo !== undefined) {
-      rpmPackageInfos.push(packageInfo);
-    } else {
-      packagesSkipped += 1;
+export async function getPackages(data: Buffer): Promise<IParserResponse> {
+  try {
+    const berkeleyDbValues = await bufferToHashDbValues(data);
+
+    let packagesSkipped = 0;
+    let packagesProcessed = 0;
+
+    const rpmPackageInfos = new Array<PackageInfo>();
+    for (const entry of berkeleyDbValues) {
+      try {
+        const packageInfo = await bufferToPackageInfo(entry);
+        if (packageInfo !== undefined) {
+          rpmPackageInfos.push(packageInfo);
+          packagesProcessed += 1;
+        } else {
+          packagesSkipped += 1;
+        }
+      } catch (error) {
+        packagesSkipped += 1;
+      }
     }
-  }
 
-  const formattedPackages = formatRpmPackages(rpmPackageInfos);
-  const response = formattedPackages.join('\n');
-
-  return {
-    response,
-    packagesSkipped,
-    packages: rpmPackageInfos,
-  };
+    const formattedPackages = formatRpmPackages(rpmPackageInfos);
+    const response = formattedPackages.join('\n');
+
+    return {
+      response,
+      rpmMetadata: {
+        packagesProcessed,
+        packagesSkipped,
+      },
+    };
+  } catch (error) {
+    return {
+      response: '',
+      error,
+    };
+  }
 }
 
 function formatRpmPackages(packages: PackageInfo[]): string[] {

lib/rpm/extensions.ts

@@ -1,7 +1,7 @@
 import { eventLoopSpinner } from 'event-loop-spinner';
 
 import { IndexEntry, PackageInfo, RpmTag, RpmType } from './types';
-import { RpmParserError } from '../types';
+import { ParserError } from '../types';
 
 /**
  * Iterate through RPM metadata entries to build the full package data.
@@ -10,17 +10,13 @@ import { RpmParserError } from '../types';
 export async function getPackageInfo(
   entries: IndexEntry[],
 ): Promise<PackageInfo | undefined> {
-  /**
-   * All of the entries in an RPM package are optional.
-   * We try to collect as much info as possible and finally we check if it's enough to construct a package.
-   */
   const packageInfo: Partial<PackageInfo> = {};
 
   for (const entry of entries) {
     switch (entry.info.tag) {
       case RpmTag.NAME:
         if (entry.info.type !== RpmType.STRING) {
-          throw new RpmParserError('Unexpected type for name tag', {
+          throw new ParserError('Unexpected type for name tag', {
             type: entry.info.type,
           });
         }
@@ -29,7 +25,7 @@ export async function getPackageInfo(
 
       case RpmTag.RELEASE:
         if (entry.info.type !== RpmType.STRING) {
-          throw new RpmParserError('Unexpected type for release tag', {
+          throw new ParserError('Unexpected type for release tag', {
             type: entry.info.type,
           });
         }
@@ -38,7 +34,7 @@ export async function getPackageInfo(
 
       case RpmTag.ARCH:
         if (entry.info.type !== RpmType.STRING) {
-          throw new RpmParserError('Unexpected type for arch tag', {
+          throw new ParserError('Unexpected type for arch tag', {
             type: entry.info.type,
           });
         }
@@ -47,7 +43,7 @@ export async function getPackageInfo(
 
       case RpmTag.EPOCH:
         if (entry.info.type !== RpmType.INT32) {
-          throw new RpmParserError('Unexpected type for epoch tag', {
+          throw new ParserError('Unexpected type for epoch tag', {
             type: entry.info.type,
           });
         }
@@ -56,7 +52,7 @@ export async function getPackageInfo(
 
       case RpmTag.SIZE:
         if (entry.info.type !== RpmType.INT32) {
-          throw new RpmParserError('Unexpected type for size tag', {
+          throw new ParserError('Unexpected type for size tag', {
             type: entry.info.type,
           });
         }
@@ -65,7 +61,7 @@ export async function getPackageInfo(
 
       case RpmTag.VERSION:
         if (entry.info.type !== RpmType.STRING) {
-          throw new RpmParserError('Unexpected type for version tag', {
+          throw new ParserError('Unexpected type for version tag', {
             type: entry.info.type,
           });
         }

lib/rpm/header.ts

@@ -1,7 +1,7 @@
 import { eventLoopSpinner } from 'event-loop-spinner';
 
 import { IndexEntry, ENTRY_INFO_SIZE, EntryInfo } from './types';
-import { RpmParserError } from '../types';
+import { ParserError } from '../types';
 
 /**
  * Transform a blob of metadadata into addressable RPM package entries.
@@ -14,7 +14,7 @@ export async function headerImport(data: Buffer): Promise<IndexEntry[]> {
 
   if (indexLength <= 0 || indexLength > 50_000) {
     // Ensure we don't allocate something crazy...
-    throw new RpmParserError('Invalid index length', { indexLength });
+    throw new ParserError('Invalid index length', { indexLength });
   }
 
   const entryInfos = new Array<EntryInfo>();

lib/rpm/types.ts

@@ -16,6 +16,10 @@ export interface IndexEntry {
   data: Buffer;
 }
 
+/**
+ * All of the entries in an RPM package are optional.
+ * When reading them we try to populate as much as we can.
+ */
 export interface PackageInfo {
   name: string;
   version: string;

lib/types.ts

@@ -1,34 +1,15 @@
-import { PackageInfo } from './rpm/types';
-
-export interface RpmParserResponse {
-  /**
-   * Returns all packages detected in an RPM database file.
-   * Currently this is a flat list of dependencies and not a dependency graph.
-   */
-  packages: PackageInfo[];
-
-  /**
-   * Returns the list of packages, each stored in a new line. Packages include the following attributes:
-   * name, version, epoch, size, architecture, and release. The attributes are delimited by a tab.
-   * @deprecated Use "packages" instead, which makes it easier to parse entries.
-   */
+export interface IParserResponse {
   response: string;
+  rpmMetadata?: IRpmMetadata;
+  error?: ParserError;
+}
 
-  /**
-   * Contains a count of the entries that could not be processed as RPM packages.
-   * In all circumstances this should be 0. However, if it does contain skipped packages
-   * then it may indicate:
-   * 1. A corrupt or bad RPM package/entry, or
-   * 2. A limitation of the parser - possible reasons could be a new RPM version or a bug in the parsing logic.
-   */
+export interface IRpmMetadata {
+  packagesProcessed: number;
   packagesSkipped: number;
 }
 
-/**
- * Thrown on any encountered exception by the parser.
- * Includes a "context" object to pass extra information about the error.
- */
-export class RpmParserError extends Error {
+export class ParserError extends Error {
   readonly context: unknown | undefined;
 
   constructor(message: string, context?: unknown) {

package.json

@@ -7,7 +7,6 @@
     "build": "tsc",
     "build-watch": "tsc -w",
     "debug": "tsc-watch --project tsconfig.json --onSuccess 'node --inspect --debug-brk .'",
-    "format": "prettier --write lib/**/*.ts test/**/*.ts",
     "lint": "eslint \"lib/**/*.ts\" && (cd test && eslint \"**/*.ts\")",
     "prepare": "npm run build",
     "test": "jest"
@@ -36,7 +35,6 @@
     "ts-jest": "23.10.5",
     "ts-node": "7.0.0",
     "tsc-watch": "4.2.3",
-    "typescript": "3.8.3",
-    "prettier": "2.1.2"
+    "typescript": "3.8.3"
   }
 }

test/.eslintrc.json

@@ -16,6 +16,7 @@
     "@typescript-eslint/no-explicit-any": "off",
     "@typescript-eslint/no-non-null-assertion": "off",
     "@typescript-eslint/no-use-before-define": "off",
+    "@typescript-eslint/interface-name-prefix": "off",
     "@typescript-eslint/await-thenable": "error",
     "@typescript-eslint/ban-ts-ignore": "error",
     "@typescript-eslint/no-unnecessary-type-assertion": "error",

test/index.test.ts

@@ -35,10 +35,18 @@ describe('Testing various RPM databases', () => {
 
       const parserOutput = await getPackages(rpmDb);
 
-      expect(parserOutput.packagesSkipped).toEqual(0);
-
-      const expectedEntries = expectedOutput.trim().split('\n').sort();
-      const parserEntries = parserOutput.response.trim().split('\n').sort();
+      expect(parserOutput.error).toBeUndefined();
+      expect(parserOutput.rpmMetadata).toBeDefined();
+      expect(parserOutput.rpmMetadata!.packagesSkipped).toEqual(0);
+
+      const expectedEntries = expectedOutput
+        .trim()
+        .split('\n')
+        .sort();
+      const parserEntries = parserOutput.response
+        .trim()
+        .split('\n')
+        .sort();
 
       for (let j = 0; j < expectedEntries.length; j++) {
         const expectedEntry = expectedEntries[j];