feat: add codspeed benchmarks tracking to compare performance changes (…

…#552)

.github/workflows/codspeed.yaml

@@ -0,0 +1,32 @@
+name: codspeed-benchmarks
+
+on:
+  push:
+    branches:
+      - "main" # or "master"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-node@v4"
+        with:
+          node-version: 20
+      - run: corepack enable
+      - run: pnpm --version
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: "**/pnpm-lock.yaml"
+      - name: install
+        run: pnpm install --frozen-lockfile --prefer-offline
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          run: pnpm run test:bench

package.json

@@ -16,6 +16,7 @@
     "dev": "turbo run dev --parallel",
     "lint": "turbo run lint",
     "test": "turbo run test",
+    "test:bench": "turbo run test:bench -- --run",
     "test:e2e": "turbo run test:e2e",
     "format": "prettier --write \"**/*.{ts,tsx,md,vue}\"",
     "version": "changeset version",

packages/api-client-next/package.json

@@ -50,6 +50,7 @@
     "test:watch": "vitest"
   },
   "devDependencies": {
+    "@codspeed/vitest-plugin": "^2.3.1",
     "@shopware/api-gen": "workspace:*",
     "@types/prettier": "^3.0.0",
     "@vitest/coverage-v8": "^1.2.2",

packages/api-client-next/src/tests/apiClient.bench.ts

@@ -0,0 +1,17 @@
+import { describe, bench } from "vitest";
+import { createAPIClient, createAdminAPIClient } from "..";
+
+describe("apiClient benchmarks", () => {
+  bench("[api-client][createAPIClient] - creating client", async () => {
+    createAPIClient({
+      baseURL: "https://example.com",
+      accessToken: "123",
+    });
+  });
+
+  bench("[api-client][createAdminAPIClient] - creating client", async () => {
+    createAdminAPIClient({
+      baseURL: "https://example.com",
+    });
+  });
+});

packages/api-client-next/src/tests/transform.bench.ts

@@ -2,40 +2,58 @@ import { describe, bench } from "vitest";
 import { transformPathToQuery } from "../transformPathToQuery";
 
 describe("transforming requests", () => {
-  bench("transform without any params", () => {
-    transformPathToQuery("readCart get /checkout/cart", {});
-  });
+  bench(
+    "[api-client][transformPathToQuery] - transform without any params",
+    () => {
+      transformPathToQuery("readCart get /checkout/cart", {});
+    },
+  );
 
-  bench("transform with query params", () => {
-    transformPathToQuery("readCart get /checkout/cart?name", {
-      name: "myId123",
-    });
-  });
+  bench(
+    "[api-client][transformPathToQuery] - transform with query params",
+    () => {
+      transformPathToQuery("readCart get /checkout/cart?name", {
+        name: "myId123",
+      });
+    },
+  );
 
-  bench("transform with body params", () => {
-    transformPathToQuery("readCart get /checkout/cart", {
-      name: "myId123",
-    });
-  });
+  bench(
+    "[api-client][transformPathToQuery] - transform with body params",
+    () => {
+      transformPathToQuery("readCart get /checkout/cart", {
+        name: "myId123",
+      });
+    },
+  );
 
-  bench("transform with path params", () => {
-    transformPathToQuery("readCart get /checkout/cart/{name}", {
-      name: "myId123",
-    });
-  });
+  bench(
+    "[api-client][transformPathToQuery] - transform with path params",
+    () => {
+      transformPathToQuery("readCart get /checkout/cart/{name}", {
+        name: "myId123",
+      });
+    },
+  );
 
-  bench("transform with header params", () => {
-    transformPathToQuery("readCart get /checkout/cart name", {
-      name: "myId123",
-    });
-  });
+  bench(
+    "[api-client][transformPathToQuery] - transform with header params",
+    () => {
+      transformPathToQuery("readCart get /checkout/cart name", {
+        name: "myId123",
+      });
+    },
+  );
 
-  bench("transform with path,query,body,header params", () => {
-    transformPathToQuery("readCart get /checkout/cart/{id}?name surname", {
-      id: "myId123",
-      name: "myId123",
-      surname: "myOtherId",
-      bodyParam: "some123",
-    });
-  });
+  bench(
+    "[api-client][transformPathToQuery] - transform with path,query,body,header params",
+    () => {
+      transformPathToQuery("readCart get /checkout/cart/{id}?name surname", {
+        id: "myId123",
+        name: "myId123",
+        surname: "myOtherId",
+        bodyParam: "some123",
+      });
+    },
+  );
 });

packages/api-client-next/vitest.config.ts

@@ -1,6 +1,8 @@
 import { defineConfig } from "vitest/config";
+import codspeedPlugin from "@codspeed/vitest-plugin";
 
 export default defineConfig({
+  plugins: [codspeedPlugin()],
   test: {
     coverage: {
       enabled: true,

packages/api-gen/src/cli.ts

@@ -13,7 +13,7 @@ export interface CommonOptions {
 function commonOptions(args: Argv<{}>): Argv<CommonOptions> {
   return args.option("cwd", {
     alias: "C",
-    default: "",
+    default: process.cwd(),
     type: "string",
     describe: "specify the current working directory",
   });

packages/api-gen/src/commands/generate.ts

@@ -1,5 +1,5 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
-import { resolve } from "node:path";
+import { resolve, join } from "node:path";
 import openapiTS from "openapi-typescript";
 import type { OpenAPI3 } from "openapi-typescript";
 import * as dotenv from "dotenv";
@@ -12,16 +12,20 @@ const config = dotenv.config().parsed || {};
 
 export async function generate(args: { cwd: string; filename: string }) {
   try {
+    const start = performance.now();
     const outputFilename = args.filename.replace(".json", ".d.ts");
 
+    const fullInputFilePath = join(args.cwd, args.filename);
+    const fullOutputFilePath = join(args.cwd, outputFilename);
+
     //check if file exist
-    const fileExist = existsSync(args.filename);
+    const fileExist = existsSync(fullInputFilePath);
 
     if (!fileExist) {
       console.log(
         c.yellow(
           `Schema file ${c.bold(
-            args.filename,
+            fullInputFilePath,
           )} does not exist. Check whether the file is created (use ${c.bold(
             "loadSchema",
           )} command first).`,
@@ -31,7 +35,7 @@ export async function generate(args: { cwd: string; filename: string }) {
     }
 
     // Apply patches
-    const schemaFile = readFileSync(args.filename, {
+    const schemaFile = readFileSync(fullInputFilePath, {
       encoding: "utf-8",
     });
     let schemaForPatching = JSON.parse(schemaFile) as OpenAPI3;
@@ -53,19 +57,19 @@ export async function generate(args: { cwd: string; filename: string }) {
       parser: "json",
     });
     const content = formatted.trim();
-    writeFileSync(args.filename, content, {
+    writeFileSync(fullInputFilePath, content, {
       encoding: "utf-8",
     });
 
-    const readedContentFromFile = readFileSync(args.filename, {
+    const readedContentFromFile = readFileSync(fullInputFilePath, {
       encoding: "utf-8",
     });
 
     const originalSchema = JSON.parse(readedContentFromFile);
     const { paths } = originalSchema;
     console.log("schema", originalSchema.info);
 
-    const address = resolve(process.cwd(), args.filename);
+    const address = resolve(fullInputFilePath);
     let schema = await openapiTS(
       // new URL(SCHEMA_FILENAME, import.meta.url),
       address,
@@ -232,13 +236,19 @@ export async function generate(args: { cwd: string; filename: string }) {
     schema = schema.trim();
 
     if (typeof schema === "string") {
-      writeFileSync(outputFilename, schema, {
+      writeFileSync(fullOutputFilePath, schema, {
         encoding: "utf-8",
       });
     } else {
       throw new Error("Schema is not a string");
     }
-    console.log(c.green(`Types generated in ${c.bold(outputFilename)}`));
+    const stop = performance.now();
+    const time = Math.round(stop - start);
+    console.log(
+      c.green(
+        `Types generated in ${c.bold(fullOutputFilePath)} (took ${time}ms)`,
+      ),
+    );
   } catch (error) {
     console.error(
       c.red(

packages/api-gen/tests/generate.bench.ts

@@ -0,0 +1,32 @@
+import { describe, bench, vi } from "vitest";
+import { generate } from "../src/commands/generate";
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import testSchema from "./testSchema.json";
+
+vi.mock("node:fs", async () => {
+  return {
+    writeFileSync: vi.fn(),
+    existsSync: vi.fn(),
+    readFileSync: vi.fn(),
+  };
+});
+// TODO: mocking prettier for more acurate tests, but it should be replaced
+vi.mock("prettier", async () => {
+  return {
+    format: vi.fn().mockImplementation((content) => content),
+  };
+});
+vi.mocked(writeFileSync).mockReturnValue();
+vi.mocked(existsSync).mockReturnValue(true);
+vi.mocked(readFileSync).mockReturnValue(JSON.stringify(testSchema));
+const consoleWarnSpy = vi.spyOn(console, "log");
+consoleWarnSpy.mockImplementation(() => {});
+
+describe("api-gen - generate", () => {
+  bench("[api-gen][generate] - generate schema command", async () => {
+    await generate({
+      cwd: __dirname,
+      filename: "testSchema.json",
+    });
+  });
+});