diff --git a/.dockerignore b/.dockerignore index 7327cb2dc..e74b02409 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,5 +19,6 @@ !frontend/tsconfig.json !frontend/vite.config.js !frontend/index.html +!frontend/*.d.ts !frontend/src/**/* !frontend/public/**/* diff --git a/Dockerfile b/Dockerfile index 8a865a264..2923ee67f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,5 @@ # syntax=docker/dockerfile:1.5 -FROM scratch as ignore - -WORKDIR /listory -COPY . /listory/ - ################## ## common ################## diff --git a/docker-compose.yml b/docker-compose.yml index 5455bb4d6..fa89e43db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,8 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318 env_file: .env volumes: - - ./src:/app/src + - ./src:/app/src:ro + - ./dist:/app/dist # build cache ports: - 3000 # API - "9464:9464" # Metrics @@ -115,7 +116,8 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318 env_file: .env volumes: - - ./src:/app/src + - ./src:/app/src:ro + - ./dist:/app/dist # build cache ports: - "9464:9464" # Metrics networks: diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 461149044..0e2381143 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -38,6 +38,7 @@ "prettier": "3.0.3", "react": "18.2.0", "react-dom": "18.2.0", + "react-files": "3.0.0", "react-router-dom": "6.16.0", "recharts": "2.8.0", "tailwind-merge": "1.14.0", @@ -8143,6 +8144,15 @@ "react": "^18.2.0" } }, + "node_modules/react-files": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-files/-/react-files-3.0.0.tgz", + "integrity": "sha512-/Zz7S98vZFYxHw3RVSZcf3dD+xO714ZQd/jEhIp8q+MofBgydXWlHdw05TA4jradL7XpZFPvJaIvM6Z6I5nIHw==", + "peerDependencies": { + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -15363,6 +15373,12 @@ "scheduler": "^0.23.0" } }, + "react-files": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-files/-/react-files-3.0.0.tgz", + "integrity": "sha512-/Zz7S98vZFYxHw3RVSZcf3dD+xO714ZQd/jEhIp8q+MofBgydXWlHdw05TA4jradL7XpZFPvJaIvM6Z6I5nIHw==", + "requires": {} + }, "react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 37fb48f8f..6238ef9dd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -37,6 +37,7 @@ "prettier": "3.0.3", "react": "18.2.0", "react-dom": "18.2.0", + "react-files": "3.0.0", "react-router-dom": "6.16.0", "recharts": "2.8.0", "tailwind-merge": "1.14.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 8a4643887..1b4a9b842 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -2,6 +2,7 @@ import React from "react"; import { Route, Routes } from "react-router-dom"; import { AuthApiTokens } from "./components/AuthApiTokens"; import { Footer } from "./components/Footer"; +import { ImportListens } from "./components/ImportListens"; import { LoginFailure } from "./components/LoginFailure"; import { LoginLoading } from "./components/LoginLoading"; import { NavBar } from "./components/NavBar"; @@ -53,6 +54,7 @@ export function App() { element={} /> } /> + } /> )} {!user && ( diff --git a/frontend/src/api/api.ts b/frontend/src/api/api.ts index b2402d0fb..8e7de2e79 100644 --- a/frontend/src/api/api.ts +++ b/frontend/src/api/api.ts @@ -14,6 +14,8 @@ import { TopGenresItem } from "./entities/top-genres-item"; import { TopGenresOptions } from "./entities/top-genres-options"; import { TopTracksItem } from "./entities/top-tracks-item"; import { TopTracksOptions } from "./entities/top-tracks-options"; +import { SpotifyExtendedStreamingHistoryItem } from "./entities/spotify-extended-streaming-history-item"; +import { ExtendedStreamingHistoryStatus } from "./entities/extended-streaming-history-status"; export class UnauthenticatedError extends Error {} @@ -276,7 +278,7 @@ export const revokeApiToken = async ( id: string, client: AxiosInstance, ): Promise => { - const res = await client.delete(`/api/v1/auth/api-tokens/${id}`); + const res = await client.delete(`/api/v1/auth/api-tokens/${id}`); switch (res.status) { case 200: { @@ -290,3 +292,50 @@ export const revokeApiToken = async ( } } }; + +export const importExtendedStreamingHistory = async ( + listens: SpotifyExtendedStreamingHistoryItem[], + client: AxiosInstance +): Promise => { + const res = await client.post(`/api/v1/import/extended-streaming-history`, { + listens, + }); + + switch (res.status) { + case 201: { + break; + } + case 401: { + throw new UnauthenticatedError(`No token or token expired`); + } + default: { + throw new Error( + `Unable to importExtendedStreamingHistory: ${res.status}` + ); + } + } +}; + +export const getExtendedStreamingHistoryStatus = async ( + client: AxiosInstance +): Promise => { + const res = await client.get( + `/api/v1/import/extended-streaming-history/status` + ); + + switch (res.status) { + case 200: { + break; + } + case 401: { + throw new UnauthenticatedError(`No token or token expired`); + } + default: { + throw new Error( + `Unable to getExtendedStreamingHistoryStatus: ${res.status}` + ); + } + } + + return res.data; +}; diff --git a/frontend/src/api/entities/extended-streaming-history-status.ts b/frontend/src/api/entities/extended-streaming-history-status.ts new file mode 100644 index 000000000..d6fa488a6 --- /dev/null +++ b/frontend/src/api/entities/extended-streaming-history-status.ts @@ -0,0 +1,4 @@ +export interface ExtendedStreamingHistoryStatus { + total: number; + imported: number; +} diff --git a/frontend/src/api/entities/spotify-extended-streaming-history-item.ts b/frontend/src/api/entities/spotify-extended-streaming-history-item.ts new file mode 100644 index 000000000..cc91d3c41 --- /dev/null +++ b/frontend/src/api/entities/spotify-extended-streaming-history-item.ts @@ -0,0 +1,4 @@ +export interface SpotifyExtendedStreamingHistoryItem { + ts: string; + spotify_track_uri: string; +} diff --git a/frontend/src/components/AuthApiTokens.tsx b/frontend/src/components/AuthApiTokens.tsx index d73b7556c..c1b94fd04 100644 --- a/frontend/src/components/AuthApiTokens.tsx +++ b/frontend/src/components/AuthApiTokens.tsx @@ -2,7 +2,6 @@ import { format, formatDistanceToNow } from "date-fns"; import React, { FormEvent, useCallback, useMemo, useState } from "react"; import { ApiToken, NewApiToken } from "../api/entities/api-token"; import { useApiTokens } from "../hooks/use-api"; -import { useAuthProtection } from "../hooks/use-auth-protection"; import { SpinnerIcon } from "../icons/Spinner"; import TrashcanIcon from "../icons/Trashcan"; import { Spinner } from "./ui/Spinner"; diff --git a/frontend/src/components/ImportListens.tsx b/frontend/src/components/ImportListens.tsx new file mode 100644 index 000000000..f0699b6f3 --- /dev/null +++ b/frontend/src/components/ImportListens.tsx @@ -0,0 +1,280 @@ +import React, { useCallback, useEffect, useState } from "react"; +import Files from "react-files"; +import type { ReactFile } from "react-files"; +import { + useSpotifyImportExtendedStreamingHistory, + useSpotifyImportExtendedStreamingHistoryStatus, +} from "../hooks/use-api"; +import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item"; +import { ErrorIcon } from "../icons/Error"; +import { numberToPercent } from "../util/numberToPercent"; +import { Button } from "./ui/button"; +import { Table, TableBody, TableCell, TableRow } from "./ui/table"; +import { Badge } from "./ui/badge"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "./ui/card"; +import { Code } from "./ui/code"; + +export const ImportListens: React.FC = () => { + return ( + <> +
+

+ Import Listens from Spotify Extended Streaming History +

+
+
+

+ Here you can import your full Spotify Listen history that was exported + from the{" "} + + Extended streaming history + + . +

+

+ The extended streaming history contains additional personally + identifiable data such as the IP address of the listen (which can be + linked to locations). To avoid saving this on the server, the data is + preprocessed in your web browser and only the necessary data + (timestamp & track ID) are sent to the server. +

+

+ If an error occurs, you can always retry uploading the file, Listory + deduplicates any listens to make sure that everything is saved only + once. +

+ + + +
+ + ); +}; + +interface FileData { + file: ReactFile; + status: Status; + error?: Error; +} + +enum Status { + Select, + Import, + Finished, + Error, +} + +const FileUpload: React.FC = () => { + // Using a map is ... meh, need to wrap all state updates in `new Map()` so react re-renders + const [fileMap, setFileMap] = useState>( + new Map(), + ); + + const [status, setStatus] = useState(Status.Select); + + const addFiles = useCallback( + (files: ReactFile[]) => { + setFileMap((_fileMap) => { + files.forEach((file) => + _fileMap.set(file.id, { file, status: Status.Select }), + ); + return new Map(_fileMap); + }); + }, + [setFileMap], + ); + + const updateFile = useCallback((data: FileData) => { + setFileMap((_fileMap) => new Map(_fileMap.set(data.file.id, data))); + }, []); + + const clearFiles = useCallback(() => { + setFileMap(new Map()); + }, [setFileMap]); + + const { importHistory } = useSpotifyImportExtendedStreamingHistory(); + + const handleImport = useCallback(async () => { + setStatus(Status.Import); + + let errorOccurred = false; + + for (const data of fileMap.values()) { + data.status = Status.Import; + updateFile(data); + + let items: SpotifyExtendedStreamingHistoryItem[]; + + // Scope so these tmp variables can be GC-ed ASAP + { + const fileContent = await data.file.text(); + + const rawItems = JSON.parse( + fileContent, + ) as SpotifyExtendedStreamingHistoryItem[]; + + items = rawItems + .filter(({ spotify_track_uri }) => spotify_track_uri !== null) + .map(({ ts, spotify_track_uri }) => ({ + ts, + spotify_track_uri, + })); + } + + try { + await importHistory(items); + + data.status = Status.Finished; + } catch (err) { + data.error = err as Error; + data.status = Status.Error; + + errorOccurred = true; + } + updateFile(data); + } + + if (!errorOccurred) { + setStatus(Status.Finished); + } + }, [fileMap, importHistory, updateFile]); + + return ( + + + File Upload + + Select endsong_XY.json files here and start the import. + + + + + Drop files here or click to upload + + + + {Array.from(fileMap.values()).map((data) => ( + + ))} + +
+ + + + + + + ); +}; + +const File: React.FC<{ data: FileData }> = ({ data }) => { + const hasErrors = data.status === Status.Error && data.error; + + return ( + + {data.file.name} + + {data.file.sizeReadable} + + + {data.status === Status.Select && Prepared for import!} + {data.status === Status.Import && Loading!} + {data.status === Status.Finished && Check!} + {hasErrors && ( + + + {data.error?.message} + + )} + + + ); +}; + +const ImportProgress: React.FC = () => { + const { + importStatus: { total, imported }, + isLoading, + reload, + } = useSpotifyImportExtendedStreamingHistoryStatus(); + + useEffect(() => { + const interval = setInterval(() => { + if (!isLoading) { + reload(); + } + }, 1000); + + return () => clearInterval(interval); + }, [isLoading, reload]); + + return ( + + + Import Progress + + Shows how many of the submitted listens are already imported and + visible to you. This will take a while, and the process might halt for + a few minutes if we hit the Spotify API rate limit. If this is not + finished after a few hours, please contact your Listory administrator. + + + +
+
+
+ Imported +
+ {imported} +
+
+
+ Total +
+ {total} +
+
+ {total > 0 && ( +
+
+
+ )} + + + ); +}; diff --git a/frontend/src/components/NavBar.tsx b/frontend/src/components/NavBar.tsx index 6f8a738dc..1af0f3baa 100644 --- a/frontend/src/components/NavBar.tsx +++ b/frontend/src/components/NavBar.tsx @@ -3,6 +3,7 @@ import { Link } from "react-router-dom"; import { User } from "../api/entities/user"; import { useAuth } from "../hooks/use-auth"; import { CogwheelIcon } from "../icons/Cogwheel"; +import { ImportIcon } from "../icons/Import"; import { SpotifyLogo } from "../icons/Spotify"; import { Avatar, AvatarImage, AvatarFallback } from "./ui/avatar"; import { @@ -188,6 +189,12 @@ const NavUserInfo: React.FC<{ user: User }> = ({ user }) => { API Tokens + + + + Import Listens + + diff --git a/frontend/src/components/reports/RecentListens.tsx b/frontend/src/components/reports/RecentListens.tsx index 1b7669b07..860e413bf 100644 --- a/frontend/src/components/reports/RecentListens.tsx +++ b/frontend/src/components/reports/RecentListens.tsx @@ -50,7 +50,7 @@ export const RecentListens: React.FC = () => { )}
{recentListens.length > 0 && ( - +
{recentListens.map((listen) => ( diff --git a/frontend/src/components/reports/TopListItem.tsx b/frontend/src/components/reports/TopListItem.tsx index 9bccb8a0c..946d782ed 100644 --- a/frontend/src/components/reports/TopListItem.tsx +++ b/frontend/src/components/reports/TopListItem.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { numberToPercent } from "../../util/numberToPercent"; export interface TopListItemProps { title: string; @@ -42,9 +43,3 @@ export const TopListItem: React.FC = ({ const isMaxCountValid = (maxCount: number) => !(Number.isNaN(maxCount) || maxCount === 0); - -const numberToPercent = (ratio: number) => - ratio.toLocaleString(undefined, { - style: "percent", - minimumFractionDigits: 2, - }); diff --git a/frontend/src/components/ui/badge.tsx b/frontend/src/components/ui/badge.tsx new file mode 100644 index 000000000..f37c22bcf --- /dev/null +++ b/frontend/src/components/ui/badge.tsx @@ -0,0 +1,36 @@ +import * as React from "react"; +import { cva, type VariantProps } from "class-variance-authority"; + +import { cn } from "src/lib/utils"; + +const badgeVariants = cva( + "inline-flex items-center rounded-full border border-gray-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 dark:border-gray-800 dark:focus:ring-gray-300", + { + variants: { + variant: { + default: + "border-transparent bg-gray-900 text-gray-50 hover:bg-gray-900/80 dark:bg-gray-50 dark:text-gray-900 dark:hover:bg-gray-50/80", + secondary: + "border-transparent bg-gray-100 text-gray-900 hover:bg-gray-100/80 dark:bg-gray-800 dark:text-gray-50 dark:hover:bg-gray-800/80", + destructive: + "border-transparent bg-red-500 text-gray-50 hover:bg-red-500/80 dark:bg-red-900 dark:text-gray-50 dark:hover:bg-red-900/80", + outline: "text-gray-900 dark:text-gray-50", + }, + }, + defaultVariants: { + variant: "default", + }, + }, +); + +export interface BadgeProps + extends React.HTMLAttributes, + VariantProps {} + +function Badge({ className, variant, ...props }: BadgeProps) { + return ( +
+ ); +} + +export { Badge, badgeVariants }; diff --git a/frontend/src/components/ui/card.tsx b/frontend/src/components/ui/card.tsx new file mode 100644 index 000000000..d50d8d345 --- /dev/null +++ b/frontend/src/components/ui/card.tsx @@ -0,0 +1,86 @@ +import * as React from "react"; + +import { cn } from "src/lib/utils"; + +const Card = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)); +Card.displayName = "Card"; + +const CardHeader = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)); +CardHeader.displayName = "CardHeader"; + +const CardTitle = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +

+)); +CardTitle.displayName = "CardTitle"; + +const CardDescription = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +

+)); +CardDescription.displayName = "CardDescription"; + +const CardContent = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +

+)); +CardContent.displayName = "CardContent"; + +const CardFooter = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)); +CardFooter.displayName = "CardFooter"; + +export { + Card, + CardHeader, + CardFooter, + CardTitle, + CardDescription, + CardContent, +}; diff --git a/frontend/src/components/ui/code.tsx b/frontend/src/components/ui/code.tsx new file mode 100644 index 000000000..f98a7251f --- /dev/null +++ b/frontend/src/components/ui/code.tsx @@ -0,0 +1,9 @@ +import React, { PropsWithChildren } from "react"; + +export const Code: React.FC = ({ children }) => { + return ( + + {children} + + ); +}; diff --git a/frontend/src/components/ui/table.tsx b/frontend/src/components/ui/table.tsx index 2eed8a57a..a7efabf48 100644 --- a/frontend/src/components/ui/table.tsx +++ b/frontend/src/components/ui/table.tsx @@ -1,6 +1,6 @@ -import * as React from "react" +import * as React from "react"; -import { cn } from "src/lib/utils" +import { cn } from "src/lib/utils"; const Table = React.forwardRef< HTMLTableElement, @@ -9,20 +9,20 @@ const Table = React.forwardRef<

-)) -Table.displayName = "Table" +)); +Table.displayName = "Table"; const TableHeader = React.forwardRef< HTMLTableSectionElement, React.HTMLAttributes >(({ className, ...props }, ref) => ( -)) -TableHeader.displayName = "TableHeader" +)); +TableHeader.displayName = "TableHeader"; const TableBody = React.forwardRef< HTMLTableSectionElement, @@ -33,8 +33,8 @@ const TableBody = React.forwardRef< className={cn("[&_tr:last-child]:border-0", className)} {...props} /> -)) -TableBody.displayName = "TableBody" +)); +TableBody.displayName = "TableBody"; const TableFooter = React.forwardRef< HTMLTableSectionElement, @@ -42,11 +42,14 @@ const TableFooter = React.forwardRef< >(({ className, ...props }, ref) => ( -)) -TableFooter.displayName = "TableFooter" +)); +TableFooter.displayName = "TableFooter"; const TableRow = React.forwardRef< HTMLTableRowElement, @@ -56,12 +59,12 @@ const TableRow = React.forwardRef< ref={ref} className={cn( "border-b transition-colors hover:bg-gray-100/50 data-[state=selected]:bg-gray-100 dark:hover:bg-gray-800/50 dark:data-[state=selected]:bg-gray-800", - className + className, )} {...props} /> -)) -TableRow.displayName = "TableRow" +)); +TableRow.displayName = "TableRow"; const TableHead = React.forwardRef< HTMLTableCellElement, @@ -71,12 +74,12 @@ const TableHead = React.forwardRef< ref={ref} className={cn( "h-12 px-4 text-left align-middle font-medium text-gray-500 [&:has([role=checkbox])]:pr-0 dark:text-gray-400", - className + className, )} {...props} /> -)) -TableHead.displayName = "TableHead" +)); +TableHead.displayName = "TableHead"; const TableCell = React.forwardRef< HTMLTableCellElement, @@ -87,8 +90,8 @@ const TableCell = React.forwardRef< className={cn("p-4 align-middle [&:has([role=checkbox])]:pr-0", className)} {...props} /> -)) -TableCell.displayName = "TableCell" +)); +TableCell.displayName = "TableCell"; const TableCaption = React.forwardRef< HTMLTableCaptionElement, @@ -99,8 +102,8 @@ const TableCaption = React.forwardRef< className={cn("mt-4 text-sm text-gray-500 dark:text-gray-400", className)} {...props} /> -)) -TableCaption.displayName = "TableCaption" +)); +TableCaption.displayName = "TableCaption"; export { Table, @@ -111,4 +114,4 @@ export { TableRow, TableCell, TableCaption, -} +}; diff --git a/frontend/src/hooks/use-api.tsx b/frontend/src/hooks/use-api.tsx index 3e1bce2c7..70b4120f5 100644 --- a/frontend/src/hooks/use-api.tsx +++ b/frontend/src/hooks/use-api.tsx @@ -2,12 +2,14 @@ import { useCallback, useMemo } from "react"; import { createApiToken, getApiTokens, + getExtendedStreamingHistoryStatus, getListensReport, getRecentListens, getTopAlbums, getTopArtists, getTopGenres, getTopTracks, + importExtendedStreamingHistory, revokeApiToken, } from "../api/api"; import { ListenReportOptions } from "../api/entities/listen-report-options"; @@ -18,6 +20,7 @@ import { TopGenresOptions } from "../api/entities/top-genres-options"; import { TopTracksOptions } from "../api/entities/top-tracks-options"; import { useApiClient } from "./use-api-client"; import { useAsync } from "./use-async"; +import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item"; const INITIAL_EMPTY_ARRAY: [] = []; Object.freeze(INITIAL_EMPTY_ARRAY); @@ -143,9 +146,7 @@ export const useApiTokens = () => { const createToken = useCallback( async (description: string) => { const apiToken = await createApiToken(description, client); - console.log("apiToken created", apiToken); await reload(); - console.log("reloaded data"); return apiToken; }, @@ -162,3 +163,38 @@ export const useApiTokens = () => { return { apiTokens, isLoading, error, createToken, revokeToken }; }; + +export const useSpotifyImportExtendedStreamingHistory = () => { + const { client } = useApiClient(); + + const importHistory = useCallback( + async (listens: SpotifyExtendedStreamingHistoryItem[]) => { + return importExtendedStreamingHistory(listens, client); + }, + [client] + ); + + const getStatus = useCallback(async () => { + return getExtendedStreamingHistoryStatus(client); + }, [client]); + + return { importHistory, getStatus }; +}; + +export const useSpotifyImportExtendedStreamingHistoryStatus = () => { + const { client } = useApiClient(); + + const fetchData = useMemo( + () => () => getExtendedStreamingHistoryStatus(client), + [client] + ); + + const { + value: importStatus, + pending: isLoading, + error, + reload, + } = useAsync(fetchData, { total: 0, imported: 0 }); + + return { importStatus, isLoading, error, reload }; +}; diff --git a/frontend/src/hooks/use-auth-protection.tsx b/frontend/src/hooks/use-auth-protection.tsx deleted file mode 100644 index a30b075ee..000000000 --- a/frontend/src/hooks/use-auth-protection.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { useCallback } from "react"; -import { useNavigate } from "react-router-dom"; -import { useAuth } from "./use-auth"; - -export function useAuthProtection() { - const { user } = useAuth(); - const navigate = useNavigate(); - - const requireUser = useCallback(async () => { - if (!user) { - navigate("/"); - } - }, [user, navigate]); - - return { requireUser }; -} diff --git a/frontend/src/icons/Error.tsx b/frontend/src/icons/Error.tsx new file mode 100644 index 000000000..e28ccc25c --- /dev/null +++ b/frontend/src/icons/Error.tsx @@ -0,0 +1,14 @@ +import React from "react"; + +export const ErrorIcon: React.FC> = (props) => { + return ( + + + + ); +}; diff --git a/frontend/src/icons/Import.tsx b/frontend/src/icons/Import.tsx new file mode 100644 index 000000000..2abd1075d --- /dev/null +++ b/frontend/src/icons/Import.tsx @@ -0,0 +1,12 @@ +import * as React from "react"; +export const ImportIcon: React.FC> = (props) => ( + + + + +); diff --git a/frontend/src/react-files.d.ts b/frontend/src/react-files.d.ts new file mode 100644 index 000000000..77fbd22ee --- /dev/null +++ b/frontend/src/react-files.d.ts @@ -0,0 +1,36 @@ +//// + +declare module "react-files" { + declare const Files: React.FC< + Partial<{ + accepts: string[]; + children: React.ReactNode; + className: string; + clickable: boolean; + dragActiveClassName: string; + inputProps: unknown; + multiple: boolean; + maxFiles: number; + maxFileSize: number; + minFileSize: number; + name: string; + onChange: (files: ReactFile[]) => void; + onDragEnter: () => void; + onDragLeave: () => void; + onError: ( + error: { code: number; message: string }, + file: ReactFile + ) => void; + style: object; + }> + >; + + export type ReactFile = File & { + id: string; + extension: string; + sizeReadable: string; + preview: { type: "image"; url: string } | { type: "file" }; + }; + + export default Files; +} diff --git a/frontend/src/util/numberToPercent.ts b/frontend/src/util/numberToPercent.ts new file mode 100644 index 000000000..7373c2c31 --- /dev/null +++ b/frontend/src/util/numberToPercent.ts @@ -0,0 +1,5 @@ +export const numberToPercent = (ratio: number) => + ratio.toLocaleString(undefined, { + style: "percent", + minimumFractionDigits: 2, + }); diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 87a2e9c99..c21e75922 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -1,11 +1,7 @@ { "compilerOptions": { - "target": "es5", - "lib": [ - "dom", - "dom.iterable", - "esnext" - ], + "target": "ES2022", + "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "esModuleInterop": true, diff --git a/package-lock.json b/package-lock.json index dcb5914e7..c113fd49c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7082,9 +7082,9 @@ "dev": true }, "node_modules/globals": { - "version": "13.21.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.21.0.tgz", - "integrity": "sha512-ybyme3s4yy/t/3s35bewwXKOf7cvzfreG2lH0lZl0JB7I4GxRP2ghxOK/Nb9EkRXdbBXZLfq/p/0W2JUONB/Gg==", + "version": "13.22.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.22.0.tgz", + "integrity": "sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -9961,9 +9961,9 @@ } }, "node_modules/path-scurry/node_modules/minipass": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", - "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", "engines": { "node": ">=16 || 14 >=14.17" } @@ -10939,9 +10939,9 @@ } }, "node_modules/rimraf/node_modules/minipass": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", - "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", "engines": { "node": ">=16 || 14 >=14.17" } @@ -18278,9 +18278,9 @@ "dev": true }, "globals": { - "version": "13.21.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.21.0.tgz", - "integrity": "sha512-ybyme3s4yy/t/3s35bewwXKOf7cvzfreG2lH0lZl0JB7I4GxRP2ghxOK/Nb9EkRXdbBXZLfq/p/0W2JUONB/Gg==", + "version": "13.22.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.22.0.tgz", + "integrity": "sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -20388,9 +20388,9 @@ "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==" }, "minipass": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", - "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==" + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" } } }, @@ -21112,9 +21112,9 @@ } }, "minipass": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", - "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==" + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" } } }, diff --git a/src/auth/auth.controller.spec.ts b/src/auth/auth.controller.spec.ts index eaf9cc580..27c0ad75c 100644 --- a/src/auth/auth.controller.spec.ts +++ b/src/auth/auth.controller.spec.ts @@ -1,5 +1,5 @@ import { Test, TestingModule } from "@nestjs/testing"; -import type { Response } from "express"; +import type { Response as ExpressResponse } from "express"; import { User } from "../users/user.entity"; import { AuthSession } from "./auth-session.entity"; import { AuthController } from "./auth.controller"; @@ -27,7 +27,7 @@ describe("AuthController", () => { describe("spotifyCallback", () => { let user: User; - let res: Response; + let res: ExpressResponse; let refreshToken: string; beforeEach(() => { @@ -36,7 +36,7 @@ describe("AuthController", () => { statusCode: 200, cookie: jest.fn(), redirect: jest.fn(), - } as unknown as Response; + } as unknown as ExpressResponse; refreshToken = "REFRESH_TOKEN"; authService.createSession = jest.fn().mockResolvedValue({ refreshToken }); diff --git a/src/auth/auth.controller.ts b/src/auth/auth.controller.ts index 6db657868..1e4602a35 100644 --- a/src/auth/auth.controller.ts +++ b/src/auth/auth.controller.ts @@ -1,5 +1,5 @@ import { - Body, + Body as NestBody, Controller, Delete, Get, @@ -10,7 +10,7 @@ import { UseGuards, } from "@nestjs/common"; import { ApiBody, ApiTags } from "@nestjs/swagger"; -import type { Response } from "express"; +import type { Response as ExpressResponse } from "express"; import { User } from "../users/user.entity"; import { AuthSession } from "./auth-session.entity"; import { AuthService } from "./auth.service"; @@ -42,7 +42,7 @@ export class AuthController { @Get("spotify/callback") @UseFilters(SpotifyAuthFilter) @UseGuards(SpotifyAuthGuard) - async spotifyCallback(@ReqUser() user: User, @Res() res: Response) { + async spotifyCallback(@ReqUser() user: User, @Res() res: ExpressResponse) { const { refreshToken } = await this.authService.createSession(user); // Refresh token should not be accessible to frontend to reduce risk @@ -69,7 +69,7 @@ export class AuthController { @AuthAccessToken() async createApiToken( @ReqUser() user: User, - @Body("description") description: string, + @NestBody("description") description: string, ): Promise { const apiToken = await this.authService.createApiToken(user, description); diff --git a/src/auth/spotify.filter.ts b/src/auth/spotify.filter.ts index aeca9ce63..ec6502fe2 100644 --- a/src/auth/spotify.filter.ts +++ b/src/auth/spotify.filter.ts @@ -5,14 +5,14 @@ import { ForbiddenException, Logger, } from "@nestjs/common"; -import type { Response } from "express"; +import type { Response as ExpressResponse } from "express"; @Catch() export class SpotifyAuthFilter implements ExceptionFilter { private readonly logger = new Logger(this.constructor.name); catch(exception: Error, host: ArgumentsHost) { - const response = host.switchToHttp().getResponse(); + const response = host.switchToHttp().getResponse(); let reason = "unknown"; diff --git a/src/database/database.module.ts b/src/database/database.module.ts index 06ac5878a..e60493a74 100644 --- a/src/database/database.module.ts +++ b/src/database/database.module.ts @@ -27,7 +27,7 @@ export const DatabaseModule = TypeOrmModule.forRootAsync({ // Debug/Development Options // - // logging: true, + //logging: true, // // synchronize: true, // migrationsRun: false, diff --git a/src/database/migrations/09-CreateSpotifyImportTables.ts b/src/database/migrations/09-CreateSpotifyImportTables.ts new file mode 100644 index 000000000..1d967a313 --- /dev/null +++ b/src/database/migrations/09-CreateSpotifyImportTables.ts @@ -0,0 +1,68 @@ +import { + MigrationInterface, + QueryRunner, + Table, + TableIndex, + TableForeignKey, +} from "typeorm"; +import { TableColumnOptions } from "typeorm/schema-builder/options/TableColumnOptions"; + +const primaryUUIDColumn: TableColumnOptions = { + name: "id", + type: "uuid", + isPrimary: true, + isGenerated: true, + generationStrategy: "uuid", +}; + +export class CreateSpotifyImportTables0000000000009 + implements MigrationInterface +{ + async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable( + new Table({ + name: "spotify_extended_streaming_history_listen", + columns: [ + primaryUUIDColumn, + { name: "userId", type: "uuid" }, + { name: "playedAt", type: "timestamp" }, + { name: "spotifyTrackUri", type: "varchar" }, + { name: "trackId", type: "uuid", isNullable: true }, + { name: "listenId", type: "uuid", isNullable: true }, + ], + indices: [ + new TableIndex({ + name: "IDX_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_PLAYED_AT", + columnNames: ["userId", "playedAt", "spotifyTrackUri"], + isUnique: true, + }), + ], + foreignKeys: [ + new TableForeignKey({ + name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_ID", + columnNames: ["userId"], + referencedColumnNames: ["id"], + referencedTableName: "user", + }), + new TableForeignKey({ + name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_TRACK_ID", + columnNames: ["trackId"], + referencedColumnNames: ["id"], + referencedTableName: "track", + }), + new TableForeignKey({ + name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_LISTEN_ID", + columnNames: ["listenId"], + referencedColumnNames: ["id"], + referencedTableName: "listen", + }), + ], + }), + true, + ); + } + + async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable("spotify_extended_streaming_history_listen"); + } +} diff --git a/src/listens/listen.repository.ts b/src/listens/listen.repository.ts index 22c22d4df..f12d8fa54 100644 --- a/src/listens/listen.repository.ts +++ b/src/listens/listen.repository.ts @@ -3,10 +3,6 @@ import { Repository, SelectQueryBuilder } from "typeorm"; import { EntityRepository } from "../database/entity-repository"; import { Interval } from "../reports/interval"; import { User } from "../users/user.entity"; -import { - CreateListenRequestDto, - CreateListenResponseDto, -} from "./dto/create-listen.dto"; import { Listen } from "./listen.entity"; export class ListenScopes extends SelectQueryBuilder { @@ -37,52 +33,4 @@ export class ListenRepository extends Repository { get scoped(): ListenScopes { return new ListenScopes(this.createQueryBuilder("listen")); } - - async insertNoConflict({ - user, - track, - playedAt, - }: CreateListenRequestDto): Promise { - const result = await this.createQueryBuilder() - .insert() - .values({ - user, - track, - playedAt, - }) - .onConflict('("playedAt", "trackId", "userId") DO NOTHING') - .execute(); - - const [insertedRowIdentifier] = result.identifiers; - - if (!insertedRowIdentifier) { - // We did not insert a new listen, it already existed - return { - listen: await this.findOneBy({ user, track, playedAt }), - isDuplicate: true, - }; - } - - return { - listen: await this.findOneBy({ id: insertedRowIdentifier.id }), - isDuplicate: false, - }; - } - - /** - * - * @param rows - * @returns A list of all new (non-duplicate) listens - */ - async insertsNoConflict(rows: CreateListenRequestDto[]): Promise { - const result = await this.createQueryBuilder() - .insert() - .values(rows) - .orIgnore() - .execute(); - - return this.findBy( - result.identifiers.filter(Boolean).map(({ id }) => ({ id })), - ); - } } diff --git a/src/listens/listens.service.spec.ts b/src/listens/listens.service.spec.ts index a943d6f50..db3505447 100644 --- a/src/listens/listens.service.spec.ts +++ b/src/listens/listens.service.spec.ts @@ -4,9 +4,7 @@ import { paginate, PaginationTypeEnum, } from "nestjs-typeorm-paginate"; -import { Track } from "../music-library/track.entity"; import { User } from "../users/user.entity"; -import { CreateListenResponseDto } from "./dto/create-listen.dto"; import { GetListensDto } from "./dto/get-listens.dto"; import { Listen } from "./listen.entity"; import { ListenRepository, ListenScopes } from "./listen.repository"; @@ -35,39 +33,6 @@ describe("ListensService", () => { expect(listenRepository).toBeDefined(); }); - describe("createListen", () => { - let user: User; - let track: Track; - let playedAt: Date; - let response: CreateListenResponseDto; - beforeEach(() => { - user = { id: "USER" } as User; - track = { id: "TRACK" } as Track; - playedAt = new Date("2021-01-01T00:00:00Z"); - - response = { - listen: { - id: "LISTEN", - } as Listen, - isDuplicate: true, - }; - listenRepository.insertNoConflict = jest.fn().mockResolvedValue(response); - }); - - it("creates the listen", async () => { - await expect( - service.createListen({ user, track, playedAt }), - ).resolves.toEqual(response); - - expect(listenRepository.insertNoConflict).toHaveBeenCalledTimes(1); - expect(listenRepository.insertNoConflict).toHaveBeenLastCalledWith({ - user, - track, - playedAt, - }); - }); - }); - describe("getListens", () => { let options: GetListensDto & IPaginationOptions; let user: User; diff --git a/src/listens/listens.service.ts b/src/listens/listens.service.ts index fea7e905e..0e144ef69 100644 --- a/src/listens/listens.service.ts +++ b/src/listens/listens.service.ts @@ -1,14 +1,12 @@ import { Injectable } from "@nestjs/common"; +import { Span } from "nestjs-otel"; import { IPaginationOptions, paginate, Pagination, PaginationTypeEnum, } from "nestjs-typeorm-paginate"; -import { - CreateListenRequestDto, - CreateListenResponseDto, -} from "./dto/create-listen.dto"; +import { CreateListenRequestDto } from "./dto/create-listen.dto"; import { GetListensDto } from "./dto/get-listens.dto"; import { Listen } from "./listen.entity"; import { ListenRepository, ListenScopes } from "./listen.repository"; @@ -17,20 +15,7 @@ import { ListenRepository, ListenScopes } from "./listen.repository"; export class ListensService { constructor(private readonly listenRepository: ListenRepository) {} - async createListen({ - user, - track, - playedAt, - }: CreateListenRequestDto): Promise { - const response = await this.listenRepository.insertNoConflict({ - user, - track, - playedAt, - }); - - return response; - } - + @Span() async createListens( listensData: CreateListenRequestDto[], ): Promise { @@ -46,9 +31,11 @@ export class ListensService { ), ); - return this.listenRepository.save( + const newListens = await this.listenRepository.save( missingListens.map((entry) => this.listenRepository.create(entry)), ); + + return [...existingListens, ...newListens]; } async getListens( diff --git a/src/main.ts b/src/main.ts index 5e4720535..2524d67b0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -43,6 +43,7 @@ async function bootstrap() { const app = await NestFactory.create(AppModule, { bufferLogs: true, + rawBody: true, }); app.useLogger(app.get(Logger)); app.useGlobalPipes( @@ -51,6 +52,10 @@ async function bootstrap() { transformOptions: { enableImplicitConversion: true }, }), ); + app.useBodyParser("json", { + limit: + "10mb" /* Need large bodies for Spotify Extended Streaming History */, + }); app.enableShutdownHooks(); const configService = app.get(ConfigService); diff --git a/src/music-library/dto/find-track.dto.ts b/src/music-library/dto/find-track.dto.ts index 7ffddc5fa..bfdf4efec 100644 --- a/src/music-library/dto/find-track.dto.ts +++ b/src/music-library/dto/find-track.dto.ts @@ -1,5 +1,6 @@ export class FindTrackDto { spotify: { - id: string; + id?: string; + uri?: string; }; } diff --git a/src/music-library/music-library.service.ts b/src/music-library/music-library.service.ts index 58040d6b9..15ce5b50a 100644 --- a/src/music-library/music-library.service.ts +++ b/src/music-library/music-library.service.ts @@ -175,9 +175,7 @@ export class MusicLibraryService { } async findTrack(query: FindTrackDto): Promise { - return this.trackRepository.findOneBy({ - spotify: { id: query.spotify.id }, - }); + return this.trackRepository.findOneBy(query); } async findTracks(query: FindTrackDto[]): Promise { diff --git a/src/override.d.ts b/src/override.d.ts deleted file mode 100644 index f8f520966..000000000 --- a/src/override.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Issue with opentelemetry-js: https://github.com/open-telemetry/opentelemetry-js/issues/3580#issuecomment-1701157270 -export {}; -declare global { - type BlobPropertyBag = unknown; -} diff --git a/src/sources/scheduler.service.ts b/src/sources/scheduler.service.ts index 041395b4c..a6f17d533 100644 --- a/src/sources/scheduler.service.ts +++ b/src/sources/scheduler.service.ts @@ -33,11 +33,11 @@ export class SchedulerService implements OnApplicationBootstrap { } private async setupSpotifyCrawlerSupervisor(): Promise { - await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {}); + // await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {}); } @Span() - @CrawlerSupervisorJob.Handle() + // @CrawlerSupervisorJob.Handle() async superviseImportJobs(): Promise { this.logger.log("Starting crawler jobs"); const userInfo = await this.spotifyService.getCrawlableUserInfo(); diff --git a/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts new file mode 100644 index 000000000..43b67d7af --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts @@ -0,0 +1,13 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class ExtendedStreamingHistoryStatusDto { + @ApiProperty({ + type: Number, + }) + total: number; + + @ApiProperty({ + type: Number, + }) + imported: number; +} diff --git a/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts new file mode 100644 index 000000000..419a2469c --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts @@ -0,0 +1,13 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { ArrayMaxSize } from "class-validator"; +import { SpotifyExtendedStreamingHistoryItemDto } from "./spotify-extended-streaming-history-item.dto"; + +export class ImportExtendedStreamingHistoryDto { + @ApiProperty({ + type: SpotifyExtendedStreamingHistoryItemDto, + isArray: true, + maxItems: 50_000, + }) + @ArrayMaxSize(50_000) // File size is ~16k by default, might need refactoring if Spotify starts exporting larger files + listens: SpotifyExtendedStreamingHistoryItemDto[]; +} diff --git a/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts new file mode 100644 index 000000000..ed7f33385 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class SpotifyExtendedStreamingHistoryItemDto { + @ApiProperty({ format: "iso8601", example: "2018-11-30T08:33:33Z" }) + ts: string; + + @ApiProperty({ example: "spotify:track:6askbS4pEVWbbDnUGEXh3G" }) + spotify_track_uri: string; +} diff --git a/src/sources/spotify/import-extended-streaming-history/import.controller.ts b/src/sources/spotify/import-extended-streaming-history/import.controller.ts new file mode 100644 index 000000000..b6764a515 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/import.controller.ts @@ -0,0 +1,32 @@ +import { Body as NestBody, Controller, Get, Post } from "@nestjs/common"; +import { ApiBody, ApiTags } from "@nestjs/swagger"; +import { AuthAccessToken } from "../../../auth/decorators/auth-access-token.decorator"; +import { ReqUser } from "../../../auth/decorators/req-user.decorator"; +import { User } from "../../../users/user.entity"; +import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto"; +import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto"; +import { ImportService } from "./import.service"; + +@ApiTags("import") +@Controller("api/v1/import") +export class ImportController { + constructor(private readonly importService: ImportService) {} + + @Post("extended-streaming-history") + @ApiBody({ type: () => ImportExtendedStreamingHistoryDto }) + @AuthAccessToken() + async importExtendedStreamingHistory( + @ReqUser() user: User, + @NestBody() data: ImportExtendedStreamingHistoryDto, + ): Promise { + return this.importService.importExtendedStreamingHistory(user, data); + } + + @Get("extended-streaming-history/status") + @AuthAccessToken() + async getExtendedStreamingHistoryStatus( + @ReqUser() user: User, + ): Promise { + return this.importService.getExtendedStreamingHistoryStatus(user); + } +} diff --git a/src/sources/spotify/import-extended-streaming-history/import.service.ts b/src/sources/spotify/import-extended-streaming-history/import.service.ts new file mode 100644 index 000000000..d26fe9956 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/import.service.ts @@ -0,0 +1,177 @@ +import { JobService } from "@apricote/nest-pg-boss"; +import { Injectable, Logger } from "@nestjs/common"; +import { uniq } from "lodash"; +import { Span } from "nestjs-otel"; +import type { Job } from "pg-boss"; +import { ListensService } from "../../../listens/listens.service"; +import { User } from "../../../users/user.entity"; +import { SpotifyService } from "../spotify.service"; +import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto"; +import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto"; +import { + IProcessSpotifyExtendedStreamingHistoryListenJob, + ProcessSpotifyExtendedStreamingHistoryListenJob, +} from "./jobs"; +import { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository"; + +@Injectable() +export class ImportService { + private readonly logger = new Logger(this.constructor.name); + + constructor( + private readonly importListenRepository: SpotifyExtendedStreamingHistoryListenRepository, + @ProcessSpotifyExtendedStreamingHistoryListenJob.Inject() + private readonly processListenJobService: JobService, + private readonly spotifyService: SpotifyService, + private readonly listensService: ListensService, + ) {} + + @Span() + async importExtendedStreamingHistory( + user: User, + { listens: importListens }: ImportExtendedStreamingHistoryDto, + ): Promise { + // IDK what's happening, but my personal data set has entries with duplicate + // listens? might be related to offline mode. + // Anyway, this cleans it up: + const uniqEntries = new Set(); + const uniqueListens = importListens.filter((listen) => { + const key = `${listen.spotify_track_uri}-${listen.ts}`; + + if (!uniqEntries.has(key)) { + // New entry + uniqEntries.add(key); + return true; + } + + return false; + }); + + let listens = uniqueListens.map((listenData) => + this.importListenRepository.create({ + user, + playedAt: new Date(listenData.ts), + spotifyTrackUri: listenData.spotify_track_uri, + }), + ); + + // Save listens to import table + const insertResult = await this.importListenRepository.upsert(listens, [ + "user", + "playedAt", + "spotifyTrackUri", + ]); + + const processJobs = insertResult.identifiers.map((listen) => ({ + data: { + id: listen.id, + }, + singletonKey: listen.id, + retryLimit: 10, + retryDelay: 5, + retryBackoff: true, + })); + + // Schedule jobs to process imports + await this.processListenJobService.insert(processJobs); + } + + @ProcessSpotifyExtendedStreamingHistoryListenJob.Handle({ + // Spotify API "Get Several XY" allows max 50 IDs + batchSize: 50, + newJobCheckInterval: 500, + }) + @Span() + async processListens( + jobs: Job[], + ): Promise { + this.logger.debug( + { jobs: jobs.length }, + "processing extended streaming history listens", + ); + const importListens = await this.importListenRepository.findBy( + jobs.map((job) => ({ id: job.data.id })), + ); + + const listensWithoutTracks = importListens.filter( + (importListen) => !importListen.track, + ); + if (listensWithoutTracks.length > 0) { + const missingTrackIDs = uniq( + listensWithoutTracks.map((importListen) => + importListen.spotifyTrackUri.replace("spotify:track:", ""), + ), + ); + + const tracks = await this.spotifyService.importTracks(missingTrackIDs); + + listensWithoutTracks.forEach((listen) => { + listen.track = tracks.find( + (track) => listen.spotifyTrackUri === track.spotify.uri, + ); + if (!listen.track) { + this.logger.warn( + { listen }, + "could not find track for extended streaming history listen", + ); + throw new Error( + `could not find track for extended streaming history listen`, + ); + } + }); + + // Using upsert instead of save to only do a single query + await this.importListenRepository.upsert(listensWithoutTracks, ["id"]); + } + + const listensWithoutListen = importListens.filter( + (importListen) => !importListen.listen, + ); + if (listensWithoutListen.length > 0) { + const listens = await this.listensService.createListens( + listensWithoutListen.map((listen) => ({ + user: listen.user, + track: listen.track, + playedAt: listen.playedAt, + })), + ); + + listensWithoutListen.forEach((importListen) => { + importListen.listen = listens.find( + (listen) => + importListen.user.id === listen.user.id && + importListen.track.id === listen.track.id && + importListen.playedAt.getTime() === listen.playedAt.getTime(), + ); + if (!importListen.listen) { + this.logger.warn( + { listen: importListen, listens: listens }, + "could not find listen for extended streaming history listen", + ); + throw new Error( + `could not find listen for extended streaming history listen`, + ); + } + }); + + // Using upsert instead of save to only do a single query + await this.importListenRepository.upsert(listensWithoutListen, ["id"]); + } + } + + @Span() + async getExtendedStreamingHistoryStatus( + user: User, + ): Promise { + const qb = this.importListenRepository + .createQueryBuilder("listen") + .where("listen.userId = :user", { user: user.id }); + + const [total, imported] = await Promise.all([ + qb.clone().getCount(), + qb.clone().andWhere("listen.listenId IS NOT NULL").getCount(), + ]); + + return { total, imported }; + } +} diff --git a/src/sources/spotify/import-extended-streaming-history/index.ts b/src/sources/spotify/import-extended-streaming-history/index.ts new file mode 100644 index 000000000..b5db823f1 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/index.ts @@ -0,0 +1,4 @@ +export { ImportController } from "./import.controller"; +export { ImportService } from "./import.service"; +export { ProcessSpotifyExtendedStreamingHistoryListenJob } from "./jobs"; +export { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository"; diff --git a/src/sources/spotify/import-extended-streaming-history/jobs.ts b/src/sources/spotify/import-extended-streaming-history/jobs.ts new file mode 100644 index 000000000..38c55c809 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/jobs.ts @@ -0,0 +1,7 @@ +import { createJob } from "@apricote/nest-pg-boss"; + +export type IProcessSpotifyExtendedStreamingHistoryListenJob = { id: string }; +export const ProcessSpotifyExtendedStreamingHistoryListenJob = + createJob( + "process-spotify-extended-streaming-history-listen", + ); diff --git a/src/sources/spotify/import-extended-streaming-history/listen.entity.ts b/src/sources/spotify/import-extended-streaming-history/listen.entity.ts new file mode 100644 index 000000000..4fa898287 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/listen.entity.ts @@ -0,0 +1,25 @@ +import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm"; +import { Track } from "../../../music-library/track.entity"; +import { User } from "../../../users/user.entity"; +import { Listen } from "../../../listens/listen.entity"; + +@Entity({ name: "spotify_extended_streaming_history_listen" }) +export class SpotifyExtendedStreamingHistoryListen { + @PrimaryGeneratedColumn("uuid") + id: string; + + @ManyToOne(() => User, { eager: true }) + user: User; + + @Column({ type: "timestamp" }) + playedAt: Date; + + @Column() + spotifyTrackUri: string; + + @ManyToOne(() => Track, { nullable: true, eager: true }) + track?: Track; + + @ManyToOne(() => Listen, { nullable: true, eager: true }) + listen?: Listen; +} diff --git a/src/sources/spotify/import-extended-streaming-history/listen.repository.ts b/src/sources/spotify/import-extended-streaming-history/listen.repository.ts new file mode 100644 index 000000000..7c40359a4 --- /dev/null +++ b/src/sources/spotify/import-extended-streaming-history/listen.repository.ts @@ -0,0 +1,6 @@ +import { Repository } from "typeorm"; +import { EntityRepository } from "../../../database/entity-repository"; +import { SpotifyExtendedStreamingHistoryListen } from "./listen.entity"; + +@EntityRepository(SpotifyExtendedStreamingHistoryListen) +export class SpotifyExtendedStreamingHistoryListenRepository extends Repository {} diff --git a/src/sources/spotify/spotify.module.ts b/src/sources/spotify/spotify.module.ts index 1df6bbf32..021ffeca6 100644 --- a/src/sources/spotify/spotify.module.ts +++ b/src/sources/spotify/spotify.module.ts @@ -1,25 +1,37 @@ import { PGBossModule } from "@apricote/nest-pg-boss"; import { Module } from "@nestjs/common"; +import { TypeOrmRepositoryModule } from "../../database/entity-repository/typeorm-repository.module"; import { ListensModule } from "../../listens/listens.module"; import { MusicLibraryModule } from "../../music-library/music-library.module"; import { UsersModule } from "../../users/users.module"; import { ImportSpotifyJob } from "../jobs"; +import { + ImportController, + ImportService, + ProcessSpotifyExtendedStreamingHistoryListenJob, + SpotifyExtendedStreamingHistoryListenRepository, +} from "./import-extended-streaming-history"; import { SpotifyApiModule } from "./spotify-api/spotify-api.module"; import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module"; import { SpotifyService } from "./spotify.service"; @Module({ imports: [ - PGBossModule.forJobs([ImportSpotifyJob]), + PGBossModule.forJobs([ + ImportSpotifyJob, + ProcessSpotifyExtendedStreamingHistoryListenJob, + ]), + TypeOrmRepositoryModule.for([ + SpotifyExtendedStreamingHistoryListenRepository, + ]), UsersModule, ListensModule, MusicLibraryModule, SpotifyApiModule, SpotifyAuthModule, ], - providers: [SpotifyService], + providers: [SpotifyService, ImportService], + controllers: [ImportController], exports: [SpotifyService], }) -export class SpotifyModule { - constructor(private readonly spotifyService: SpotifyService) {} -} +export class SpotifyModule {} diff --git a/tsconfig.json b/tsconfig.json index cce4f57fc..78c93822a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,13 +5,12 @@ "removeComments": true, "emitDecoratorMetadata": true, "experimentalDecorators": true, - "target": "ES2020", + "target": "ES2022", "sourceMap": true, "outDir": "./dist", "baseUrl": "./", "incremental": true, "paths": {}, - "lib": ["ES2020"] }, "exclude": ["node_modules", "dist"], "include": ["src", "test"]