diff --git a/.run/MLib [Sort pom].run.xml b/.run/MLib [Sort pom].run.xml new file mode 100644 index 00000000..81fb3f36 --- /dev/null +++ b/.run/MLib [Sort pom].run.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index d5c5f4e0..86068b16 100644 --- a/pom.xml +++ b/pom.xml @@ -1,6 +1,13 @@ 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.6.3 + + + de.mediathekview MLib @@ -69,62 +76,20 @@ 17 17 - 3.21.0 1.21 1.9 - 2.8.9 - 2.0.1 - 2.4.0-b180830.0359 22.0.0 - 3.0.3 - 5.7.2 - 2.17.1 - 3.1.0 - 3.8.1 - 2.8.2 - 1.6 - 2.5.2 - 3.1.1 - 3.2.0 - 3.0.0 - 3.2.0 - 3.2.1 - 2.22.2 - 3.8.2 - 4.2.0 - 1.6.8 - 4.9.3 - 2.27.2 + 2.32.0 0.9.2 1.9 + 1.4.2.Final + 0.2.0 + 1.6.8 + 3.0.1 + 3.8.2 + 3.0.0 - - - - org.glassfish.jersey - jersey-bom - ${jersey.version} - pom - import - - - org.apache.logging.log4j - log4j-bom - ${log4j2.version} - pom - import - - - org.junit - junit-bom - ${junit-jupiter.version} - pom - import - - - - org.apache.commons @@ -185,10 +150,18 @@ + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-slf4j-impl + @@ -235,11 +208,22 @@ com.github.tomakehurst - wiremock + wiremock-jre8 ${wiremock.version} test + + org.mapstruct + mapstruct + ${mapstruct.version} + + + org.projectlombok + lombok + true + + @@ -305,24 +289,28 @@ + org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} - - - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - maven-jar-plugin - ${maven-jar-plugin.version} - - - maven-install-plugin - ${maven-install-plugin.version} - - - maven-deploy-plugin - ${maven-deploy-plugin.version} + + + + org.mapstruct + mapstruct-processor + ${mapstruct.version} + + + org.projectlombok + lombok + ${lombok.version} + + + org.projectlombok + lombok-mapstruct-binding + ${lombok-mapstruct-binding.version} + + + @@ -334,7 +322,6 @@ org.apache.maven.plugins maven-javadoc-plugin - ${maven-javadoc-plugin.version} private true @@ -352,7 +339,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - ${nexus-staging-maven-plugin.version} + 1.6.8 true ossrh @@ -363,7 +350,7 @@ org.apache.maven.plugins maven-gpg-plugin - ${maven-gpg-plugin.version} + 3.0.1 sign-artifacts diff --git a/src/main/java/de/mediathekview/mlib/daten/Film.java b/src/main/java/de/mediathekview/mlib/daten/Film.java index e9f9c119..bfdc5148 100644 --- a/src/main/java/de/mediathekview/mlib/daten/Film.java +++ b/src/main/java/de/mediathekview/mlib/daten/Film.java @@ -1,5 +1,6 @@ package de.mediathekview.mlib.daten; +import java.io.Serial; import java.net.URL; import java.time.Duration; import java.time.LocalDateTime; @@ -8,11 +9,11 @@ /** Represents a found film. */ public class Film extends Podcast { + @Serial private static final long serialVersionUID = -7834270191129532291L; - private final Set subtitles; + private Set subtitles; private Map audioDescriptions; private Map signLanguages; - public Film( final UUID aUuid, final Sender aSender, @@ -119,6 +120,10 @@ public Collection getSubtitles() { return new ArrayList<>(subtitles); } + public void setSubtitles(Set subtitles) { + this.subtitles = subtitles; + } + @Override public String toString() { return "Film{" diff --git a/src/main/java/de/mediathekview/mlib/daten/GeoLocations.java b/src/main/java/de/mediathekview/mlib/daten/GeoLocations.java index f701541a..a4623803 100644 --- a/src/main/java/de/mediathekview/mlib/daten/GeoLocations.java +++ b/src/main/java/de/mediathekview/mlib/daten/GeoLocations.java @@ -17,7 +17,7 @@ public enum GeoLocations { GEO_DE_AT_CH_EU("DE-AT-CH-EU", "SAT", "EBU"); private final String description; - private String[] alternatives; + private final String[] alternatives; GeoLocations(final String aDescription, final String... aAlternatives) { description = aDescription; @@ -47,4 +47,8 @@ public static GeoLocations getFromDescription(final String aDescription) { public String getDescription() { return description; } + + public String[] getAlternatives() { + return alternatives; + } } diff --git a/src/main/java/de/mediathekview/mlib/daten/Sender.java b/src/main/java/de/mediathekview/mlib/daten/Sender.java index d9617afe..1b348fa7 100644 --- a/src/main/java/de/mediathekview/mlib/daten/Sender.java +++ b/src/main/java/de/mediathekview/mlib/daten/Sender.java @@ -84,4 +84,8 @@ public String getName() { public String toString() { return getName(); } + + public String[] getNameAlternatives() { + return nameAlternatives; + } } diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/CantReadFilmException.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/CantReadFilmException.java deleted file mode 100644 index 45653f5e..00000000 --- a/src/main/java/de/mediathekview/mlib/filmlisten/reader/CantReadFilmException.java +++ /dev/null @@ -1,9 +0,0 @@ -package de.mediathekview.mlib.filmlisten.reader; - -public class CantReadFilmException extends Exception { - private static final long serialVersionUID = -1839526789842123501L; - - public CantReadFilmException(final String aExceptionText) { - super(aExceptionText); - } -} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java index 31ad3fed..229ac446 100644 --- a/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java @@ -2,173 +2,37 @@ import de.mediathekview.mlib.daten.Film; import de.mediathekview.mlib.daten.Filmlist; -import de.mediathekview.mlib.tool.TextCleaner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static java.time.format.FormatStyle.MEDIUM; -import static java.time.format.FormatStyle.SHORT; +import java.util.Optional; public class FilmlistOldFormatReader extends AbstractFilmlistReader { - private static final String ENTRY_DELIMETER = "\\],"; private static final Logger LOG = LogManager.getLogger(FilmlistOldFormatReader.class); - private static final DateTimeFormatter DATE_FORMATTER = - DateTimeFormatter.ofLocalizedDate(MEDIUM).withLocale(Locale.GERMANY); - private static final DateTimeFormatter TIME_FORMATTER = - DateTimeFormatter.ofLocalizedTime(SHORT).withLocale(Locale.GERMANY); - private static final String ENTRY_PATTERN = "\"\\w*\"\\s?:\\s*\\[\\s?(\"([^\"]|\\\")*\",?\\s?)*"; - private static final String ENTRY_SPLIT_PATTERN = "\"(\\\\\"|[^\"])*\""; - private static final String FILM_ENTRY_ID = "X"; - private static final String DATE_TIME_SPLITTERATOR = ",?\\s+"; - private static final String QUOTATION_MARK = "\""; @Override - public Optional read(final InputStream aInputStream) { - try (final Scanner scanner = new Scanner(aInputStream, StandardCharsets.UTF_8.name()); - final Scanner entryScanner = scanner.useDelimiter(ENTRY_DELIMETER)) { - return convertEntriesToFilms(findEntries(entryScanner)); - } finally { - try { - aInputStream.close(); - } catch (final IOException exception) { - LOG.debug("Can't close the ioStream", exception); - } - } - } - - @NotNull - private Optional convertEntriesToFilms(final List entries) { - final Filmlist filmlist = new Filmlist(); - final List> futureFilms = asyncConvertEntriesToFilms(entries, filmlist); - futureFilms.stream() - .map( - filmFuture -> { - try { - return filmFuture.get(); - } catch (final InterruptedException interruptedException) { - LOG.debug( - "Some error occured during converting a old film list entry to an film.", - interruptedException); - Thread.currentThread().interrupt(); - } catch (final Exception exception) { - LOG.debug( - "Some error occured during converting a old film list entry to an film.", - exception); - } - return null; - }) - .filter(Objects::nonNull) - .filter(film -> !film.getUrls().isEmpty()) - .forEach(filmlist::add); + public Optional read(final InputStream filmlistInputStream) { + OldFilmlistToRawFilmlistReader oldFilmlistToRawFilmlistReader = + new OldFilmlistToRawFilmlistReader(filmlistInputStream); + RawFilmlist rawFilmlist = oldFilmlistToRawFilmlistReader.read(); + + Filmlist filmlist = new Filmlist(rawFilmlist.getListId(), rawFilmlist.getCreationDate()); + filmlist.addAllFilms( + rawFilmlist.getRawFilms().parallelStream() + .map(this::mapRawFilmToFilm) + .filter(Optional::isPresent) + .map(Optional::get) + .toList()); return Optional.of(filmlist); } - @NotNull - private List> asyncConvertEntriesToFilms( - final List entries, final Filmlist filmlist) { - final ExecutorService executorService = Executors.newWorkStealingPool(); - boolean isFirst = true; - Future filmEntryBefore = null; - final List> futureFilms = new ArrayList<>(); - - final List> splittedEntries = - entries.stream() - .map(this::splittEntry) - .filter(splittEntry -> !splittEntry.isEmpty()) - .toList(); - - for (final List splittedEntry : splittedEntries) { - if (isFirst) { - setMetaInfo(filmlist, splittedEntry); - isFirst = false; - } else if (splittedEntry.size() == 21 && FILM_ENTRY_ID.equals(splittedEntry.get(0))) { - filmEntryBefore = - convertEntryToFilm(filmEntryBefore, executorService, futureFilms, splittedEntry); - } - } - return futureFilms; - } - - private Future convertEntryToFilm( - Future filmEntryBefore, - final ExecutorService executorService, - final List> futureFilms, - final List splittedEntry) { + private Optional mapRawFilmToFilm(RawFilm rawFilm) { try { - final Future newEntry = - executorService.submit(new OldFilmlistEntryToFilmTask(splittedEntry, filmEntryBefore)); - futureFilms.add(newEntry); - filmEntryBefore = newEntry; - } catch (final Exception exception) { - LOG.debug( - String.format("Error on converting the following text to a film:%n %s ", splittedEntry)); + return Optional.of(RawFilmToFilmMapper.INSTANCE.rawFilmToFilm(rawFilm)); + } catch (RawFilmToFilmException rawFilmToFilmException) { + LOG.error("Skipping a film with invalid data.", rawFilmToFilmException); + return Optional.empty(); } - return filmEntryBefore; - } - - private List findEntries(final Scanner entryScanner) { - final List entries = new ArrayList<>(); - - while (entryScanner.hasNext()) { - final String entry = entryScanner.next(); - final Matcher entryMatcher = Pattern.compile(ENTRY_PATTERN).matcher(entry); - if (entryMatcher.find()) { - entries.add(entryMatcher.group()); - } - } - return entries; - } - - private void setMetaInfo(final Filmlist aFilmlist, final List aSplittedEntry) { - try { - setCreationTime(aFilmlist, aSplittedEntry); - setListId(aFilmlist, aSplittedEntry); - } catch (final Exception exception) { - LOG.debug("Somethin went wrong on setting the meta data of filmlist.", exception); - } - } - - private void setListId(final Filmlist aFilmlist, final List aSplittedEntry) { - try { - aFilmlist.setListId(UUID.fromString(aSplittedEntry.get(4))); - } catch (final IllegalArgumentException illegalArgumentException) { - LOG.debug("Can't parse the film list id. Setting a random uuid.", illegalArgumentException); - aFilmlist.setListId(UUID.randomUUID()); - } - } - - private void setCreationTime(final Filmlist aFilmlist, final List aSplittedEntry) { - final String[] dateTimeSplitted = aSplittedEntry.get(1).split(DATE_TIME_SPLITTERATOR); - aFilmlist.setCreationDate( - LocalDateTime.of( - LocalDate.parse(dateTimeSplitted[0], DATE_FORMATTER), - LocalTime.parse(dateTimeSplitted[1], TIME_FORMATTER))); - } - - private List splittEntry(final String aEntry) { - final List entrySplits = new ArrayList<>(); - final Matcher entrySplitMatcher = Pattern.compile(ENTRY_SPLIT_PATTERN).matcher(aEntry); - while (entrySplitMatcher.find()) { - entrySplits.add( - TextCleaner.clean( - entrySplitMatcher.group().replaceFirst(QUOTATION_MARK, "").replaceAll("\"$", ""))); - } - - return entrySplits; } } diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistEntryToFilmTask.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistEntryToFilmTask.java deleted file mode 100644 index a1368474..00000000 --- a/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistEntryToFilmTask.java +++ /dev/null @@ -1,268 +0,0 @@ -package de.mediathekview.mlib.filmlisten.reader; - -import de.mediathekview.mlib.daten.*; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.text.StringEscapeUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.net.MalformedURLException; -import java.net.URL; -import java.time.Duration; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Function; - -import static java.lang.String.format; -import static java.time.format.FormatStyle.MEDIUM; - -public class OldFilmlistEntryToFilmTask implements Callable { - private static final Logger LOG = LogManager.getLogger(OldFilmlistEntryToFilmTask.class); - private static final DateTimeFormatter DATE_FORMATTER = - DateTimeFormatter.ofLocalizedDate(MEDIUM).withLocale(Locale.GERMANY); - private static final DateTimeFormatter TIME_FORMATTER = - DateTimeFormatter.ofLocalizedTime(MEDIUM).withLocale(Locale.GERMANY); - private static final char GEO_SPLITTERATOR = '-'; - private static final String URL_SPLITTERATOR = "\\|"; - private static final String EXCEPTION_TEXT_CANT_BUILD_FILM = "Can't build a Film from splits."; - - private final List entrySplits; - private final Future entryBefore; - - OldFilmlistEntryToFilmTask(final List aEntrySplits, final Future aFilmEntryBefore) { - entrySplits = aEntrySplits.stream().map(String::trim).toList(); - entryBefore = aFilmEntryBefore; - } - - @Override - public Film call() throws OldFilmlistImportException { - try { - final String senderText = entrySplits.get(1); - final Sender sender = gatherSender(senderText); - final String thema = gatherTextOrUseAlternativ(2, AbstractMediaResource::getThema); - final String titel = gatherTextOrUseAlternativ(3, AbstractMediaResource::getTitel); - - if (sender == null) { - throw new CantReadFilmException( - format("Can't gather a Sender for the film \"%s\" - \"%s\".", thema, titel)); - } - - final LocalDate date = gatherDate(sender, thema, titel); - final LocalTime time = gatherTime(); - final Duration dauer = gatherDuration(sender, thema, titel); - final long groesse = gatherGroesse(sender, thema, titel); - - final String beschreibung = entrySplits.get(8); - - // Ignoring RTMP because can't find any usage. - // Ignoring Film URL History because can't find any usage. - final Film film = - new Film( - UUID.randomUUID(), - sender, - titel, - thema, - date == null ? null : LocalDateTime.of(date, time), - dauer); - addGeoLocations(film); - setWebsite(film); - setNeu(film); - film.setBeschreibung(beschreibung); - addSubtitle(film); - // Here we import also films without a download URL so the next entry can use the Sender, - // Thema and Titel. After the import all films without download URLs will be removed. - addUrls(groesse, film); - - return film; - } catch (final CantReadFilmException | MalformedURLException | ExecutionException exception) { - throw new OldFilmlistImportException(EXCEPTION_TEXT_CANT_BUILD_FILM, exception); - } catch (final InterruptedException exception) { - Thread.currentThread().interrupt(); - throw new OldFilmlistImportException(EXCEPTION_TEXT_CANT_BUILD_FILM, exception); - } - } - - private void addUrls(final long groesse, final Film film) throws MalformedURLException { - final Optional oprionalUrlNormal = gatherNormalUrl(); - if (oprionalUrlNormal.isPresent()) { - final URL urlNormal = oprionalUrlNormal.get(); - film.addUrl(Resolution.NORMAL, new FilmUrl(urlNormal, groesse)); - addAlternativUrl(groesse, film, urlNormal, 13, Resolution.SMALL); - addAlternativUrl(groesse, film, urlNormal, 15, Resolution.HD); - } - } - - private void addAlternativUrl( - final long groesse, final Film film, final URL urlNormal, final int i, final Resolution small) - throws MalformedURLException { - final String urlTextKlein = entrySplits.get(i); - if (!urlTextKlein.isEmpty()) { - final FilmUrl urlKlein = urlTextToUri(urlNormal, groesse, urlTextKlein); - if (urlKlein != null) { - film.addUrl(small, urlKlein); - } - } - } - - private void setNeu(final Film film) { - final String neu = entrySplits.get(20); - if (StringUtils.isNotBlank(neu)) { - film.setNeu(Boolean.parseBoolean(neu)); - } - } - - private void setWebsite(final Film film) { - final Optional urlWebseite = gatherWebsiteUrl(); - film.setWebsite(urlWebseite.orElse(null)); - } - - private void addGeoLocations(final Film film) { - final Collection geoLocations = readGeoLocations(entrySplits.get(19)); - film.addAllGeoLocations(geoLocations); - } - - private void addSubtitle(final Film film) throws MalformedURLException { - final String urlTextUntertitel = entrySplits.get(11); - if (!urlTextUntertitel.isEmpty()) { - film.addSubtitle(new URL(urlTextUntertitel)); - } - } - - private long gatherGroesse(final Sender sender, final String thema, final String titel) { - final String groesseText = entrySplits.get(7); - - final long groesse; - if (StringUtils.isNotBlank(groesseText)) { - groesse = Long.parseLong(groesseText); - } else { - groesse = 0L; - if (LOG.isDebugEnabled()) { - LOG.debug(format("Film ohne Größe \"%s %s - %s\".", sender.getName(), thema, titel)); - } - } - return groesse; - } - - private Duration gatherDuration(final Sender sender, final String thema, final String titel) { - final String durationText = entrySplits.get(6); - final Duration dauer; - if (StringUtils.isNotBlank(durationText)) { - dauer = Duration.between(LocalTime.MIDNIGHT, LocalTime.parse(durationText)); - } else { - dauer = Duration.ZERO; - if (LOG.isDebugEnabled()) { - LOG.debug(format("Film ohne Dauer \"%s %s - %s\".", sender.getName(), thema, titel)); - } - } - return dauer; - } - - private LocalTime gatherTime() { - final LocalTime time; - final String timeText = entrySplits.get(5); - if (StringUtils.isNotBlank(timeText)) { - time = LocalTime.parse(timeText, TIME_FORMATTER); - } else { - time = LocalTime.MIDNIGHT; - } - return time; - } - - @Nullable - private LocalDate gatherDate(final Sender sender, final String thema, final String titel) { - final String dateText = entrySplits.get(4); - final LocalDate date; - if (StringUtils.isNotBlank(dateText)) { - date = LocalDate.parse(dateText, DATE_FORMATTER); - } else { - date = null; - if (LOG.isDebugEnabled()) { - LOG.debug(format("Film ohne Datum \"%s %s - %s\".", sender.getName(), thema, titel)); - } - } - return date; - } - - @Nullable - private String gatherTextOrUseAlternativ(final int i, final Function alternativ) - throws ExecutionException, InterruptedException { - String text = entrySplits.get(i); - if (StringUtils.isBlank(text) && entryBefore != null) { - text = alternativ.apply(entryBefore.get()); - } - return text; - } - - private Sender gatherSender(final String senderText) - throws InterruptedException, java.util.concurrent.ExecutionException { - final Sender sender; - if (StringUtils.isBlank(senderText) && entryBefore != null) { - sender = entryBefore.get().getSender(); - } else { - sender = Sender.getSenderByName(senderText).orElse(null); - } - return sender; - } - - private Optional gatherNormalUrl() { - final String urlNormalText = entrySplits.get(9).trim(); - try { - return Optional.of(new URL(StringEscapeUtils.unescapeJava(urlNormalText))); - } catch (final MalformedURLException malformedURLException) { - LOG.debug( - format("The normal download URL \"%s\" can't be prased.", urlNormalText), - malformedURLException); - return Optional.empty(); - } - } - - private Optional gatherWebsiteUrl() { - final String websiteUrlText = entrySplits.get(10).trim(); - try { - return Optional.of(new URL(StringEscapeUtils.unescapeJava(websiteUrlText))); - } catch (final MalformedURLException malformedURLException) { - LOG.debug( - format("The website URL \"%s\" can't be prased.", websiteUrlText), malformedURLException); - return Optional.empty(); - } - } - - private Collection readGeoLocations(final String aGeoText) { - final Collection geoLocations = new ArrayList<>(); - - final GeoLocations singleGeoLocation = GeoLocations.getFromDescription(aGeoText); - if (singleGeoLocation == null) { - for (final String geoText : aGeoText.split(String.valueOf(GEO_SPLITTERATOR))) { - final GeoLocations geoLocation = GeoLocations.getFromDescription(geoText); - if (geoLocation != null) { - geoLocations.add(geoLocation); - } - } - } else { - geoLocations.add(singleGeoLocation); - } - - return geoLocations; - } - - private FilmUrl urlTextToUri(final URL aUrlNormal, final long aGroesse, final String aUrlText) - throws MalformedURLException { - FilmUrl filmUrl = null; - - final String[] splittedUrlText = aUrlText.split(URL_SPLITTERATOR); - if (splittedUrlText.length == 2) { - final int lengthOfOld = Integer.parseInt(splittedUrlText[0]); - - final String newUrl = aUrlNormal.toString().substring(0, lengthOfOld) + splittedUrlText[1]; - filmUrl = new FilmUrl(new URL(newUrl), aGroesse); - } - return filmUrl; - } -} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistImportException.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistImportException.java deleted file mode 100644 index c4a24a7d..00000000 --- a/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistImportException.java +++ /dev/null @@ -1,7 +0,0 @@ -package de.mediathekview.mlib.filmlisten.reader; - -class OldFilmlistImportException extends Exception { - OldFilmlistImportException(final String text, final Throwable cause) { - super(text, cause); - } -} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistToRawFilmlistReader.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistToRawFilmlistReader.java new file mode 100644 index 00000000..227fe16e --- /dev/null +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/OldFilmlistToRawFilmlistReader.java @@ -0,0 +1,128 @@ +package de.mediathekview.mlib.filmlisten.reader; + +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.Scanner; +import java.util.UUID; + +import static java.time.format.FormatStyle.MEDIUM; +import static java.time.format.FormatStyle.SHORT; + +@RequiredArgsConstructor +public class OldFilmlistToRawFilmlistReader { + private static final Logger LOG = LogManager.getLogger(OldFilmlistToRawFilmlistReader.class); + private static final String ENTRY_DELIMITER = "],\"X\":\s*\\["; + private static final String QUOTATION_MARK = "\""; + private static final String ENTRY_SPLIT_PATTERN = "\",\""; + private static final DateTimeFormatter DATE_FORMATTER = + DateTimeFormatter.ofLocalizedDate(MEDIUM).withLocale(Locale.GERMANY); + private static final DateTimeFormatter TIME_FORMATTER = + DateTimeFormatter.ofLocalizedTime(SHORT).withLocale(Locale.GERMANY); + private static final String DATE_TIME_SPLITERATOR = ",?\\s+"; + private static final String ALL_AFTER_UUID_PATTERN = "\"].*"; + private final InputStream filmlistStream; + + public RawFilmlist read() { + Instant start = Instant.now(); + RawFilmlist rawFilmlist = readEntryArrays(); + rawFilmlist.resolveEmptyFields(); + LOG.debug( + "Read {} raw films in {}", + rawFilmlist.getFilmCount(), + Duration.between(start, Instant.now())); + return rawFilmlist; + } + + private UUID toListId(String entryPart) { + try { + String rawUUID = entryPart.replaceFirst(ALL_AFTER_UUID_PATTERN, ""); + return UUID.fromString( + String.format( + "%s-%s-%s-%s-%s", + rawUUID.substring(0, 8), + rawUUID.substring(8, 12), + rawUUID.substring(12, 16), + rawUUID.substring(16, 20), + rawUUID.substring(20, 32))); + } catch (final IllegalArgumentException illegalArgumentException) { + LOG.debug("Can't parse the film list id. Setting a random uuid.", illegalArgumentException); + return UUID.randomUUID(); + } + } + + private LocalDateTime toCreationTime(String entryPart) { + final String[] dateTimeSplitted = + entryPart.replaceFirst(".*\\[\"", "").split(DATE_TIME_SPLITERATOR); + return LocalDateTime.of( + LocalDate.parse(dateTimeSplitted[0], DATE_FORMATTER), + LocalTime.parse(dateTimeSplitted[1], TIME_FORMATTER)); + } + + private String removeFirstAndLastQuotationMarkFromField(String rawField) { + if (rawField.isEmpty()) { + return rawField; + } + + String cleanedField = rawField; + if (cleanedField.startsWith(QUOTATION_MARK)) { + cleanedField = cleanedField.substring(1); + } + if (cleanedField.endsWith(QUOTATION_MARK)) { + cleanedField = cleanedField.substring(0, cleanedField.length() - 1); + } + return cleanedField; + } + + @NotNull + private RawFilmlist readEntryArrays() { + RawFilmlist.RawFilmlistBuilder rawFilmlistBuilder = RawFilmlist.builder(); + try (final Scanner scanner = new Scanner(filmlistStream, StandardCharsets.UTF_8.name()); + final Scanner entryScanner = scanner.useDelimiter(ENTRY_DELIMITER)) { + + while (entryScanner.hasNext()) { + final String rawEntry = entryScanner.next(); + + String[] entryArray = rawEntry.split(ENTRY_SPLIT_PATTERN); + if (entryArray.length == 20) { + rawFilmlistBuilder.rawFilm( + new RawFilm( + removeFirstAndLastQuotationMarkFromField(entryArray[0]), + removeFirstAndLastQuotationMarkFromField(entryArray[1]), + removeFirstAndLastQuotationMarkFromField(entryArray[2]), + removeFirstAndLastQuotationMarkFromField(entryArray[3]), + removeFirstAndLastQuotationMarkFromField(entryArray[4]), + removeFirstAndLastQuotationMarkFromField(entryArray[5]), + removeFirstAndLastQuotationMarkFromField(entryArray[6]), + removeFirstAndLastQuotationMarkFromField(entryArray[7]), + removeFirstAndLastQuotationMarkFromField(entryArray[8]), + removeFirstAndLastQuotationMarkFromField(entryArray[9]), + removeFirstAndLastQuotationMarkFromField(entryArray[10]), + removeFirstAndLastQuotationMarkFromField(entryArray[12]), + removeFirstAndLastQuotationMarkFromField(entryArray[14]), + removeFirstAndLastQuotationMarkFromField(entryArray[18]), + removeFirstAndLastQuotationMarkFromField(entryArray[19]))); + } else { + rawFilmlistBuilder + .creationDate(toCreationTime(entryArray[0])) + .listId(toListId(entryArray[4])); + } + } + } finally { + try { + filmlistStream.close(); + } catch (final IOException exception) { + LOG.debug("Can't close the ioStream", exception); + } + } + return rawFilmlistBuilder.build(); + } +} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilm.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilm.java new file mode 100644 index 00000000..aa608c1b --- /dev/null +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilm.java @@ -0,0 +1,27 @@ +package de.mediathekview.mlib.filmlisten.reader; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@AllArgsConstructor +public class RawFilm { + + String sender; + String thema; + String titel; + String datum; + String zeit; + String dauer; + String groesseMb; + String beschreibung; + String url; + String website; + String urlUntertitel; + String urlKlein; + String urlHd; + String geo; + String neu; +} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmToFilmException.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmToFilmException.java new file mode 100644 index 00000000..b5cc61a2 --- /dev/null +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmToFilmException.java @@ -0,0 +1,7 @@ +package de.mediathekview.mlib.filmlisten.reader; + +public class RawFilmToFilmException extends RuntimeException { + public RawFilmToFilmException(String message) { + super(message); + } +} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmToFilmMapper.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmToFilmMapper.java new file mode 100644 index 00000000..1290efcf --- /dev/null +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmToFilmMapper.java @@ -0,0 +1,252 @@ +package de.mediathekview.mlib.filmlisten.reader; + +import de.mediathekview.mlib.daten.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringEscapeUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.mapstruct.*; +import org.mapstruct.factory.Mappers; + +import java.net.MalformedURLException; +import java.net.URL; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; + +import static java.time.format.FormatStyle.MEDIUM; +import static java.time.format.FormatStyle.SHORT; + +@Mapper +public interface RawFilmToFilmMapper { + Logger LOG = LogManager.getLogger(RawFilmToFilmMapper.class); + DateTimeFormatter DATE_FORMATTER = + DateTimeFormatter.ofLocalizedDate(MEDIUM).withLocale(Locale.GERMANY); + DateTimeFormatter DATE_SHORT_FORMATTER = + DateTimeFormatter.ofLocalizedDate(SHORT).withLocale(Locale.GERMANY); + DateTimeFormatter TIME_FORMATTER = + DateTimeFormatter.ofLocalizedTime(MEDIUM).withLocale(Locale.GERMANY); + RawFilmToFilmMapper INSTANCE = Mappers.getMapper(RawFilmToFilmMapper.class); + String URL_SPLITERATOR = "\\|"; + int SHORT_GERMAN_DATE_LENGTH = 8; + + @Mapping(target = "urls", ignore = true) + @Mapping(target = "signLanguages", ignore = true) + @Mapping(target = "merge", ignore = true) + @Mapping(target = "audioDescriptions", ignore = true) + @Mapping(target = "uuid", expression = "java(java.util.UUID.randomUUID())") + @Mapping(target = "website", expression = "java(websiteToWebsiteUrl(rawFilm))") + @Mapping(target = "sender", source = "sender", qualifiedByName = "senderTextToSender") + @Mapping(target = "time", expression = "java(mapDateTime(rawFilm))") + @Mapping(target = "geoLocations", expression = "java(mapGeolocation(rawFilm))") + @Mapping(target = "duration", expression = "java(mapDuration(rawFilm))") + @Mapping(target = "subtitles", expression = "java(mapSubtitleUrl(rawFilm))") + Film rawFilmToFilm(RawFilm rawFilm); + + default List mapGeolocation(RawFilm rawFilm) { + return GeoLocations.find(rawFilm.getGeo()).stream().toList(); + } + + default LocalDateTime mapDateTime(RawFilm rawFilm) { + final Optional optionalDate = gatherDate(rawFilm); + + final LocalTime time = gatherTime(rawFilm.getZeit()); + return optionalDate.map(date -> LocalDateTime.of(date, time)).orElse(null); + } + + default URL websiteToWebsiteUrl(RawFilm rawFilm) { + try { + return new URL(StringEscapeUtils.unescapeJava(rawFilm.getWebsite())); + } catch (MalformedURLException malformedURLException) { + LOG.debug( + "The film \"{} {} - {}\" has a invalid website URL \"{}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel(), + malformedURLException); + return null; + } + } + + @Named("senderTextToSender") + default Sender senderTextToSender(String senderText) { + return Sender.getSenderByName(senderText) + .orElseThrow( + () -> + new RawFilmToFilmException( + String.format("The sender \"%s\" is unknown!", senderText))); + } + + @AfterMapping + default void complexMappings(RawFilm rawFilm, @MappingTarget Film film) { + long groesse = mapSize(rawFilm); + final Optional optionalUrlNormal = gatherNormalUrl(rawFilm.getUrl()); + if (optionalUrlNormal.isPresent()) { + final URL urlNormal = optionalUrlNormal.get(); + film.addUrl(Resolution.NORMAL, new FilmUrl(urlNormal, groesse)); + + buildAlternativeUrl(film, groesse, urlNormal, rawFilm.getUrlKlein()) + .ifPresent(url -> film.addUrl(Resolution.SMALL, url)); + buildAlternativeUrl(film, groesse, urlNormal, rawFilm.getUrlHd()) + .ifPresent(url -> film.addUrl(Resolution.HD, url)); + } + } + + default Set mapSubtitleUrl(RawFilm rawFilm) { + String untertitelUrl = rawFilm.getUrlUntertitel(); + if (untertitelUrl != null && !untertitelUrl.isEmpty()) { + try { + return Set.of(new URL(untertitelUrl)); + } catch (MalformedURLException malformedURLException) { + LOG.debug( + "The film \"{} {} - {}\" has a invalid subtitle URL \"{}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel(), + malformedURLException); + } + } + return new HashSet<>(); + } + + default Optional gatherNormalUrl(String url) { + try { + return Optional.of(new URL(StringEscapeUtils.unescapeJava(url))); + } catch (final MalformedURLException malformedURLException) { + LOG.debug("The normal download URL \"{}\" can't be parsed.", url, malformedURLException); + return Optional.empty(); + } + } + + /** + * In the old format all others film urls then the normal one are shortened. All chars which are + * same between the URLs getting counted and will be replaced by the count and the char |. This + * method inverts this behavior.
+ *
+ * Example: https://www.google.com as base (URL for size normal) and + * 19|de will get to: https://www.google.de. + * + * @param film The film is used for the logging if a URL is invalid so Sender, Thema and Title of + * the Film with the invalid URL can be printed. + * @param groesse The size of the video file for the normal URL. + * @param urlNormal The URL for normal. + * @param url The URL with the char count part. + * @return The full URL without the char count part. + */ + default Optional buildAlternativeUrl( + Film film, final long groesse, final URL urlNormal, final String url) { + if (url == null || url.isEmpty()) { + return Optional.empty(); + } + + final String[] splittedUrlText = url.split(URL_SPLITERATOR); + if (splittedUrlText.length == 2) { + final int lengthOfOld = Integer.parseInt(splittedUrlText[0]); + String urlNormalText = urlNormal.toString(); + if (lengthOfOld >= 0 && lengthOfOld <= urlNormalText.length()) { + final String newUrl = urlNormalText.substring(0, lengthOfOld) + splittedUrlText[1]; + try { + return Optional.of(new FilmUrl(new URL(newUrl), groesse)); + } catch (MalformedURLException malformedURLException) { + LOG.debug( + "The film \"{} {} - {}\" has a invalid film URL \"{}\".", + film.getSender().getName(), + film.getThema(), + film.getTitel(), + malformedURLException); + } + } + } + return Optional.empty(); + } + + default Duration mapDuration(RawFilm rawFilm) { + String dauer = rawFilm.getDauer(); + try { + if (StringUtils.isNotBlank(dauer)) { + return Duration.between(LocalTime.MIDNIGHT, LocalTime.parse(dauer)); + } else { + LOG.debug( + "A film without duration \"{} {} - {}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel()); + } + } catch (DateTimeParseException dateTimeParseException) { + LOG.debug( + "The film \"{} {} - {}\" has a invalid duartion: \"{}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel(), + dauer, + dateTimeParseException); + } + return Duration.ZERO; + } + + default LocalTime gatherTime(String zeit) { + if (StringUtils.isNotBlank(zeit)) { + try { + return LocalTime.parse(zeit, TIME_FORMATTER); + } catch (DateTimeParseException dateTimeParseException) { + LOG.debug("A film has a invalid time: \"{}\".", zeit, dateTimeParseException); + } + } + return LocalTime.MIDNIGHT; + } + + default long mapSize(RawFilm rawFilm) { + String groesseText = rawFilm.getGroesseMb(); + if (StringUtils.isNotBlank(groesseText)) { + try { + return Long.parseLong(groesseText); + } catch (NumberFormatException numberFormatException) { + LOG.debug( + "The film \"{} {} - {}\" has an invalid size \"{}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel(), + groesseText, + numberFormatException); + } + } + LOG.debug( + "A film without a size \"{} {} - {}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel()); + + return 0L; + } + + default Optional gatherDate(RawFilm rawFilm) { + String datum = rawFilm.getDatum(); + try { + if (StringUtils.isNotBlank(datum)) { + if (datum.length() == SHORT_GERMAN_DATE_LENGTH) { + return Optional.of(LocalDate.parse(datum, DATE_SHORT_FORMATTER)); + } + return Optional.of(LocalDate.parse(datum, DATE_FORMATTER)); + } else { + LOG.debug( + "A film without date \"{} {} - {}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel()); + } + } catch (DateTimeParseException dateTimeParseException) { + LOG.debug( + "The film \"{} {} - {}\" has a invalid date: \"{}\".", + rawFilm.getSender(), + rawFilm.getThema(), + rawFilm.getTitel(), + datum, + dateTimeParseException); + } + return Optional.empty(); + } +} diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmlist.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmlist.java new file mode 100644 index 00000000..e80c2d23 --- /dev/null +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/RawFilmlist.java @@ -0,0 +1,39 @@ +package de.mediathekview.mlib.filmlisten.reader; + +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.Singular; +import org.apache.logging.log4j.util.Strings; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.UUID; + +@Data +@Builder +@RequiredArgsConstructor +public class RawFilmlist { + private final LocalDateTime creationDate; + private final UUID listId; + @Singular private final List rawFilms; + + public int getFilmCount() { + return rawFilms.size(); + } + + public void resolveEmptyFields() { + for (int i = 1; i < rawFilms.size(); i++) { + RawFilm currentFilm = rawFilms.get(i); + if (Strings.isEmpty(currentFilm.getSender())) { + currentFilm.setSender(rawFilms.get(i - 1).getSender()); + } + if (Strings.isEmpty(currentFilm.getThema())) { + currentFilm.setThema(rawFilms.get(i - 1).getThema()); + } + if (Strings.isEmpty(currentFilm.getTitel())) { + currentFilm.setTitel(rawFilms.get(i - 1).getTitel()); + } + } + } +} diff --git a/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java b/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java index ce863063..03d5c7ae 100644 --- a/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java +++ b/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java @@ -9,7 +9,7 @@ import java.io.IOException; -import static jakarta.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; public class FileSizeDeterminer { private static final Logger LOG = LogManager.getLogger(FileSizeDeterminer.class); diff --git a/src/test/java/de/mediathekview/mlib/daten/FilmlistMergeTest.java b/src/test/java/de/mediathekview/mlib/daten/FilmlistMergeTest.java index 7630f3cf..8a8588f0 100644 --- a/src/test/java/de/mediathekview/mlib/daten/FilmlistMergeTest.java +++ b/src/test/java/de/mediathekview/mlib/daten/FilmlistMergeTest.java @@ -24,9 +24,9 @@ public Film createTestFilm1() throws MalformedURLException { Duration.of(10, ChronoUnit.MINUTES)); testFilm1.setWebsite(new URL("http://www.example.org/")); testFilm1.setBeschreibung("Test beschreibung."); - testFilm1.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein.mp4"), 42l)); - testFilm1.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test.mp4"), 42l)); - testFilm1.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd.mp4"), 42l)); + testFilm1.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein.mp4"), 42L)); + testFilm1.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test.mp4"), 42L)); + testFilm1.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd.mp4"), 42L)); return testFilm1; } @@ -41,9 +41,9 @@ public Film createTestFilm2() throws MalformedURLException { Duration.of(10, ChronoUnit.MINUTES)); testFilm2.setWebsite(new URL("http://www.example.org/2")); testFilm2.setBeschreibung("Test beschreibung."); - testFilm2.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein2.mp4"), 42l)); - testFilm2.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test2.mp4"), 42l)); - testFilm2.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd2.mp4"), 42l)); + testFilm2.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein2.mp4"), 42L)); + testFilm2.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test2.mp4"), 42L)); + testFilm2.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd2.mp4"), 42L)); return testFilm2; } @@ -58,9 +58,9 @@ public Film createTestFilm3() throws MalformedURLException { Duration.of(10, ChronoUnit.MINUTES)); testFilm3.setWebsite(new URL("http://www.example.org/")); testFilm3.setBeschreibung("Test beschreibung."); - testFilm3.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein.mp4"), 42l)); - testFilm3.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test.mp4"), 42l)); - testFilm3.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd.mp4"), 42l)); + testFilm3.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein.mp4"), 42L)); + testFilm3.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test.mp4"), 42L)); + testFilm3.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd.mp4"), 42L)); return testFilm3; } @@ -85,7 +85,7 @@ void testMergeNotEqualsSender() throws MalformedURLException { testFilm3.getDuration()); testFilm4.setWebsite(testFilm3.getWebsite().orElse(null)); testFilm4.setBeschreibung(testFilm3.getBeschreibung()); - testFilm3.getUrls().entrySet().forEach(e -> testFilm4.addUrl(e.getKey(), e.getValue())); + testFilm3.getUrls().forEach(testFilm4::addUrl); final Filmlist testFilmlist2 = new Filmlist(); testFilmlist2.add(testFilm1); @@ -177,7 +177,7 @@ void testMergeUpdateUUID() throws MalformedURLException { testFilm3.getDuration()); testFilm4.setWebsite(testFilm3.getWebsite().orElse(null)); testFilm4.setBeschreibung(testFilm3.getBeschreibung()); - testFilm3.getUrls().entrySet().forEach(e -> testFilm4.addUrl(e.getKey(), e.getValue())); + testFilm3.getUrls().forEach(testFilm4::addUrl); final Filmlist testFilmlist2 = new Filmlist(); testFilmlist2.add(testFilm1); @@ -185,7 +185,7 @@ void testMergeUpdateUUID() throws MalformedURLException { testFilmlist1.add(testFilm4); final int sizeOld = testFilmlist1.getFilms().size(); testFilmlist1.merge(testFilmlist2); - assertThat(testFilmlist1.getFilms().size()).isEqualTo(sizeOld); + assertThat(testFilmlist1.getFilms()).hasSize(sizeOld); assertThat(testFilm3).hasSameHashCodeAs(testFilm4); } } diff --git a/src/test/java/de/mediathekview/mlib/daten/FilmlistTest.java b/src/test/java/de/mediathekview/mlib/daten/FilmlistTest.java index 5d1cf830..5375903f 100644 --- a/src/test/java/de/mediathekview/mlib/daten/FilmlistTest.java +++ b/src/test/java/de/mediathekview/mlib/daten/FilmlistTest.java @@ -35,13 +35,13 @@ void testMerge() throws MalformedURLException { filmlist2.add(testLivestream2); final Filmlist differenceList = filmlist1.merge(filmlist2); - assertThat(differenceList.getFilms().size()).isZero(); - assertThat(differenceList.getPodcasts()).hasSize(1).allSatisfy((currentKey, currentElement) -> { - assertThat(currentElement).isEqualTo(testPodcast1); - }); - assertThat(differenceList.getLivestreams()).hasSize(1).allSatisfy((currentKey, currentElement) -> { - assertThat(currentElement).isEqualTo(testLivestream2); - }); + assertThat(differenceList.getFilms()).isEmpty(); + assertThat(differenceList.getPodcasts()).hasSize(1).allSatisfy((currentKey, currentElement) -> + assertThat(currentElement).isEqualTo(testPodcast1) + ); + assertThat(differenceList.getLivestreams()).hasSize(1).allSatisfy((currentKey, currentElement) -> + assertThat(currentElement).isEqualTo(testLivestream2) + ); } private Film createTestFilm1() throws MalformedURLException { @@ -49,9 +49,9 @@ private Film createTestFilm1() throws MalformedURLException { LocalDateTime.parse("2017-01-01T23:55:00"), Duration.of(10, ChronoUnit.MINUTES)); testFilm1.setWebsite(new URL("http://www.example.org/")); testFilm1.setBeschreibung("Test beschreibung."); - testFilm1.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein.mp4"), 42l)); - testFilm1.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test.mp4"), 42l)); - testFilm1.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd.mp4"), 42l)); + testFilm1.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein.mp4"), 42L)); + testFilm1.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test.mp4"), 42L)); + testFilm1.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd.mp4"), 42L)); return testFilm1; } @@ -60,9 +60,9 @@ private Film createTestFilm2() throws MalformedURLException { LocalDateTime.parse("2017-01-01T23:55:00"), Duration.of(10, ChronoUnit.MINUTES)); testFilm2.setWebsite(new URL("http://www.example.org/2")); testFilm2.setBeschreibung("Test beschreibung."); - testFilm2.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein2.mp4"), 42l)); - testFilm2.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test2.mp4"), 42l)); - testFilm2.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd2.mp4"), 42l)); + testFilm2.addUrl(Resolution.SMALL, new FilmUrl(new URL("http://example.org/klein2.mp4"), 42L)); + testFilm2.addUrl(Resolution.NORMAL, new FilmUrl(new URL("http://example.org/Test2.mp4"), 42L)); + testFilm2.addUrl(Resolution.HD, new FilmUrl(new URL("http://example.org/hd2.mp4"), 42L)); return testFilm2; } @@ -87,7 +87,7 @@ private Podcast createTestPodcast1() throws MalformedURLException { LocalDateTime.parse("2017-01-01T23:55:00"), Duration.of(10, ChronoUnit.MINUTES)); testPodcast1.setWebsite(new URL("http://www.example.org/2")); testPodcast1.addUrl(Resolution.NORMAL, - new FilmUrl(new URL("http://example.org/normal.mp3"), 42l)); + new FilmUrl(new URL("http://example.org/normal.mp3"), 42L)); return testPodcast1; } @@ -96,7 +96,7 @@ private Podcast createTestPodcast2() throws MalformedURLException { LocalDateTime.parse("2017-01-01T23:55:00"), Duration.of(10, ChronoUnit.MINUTES)); testPodcast2.setWebsite(new URL("http://www.example.org/2")); testPodcast2.addUrl(Resolution.NORMAL, - new FilmUrl(new URL("http://example.org/normal.mp3"), 42l)); + new FilmUrl(new URL("http://example.org/normal.mp3"), 42L)); return testPodcast2; } diff --git a/src/test/java/de/mediathekview/mlib/filmlisten/RawFilmToFilmMapperTest.java b/src/test/java/de/mediathekview/mlib/filmlisten/RawFilmToFilmMapperTest.java new file mode 100644 index 00000000..613e9629 --- /dev/null +++ b/src/test/java/de/mediathekview/mlib/filmlisten/RawFilmToFilmMapperTest.java @@ -0,0 +1,363 @@ +package de.mediathekview.mlib.filmlisten; + +import de.mediathekview.mlib.daten.*; +import de.mediathekview.mlib.filmlisten.reader.RawFilm; +import de.mediathekview.mlib.filmlisten.reader.RawFilmToFilmException; +import de.mediathekview.mlib.filmlisten.reader.RawFilmToFilmMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.mapstruct.factory.Mappers; + +import java.net.MalformedURLException; +import java.net.URL; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.*; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +class RawFilmToFilmMapperTest { + private final RawFilmToFilmMapper classUnderTest = Mappers.getMapper(RawFilmToFilmMapper.class); + + private static Stream mapSubtitleUrlArguments() throws MalformedURLException { + return Stream.of( + Arguments.of(null, new ArrayList<>()), + Arguments.of("", new ArrayList<>()), + Arguments.of("invalidUrl", new ArrayList<>()), + Arguments.of("https://www.google.com/", List.of(new URL("https://www.google.com/")))); + } + + private static Stream gatherNormalUrlArguments() throws MalformedURLException { + return Stream.of( + Arguments.of(null, null), + Arguments.of("", null), + Arguments.of("invalidUrl", null), + Arguments.of("https://www.google.com/", new URL("https://www.google.com/")), + Arguments.of("https://fr\u00FCher.xyz/", new URL("https://früher.xyz/"))); + } + + private static Stream buildAlternativeUrlArguments() throws MalformedURLException { + return Stream.of( + Arguments.of(null, null, null), + Arguments.of(new URL("https://google.com"), null, null), + Arguments.of(new URL("https://google.com"), null, null), + Arguments.of(new URL("https://google.com"), "", null), + Arguments.of(new URL("https://google.com"), "0|invalidUrl", null), + Arguments.of( + new URL("https://google.com"), + "0|http://localhost/", + new FilmUrl("http://localhost/", -42L)), + Arguments.of(new URL("https://google.com"), "-1|http://localhost/", null), + Arguments.of( + new URL("https://google.com"), "15|de", new FilmUrl("https://google.de", -42L)), + Arguments.of( + new URL("https://google.com"), + "18|/search", + new FilmUrl("https://google.com/search", -42L)), + Arguments.of(new URL("https://google.com"), "19|/search", null), + Arguments.of( + new URL("https://fr\u00FCher.xyz"), "15|de", new FilmUrl("https://früher.de", -42L))); + } + + private static Stream mapDurationArguments() { + return Stream.of( + Arguments.of(null, Duration.ZERO), + Arguments.of("", Duration.ZERO), + Arguments.of("invalid Duration", Duration.ZERO), + Arguments.of("00:00:01", Duration.ofSeconds(1)), + Arguments.of("00:01:00", Duration.ofMinutes(1)), + Arguments.of("01:00:00", Duration.ofHours(1)), + Arguments.of("23:59:59", Duration.ofHours(23).plusMinutes(59).plusSeconds(59))); + } + + private static Stream gatherTimeArguments() { + return Stream.of( + Arguments.of(null, LocalTime.MIDNIGHT), + Arguments.of("", LocalTime.MIDNIGHT), + Arguments.of("invalid time", LocalTime.MIDNIGHT), + Arguments.of("00:00:01", LocalTime.of(0, 0, 1)), + Arguments.of("00:01:00", LocalTime.of(0, 1, 0)), + Arguments.of("01:00:00", LocalTime.of(1, 0, 0)), + Arguments.of("23:59:59", LocalTime.of(23, 59, 59))); + } + + private static Stream mapSizeArguments() { + return Stream.of( + Arguments.of(null, 0L), + Arguments.of("", 0L), + Arguments.of("invalid size", 0L), + Arguments.of("753", 753L), + Arguments.of("-3", -3L)); + } + + private static Stream gatherDateArguments() { + return Stream.of( + Arguments.of(null, null), + Arguments.of("", null), + Arguments.of("invalid date", null), + Arguments.of("01.01.2000", LocalDate.of(2000, 1, 1)), + Arguments.of("31.12.1999", LocalDate.of(1999, 12, 31)), + Arguments.of("09.03.2022", LocalDate.of(2022, 3, 9))); + } + + @Test + void rawFilmToFilm_null_noExceptions() { + assertThat(classUnderTest.rawFilmToFilm(null)).isNull(); + } + + @Test + void gatherNormalUrl_withValidURL_createsValidURL() throws Exception { + + Optional result = classUnderTest.gatherNormalUrl("https://www.heise.de"); + + assertThat(result.orElse(null)).isEqualTo(new URL("https://www.heise.de")); + } + + @Test + void gatherNormalUrl_withEscapedJava_GetsEmpty() { + + Optional result = classUnderTest.gatherNormalUrl("https://www.heise\n.de"); + + assertThat(result).isEmpty(); + } + + @Test + void rawFilmToFilm_validRawFilm_correctFilm() throws MalformedURLException { + // GIVEN + var rawFilm = + new RawFilm( + "3Sat", + "37 Grad", + "37°: Gewalt in den Familien (Audiodeskription)", + "07.12.2021", + "00:18:00", + "00:28:50", + "396", + "Laut einer Studie des Bundesfamilienministeriums wird etwa jede vierte Frau mindestens einmal Opfer körperlicher oder sexueller Gewalt durch ihren aktuellen oder früheren Partner.", + "https://rodlzdf-a.akamaihd.net/none/zdf/21/11/211130_sendung_37g/4/211130_sendung_37g_a3a4_2360k_p35v15.mp4", + "https://www.3sat.de/gesellschaft/37-grad/37-schlag-ins-herz-100.html", + "https://utstreaming.zdf.de/mtt/zdf/21/11/211130_sendung_37g/4/F1033253_hoh_deu_37_Grad_Schlag_ins_Herz_301121.xml", + "91|808k_p11v15.mp4", + "91|3360k_p36v15.mp4", + "", + "false"); + + var expectedFilm = + new Film( + UUID.randomUUID(), + Sender.DREISAT, + "37°: Gewalt in den Familien (Audiodeskription)", + "37 Grad", + LocalDateTime.of(2021, 12, 7, 0, 18, 0), + Duration.ofMinutes(28).plusSeconds(50)); + expectedFilm.setBeschreibung( + "Laut einer Studie des Bundesfamilienministeriums wird etwa jede vierte Frau mindestens einmal Opfer körperlicher oder sexueller Gewalt durch ihren aktuellen oder früheren Partner."); + expectedFilm.addUrl( + Resolution.NORMAL, + new FilmUrl( + "https://rodlzdf-a.akamaihd.net/none/zdf/21/11/211130_sendung_37g/4/211130_sendung_37g_a3a4_2360k_p35v15.mp4", + 396L)); + expectedFilm.setWebsite( + new URL("https://www.3sat.de/gesellschaft/37-grad/37-schlag-ins-herz-100.html")); + expectedFilm.addSubtitle( + new URL( + "https://utstreaming.zdf.de/mtt/zdf/21/11/211130_sendung_37g/4/F1033253_hoh_deu_37_Grad_Schlag_ins_Herz_301121.xml")); + expectedFilm.addUrl( + Resolution.SMALL, + new FilmUrl( + "https://rodlzdf-a.akamaihd.net/none/zdf/21/11/211130_sendung_37g/4/211130_sendung_37g_a3a4_808k_p11v15.mp4", + 396L)); + expectedFilm.addUrl( + Resolution.HD, + new FilmUrl( + "https://rodlzdf-a.akamaihd.net/none/zdf/21/11/211130_sendung_37g/4/211130_sendung_37g_a3a4_3360k_p36v15.mp4", + 396L)); + expectedFilm.addGeolocation(GeoLocations.GEO_NONE); + + // WHEN + var film = classUnderTest.rawFilmToFilm(rawFilm); + + // THEN + assertThat(film).usingRecursiveComparison().ignoringFields("uuid").isEqualTo(expectedFilm); + } + + @Test + void mapGeolocation_geoNull_emptyList() { + assertThat(classUnderTest.mapGeolocation(RawFilm.builder().build())).isEmpty(); + } + + @Test + void mapGeolocation_unknownGeo_emptyList() { + assertThat(classUnderTest.mapGeolocation(RawFilm.builder().geo("unknown").build())).isEmpty(); + } + + @ParameterizedTest + @EnumSource(GeoLocations.class) + void mapGeolocation_validGeo_listWithGeo(GeoLocations testGeoLocation) { + // Description + assertThat( + classUnderTest.mapGeolocation( + RawFilm.builder().geo(testGeoLocation.getDescription()).build())) + .containsExactly(testGeoLocation); + + // Alternatives + Arrays.stream(testGeoLocation.getAlternatives()) + .forEach( + alternative -> + assertThat( + classUnderTest.mapGeolocation(RawFilm.builder().geo(alternative).build())) + .containsExactly(testGeoLocation)); + } + + @Test + void websiteToWebsiteUrl_null_null() { + assertThat(classUnderTest.websiteToWebsiteUrl(RawFilm.builder().website(null).build())) + .isNull(); + } + + @Test + void websiteToWebsiteUrl_emptyString_null() { + assertThat(classUnderTest.websiteToWebsiteUrl(RawFilm.builder().website("").build())).isNull(); + } + + @Test + void websiteToWebsiteUrl_invalidUrl_null() { + assertThat(classUnderTest.websiteToWebsiteUrl(RawFilm.builder().website("unknown").build())) + .isNull(); + } + + @Test + void websiteToWebsiteUrl_urlWithSpecialChars_url() throws MalformedURLException { + assertThat( + classUnderTest.websiteToWebsiteUrl( + RawFilm.builder().website("https://fr\u00FCher.xyz/").build())) + .isEqualTo(new URL("https://früher.xyz/")); + } + + @Test + void senderTextToSender_null_exception() { + assertThat(catchThrowable(() -> classUnderTest.senderTextToSender(null))) + .isInstanceOf(RawFilmToFilmException.class); + } + + @Test + void senderTextToSender_empty_exception() { + assertThat(catchThrowable(() -> classUnderTest.senderTextToSender(""))) + .isInstanceOf(RawFilmToFilmException.class); + } + + @Test + void senderTextToSender_invalidSender_exception() { + assertThat(catchThrowable(() -> classUnderTest.senderTextToSender("unknown"))) + .isInstanceOf(RawFilmToFilmException.class); + } + + @ParameterizedTest + @EnumSource(Sender.class) + void senderTextToSender_validSenderName_sender(Sender testSender) { + assertThat(classUnderTest.senderTextToSender(testSender.getName())).isEqualTo(testSender); + + Arrays.stream(testSender.getNameAlternatives()) + .forEach( + alternativeName -> + assertThat(classUnderTest.senderTextToSender(alternativeName)) + .isEqualTo(testSender)); + } + + @ParameterizedTest + @MethodSource("mapSubtitleUrlArguments") + void mapSubtitleUrl(String testUrl, List expectedUrl) { + RawFilm testFilm = RawFilm.builder().urlUntertitel(testUrl).build(); + + if (expectedUrl.isEmpty()) { + assertThat(classUnderTest.mapSubtitleUrl(testFilm)).isEmpty(); + } else { + assertThat(classUnderTest.mapSubtitleUrl(testFilm)).containsExactlyElementsOf(expectedUrl); + } + } + + @ParameterizedTest + @MethodSource("gatherNormalUrlArguments") + void gatherNormalUrl(String testUrl, URL expectedUrl) { + if (expectedUrl == null) { + assertThat(classUnderTest.gatherNormalUrl(testUrl)).isNotPresent(); + } else { + assertThat(classUnderTest.gatherNormalUrl(testUrl)).isPresent().get().isEqualTo(expectedUrl); + } + } + + @ParameterizedTest + @MethodSource("buildAlternativeUrlArguments") + void buildAlternativeUrl(URL testBaseUrl, String testUrl, FilmUrl expectedUrl) { + // Just for the logging ¯\_(ツ)_/¯ + Film testFilm = + new Film( + UUID.randomUUID(), + Sender.NDR, + "Test title", + "Test thema", + LocalDateTime.now(), + Duration.ZERO); + long testGroesse = -42L; + if (expectedUrl == null) { + assertThat(classUnderTest.buildAlternativeUrl(testFilm, testGroesse, testBaseUrl, testUrl)) + .isNotPresent(); + } else { + assertThat(classUnderTest.buildAlternativeUrl(testFilm, testGroesse, testBaseUrl, testUrl)) + .isPresent() + .get() + .isEqualTo(expectedUrl); + } + } + + @ParameterizedTest + @MethodSource("mapDurationArguments") + void mapDuration(String dauer, Duration expectedDuration) { + RawFilm film = + RawFilm.builder().sender("BR").titel("Test titel").thema("Test thema").dauer(dauer).build(); + assertThat(classUnderTest.mapDuration(film)).isEqualTo(expectedDuration); + } + + @ParameterizedTest + @MethodSource("gatherTimeArguments") + void gatherTime(String time, LocalTime expectedTime) { + assertThat(classUnderTest.gatherTime(time)).isEqualTo(expectedTime); + } + + @ParameterizedTest + @MethodSource("mapSizeArguments") + void mapSize(String sizeInMbText, Long sizeinMb) { + RawFilm film = + RawFilm.builder() + .sender("BR") + .titel("Test titel") + .thema("Test thema") + .groesseMb(sizeInMbText) + .build(); + assertThat(classUnderTest.mapSize(film)).isEqualTo(sizeinMb); + } + + @ParameterizedTest + @MethodSource("gatherDateArguments") + void gatherDate(String dateText, LocalDate expectedDate) { + RawFilm film = + RawFilm.builder() + .sender("BR") + .titel("Test titel") + .thema("Test thema") + .datum(dateText) + .build(); + if (expectedDate == null) { + assertThat(classUnderTest.gatherDate(film)).isNotPresent(); + } else { + assertThat(classUnderTest.gatherDate(film)).isPresent().get().isEqualTo(expectedDate); + } + } +} diff --git a/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java b/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java index 4c1d50f7..269920a4 100644 --- a/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java +++ b/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java @@ -8,7 +8,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.*; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static jakarta.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; class FileSizeDeterminerTest { @@ -20,8 +20,7 @@ class FileSizeDeterminerTest { public static void setUpWiremock() { wireMockServer.stubFor( head(urlEqualTo("/" + TEST_FILE_NAME)) - .willReturn( - aResponse().withStatus(200).withHeader(CONTENT_LENGTH, "5643"))); + .willReturn(aResponse().withStatus(200).withHeader(CONTENT_LENGTH, "5643"))); } @BeforeEach diff --git a/src/test/resources/TestFilmlist.json b/src/test/resources/TestFilmlist.json index f722a1d3..22132d57 100644 --- a/src/test/resources/TestFilmlist.json +++ b/src/test/resources/TestFilmlist.json @@ -1,7 +1 @@ -{ -"Filmliste": ["20.10.2019, 20:04","20.10.2019, 20:04","0.0.0"," [Vers.: 0.0.0 ]","5f330449-c9d2-4b67-89b4-75373b38b9f8"], -"Filmliste": ["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"], -"X": ["ARD","TestThema","TestTitel","01.01.2017","23:55:00","00:10:00","42","Test beschreibung.","http://example.org/Test.mp4","http://www.example.org/","","","19|klein.mp4","","19|hd.mp4","","1483311300","","","false"], -"X": ["","","TestTitel","01.01.2017","23:55:00","00:10:00","42","Test beschreibung.","http://example.org/Test2.mp4","http://www.example.org/2","","","19|klein2.mp4","","19|hd2.mp4","","1483311300","","","false"], -"X": ["BR","TestThema2","TestTitel","01.01.2017","23:55:00","00:10:00","42","Test beschreibung.","http://example.org/Test.mp4","http://www.example.org/","","","19|klein.mp4","","19|hd.mp4","","1483311300","","","false"] -} +{"Filmliste": ["20.10.2019, 20:04","20.10.2019, 20:04","0.0.0"," [Vers.: 0.0.0 ]","5f330449c9d24b6789b475373b38b9f8"],"Filmliste": ["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"],"X":["ARD","TestThema","TestTitel","01.01.2017","23:55:00","00:10:00","42","Test beschreibung.","http://example.org/Test.mp4","http://www.example.org/","","","19|klein.mp4","","19|hd.mp4","","1483311300","","","false"],"X":["","","TestTitel","01.01.2017","23:55:00","00:10:00","42","Test beschreibung.","http://example.org/Test2.mp4","http://www.example.org/2","","","19|klein2.mp4","","19|hd2.mp4","","1483311300","","","false"],"X":["BR","TestThema2","TestTitel","01.01.2017","23:55:00","00:10:00","42","Test beschreibung.","http://example.org/Test.mp4","http://www.example.org/","","","19|klein.mp4","","19|hd.mp4","","1483311300","","","false"]} \ No newline at end of file diff --git a/src/test/resources/TestFilmlist.json.bz b/src/test/resources/TestFilmlist.json.bz index 75db9038..a2cbd633 100644 Binary files a/src/test/resources/TestFilmlist.json.bz and b/src/test/resources/TestFilmlist.json.bz differ diff --git a/src/test/resources/TestFilmlist.json.gz b/src/test/resources/TestFilmlist.json.gz index b57788e0..3249634d 100644 Binary files a/src/test/resources/TestFilmlist.json.gz and b/src/test/resources/TestFilmlist.json.gz differ diff --git a/src/test/resources/TestFilmlist.json.xz b/src/test/resources/TestFilmlist.json.xz index ea2f0f2c..2e275776 100644 Binary files a/src/test/resources/TestFilmlist.json.xz and b/src/test/resources/TestFilmlist.json.xz differ