Merge pull request #2 from clarin-eric/0.0.2-rc1

0.0.2 rc1

CHANGES.md

@@ -0,0 +1,17 @@
+# version 0.0.1-alpha2
+- modifications in schema.sql
+-- enlarging field context.origin from VARCHAR(256) to VARCHAR(512)
+-- changing type of obsolete.client_name from INT to VARCHAR(256)
+-- renaming  aggregated_status.number to aggregated_status.number_id and adding field  aggregated_status.number_duration
+- corrections in script /add-on/transferDB.sql
+- corrections in model
+-- adding index definition to UrlContext.class
+-- adapting AggregatedStatus.class to modified schema
+- corrections, modifications and additions in repository
+-- changing parameter of HistoryRepository.saveHistoryLinksOlderThan from int periodOfDays to LocalDateTime
+-- using custom queries in StatusDetailRepository for methods findAllByCategory and findAllByProvidergroupnameAndCategory
+-- changing parameter of StatusRepository.saveStatusLinksOlderThan from int periodOfDays to LocalDateTime
+-- using custom query UrlConext.findByUrlAndContextAndExpectedMimeType
+- corrections, modifications and additions in service
+-- adding methods StatusService.findAllDetail(Category category) and StatusService.findAllDetail(String providergroupname, Category category)
+-- modifications in LinkService.class for better performance

add-on/transferDB.sh

@@ -24,7 +24,7 @@ echo "removing dump file ${DUMP_FILE}"
 rm ${DUMP_FILE}
 echo "done removing"
 echo "dropping database ${MYSQL_DATABASE}"
-mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "DROP DATABASE IF EXISTS ;"
+mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};"
 echo "done dropping"
 echo "creating new database ${MYSQL_DATABASE}"
 mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "CREATE DATABASE ${MYSQL_DATABASE} CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;"

pom.xml

@@ -1,4 +1,3 @@
-<?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
@@ -9,12 +8,13 @@
    </parent>
    <groupId>eu.clarin</groupId>
    <artifactId>linkchecker-persistence</artifactId>
-   <version>0.0.1-alpha1</version>
+   <version>0.0.2</version>
    <name>Linkchecker Persistence API</name>
    <properties>
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <maven.compiler.source>17</maven.compiler.source>
-      <maven.compiler.target>17</maven.compiler.target>      
+      <maven.compiler.target>17</maven.compiler.target>
+      <vlo.version>4.11.2</vlo.version>
    </properties>
    <dependencies>
       <dependency>
@@ -39,7 +39,7 @@
       <dependency>
       	<groupId>eu.clarin.cmdi</groupId>
       	<artifactId>vlo-commons</artifactId>
-      	<version>4.11.1</version>
+      	<version>${vlo.version}</version>
       </dependency>
    </dependencies>
    <distributionManagement>

src/main/java/eu/clarin/linkchecker/persistence/model/AggregatedStatus.java

@@ -34,6 +34,8 @@ public class AggregatedStatus {
    @Nullable
    private Long maxDuration;
 
-   private Long number;
+   private Long numberId;
+
+   private Long numberDuration;
 
 }

src/main/java/eu/clarin/linkchecker/persistence/model/UrlContext.java

@@ -6,6 +6,7 @@
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
+import javax.persistence.Index;
 import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
 import javax.persistence.Table;
@@ -20,7 +21,7 @@
 @RequiredArgsConstructor
 @Data
 @Entity
-@Table(name = "url_context")
+@Table(name = "url_context", indexes = @Index(columnList = "url_id, context_id, expectedMimeType", unique = true))
 public class UrlContext {
 
    @Id

src/main/java/eu/clarin/linkchecker/persistence/repository/HistoryRepository.java

@@ -1,5 +1,7 @@
 package eu.clarin.linkchecker.persistence.repository;
 
+import java.time.LocalDateTime;
+
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.PagingAndSortingRepository;
@@ -9,23 +11,25 @@
 public interface HistoryRepository extends PagingAndSortingRepository<History, Long> {
 
    @Query(
-         value = "INSERT INTO obsolete (url_name, client_name, providergroup_name, origin, expected_mime_type, ingestion_date, status_code, message, category, method, content_type, content_length, duration, checking_date, redirect_count) "
-               + "SELECT u.name, cl.name, p.name, c.origin, uc.expected_mime_type, uc.ingestion_date, h.status_code, h.message, h.category, h.method, h.content_type, h.content_length, h.duration, h.checking_date, h.redirect_count "
-               + "FROM url_context uc "
-               + "INNER JOIN (url u) "
-               + "ON u.id=uc.url_id "
-               + "INNER JOIN (context c) "
-               + "ON c.id=uc.context_id "
-               + "INNER JOIN providergroup p "
-               + "ON p.id=c.providergroup_id "
-               + "INNER JOIN history h "
-               + "ON h.url_id=u.id "
-               + "INNER JOIN client cl "
-               + "ON cl.id=c.client_id "
-               + "WHERE uc.ingestion_date < ?1",
+         value = """
+               INSERT INTO obsolete (url_name, client_name, providergroup_name, origin, expected_mime_type, ingestion_date, status_code, message, category, method, content_type, content_length, duration, checking_date, redirect_count, deletion_date) 
+               SELECT u.name, cl.name, p.name, c.origin, uc.expected_mime_type, uc.ingestion_date, h.status_code, h.message, h.category, h.method, h.content_type, h.content_length, h.duration, h.checking_date, h.redirect_count, NOW() 
+               FROM url_context uc 
+               INNER JOIN (url u) 
+               ON u.id=uc.url_id 
+               INNER JOIN (context c) 
+               ON c.id=uc.context_id 
+               INNER JOIN providergroup p 
+               ON p.id=c.providergroup_id 
+               INNER JOIN history h 
+               ON h.url_id=u.id 
+               INNER JOIN client cl 
+               ON cl.id=c.client_id 
+               WHERE uc.ingestion_date < ?1
+               """,
          nativeQuery = true
       )
    @Modifying
-   public void saveHistoryLinksOlderThan(int persiodOfDays);
+   public void saveHistoryLinksOlderThan(LocalDateTime dateTime);
 
 }

src/main/java/eu/clarin/linkchecker/persistence/repository/StatusDetailRepository.java

@@ -6,20 +6,47 @@
 
 import java.util.stream.Stream;
 
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.CrudRepository;
 
 import eu.clarin.linkchecker.persistence.model.StatusDetail;
 import eu.clarin.linkchecker.persistence.model.StatusDetailId;
-import eu.clarin.linkchecker.persistence.utils.Category;
+
 
 /**
  *
  */
 public interface StatusDetailRepository extends CrudRepository<StatusDetail, StatusDetailId>{
 
-   public Stream<StatusDetail> findAllByCategory(Category category);
-
-   public Stream<StatusDetail> findAllByProvidergroupnameAndCategory(String providergroupname, Category category);
+   @Query(
+      value = """
+         SELECT NULL AS order_nr, s.*, u.name AS urlname, p.name AS providergroupname, c.origin, uc.expected_mime_type
+            FROM status s 
+            INNER JOIN url u ON s.url_id = u.id 
+            INNER JOIN url_context uc ON uc.url_id = u.id
+            INNER JOIN context c ON c.id = uc.context_id
+            INNER JOIN providergroup p ON p.id = c.providergroup_id
+            WHERE s.category = ?1
+            AND uc.active = true       
+            """,
+      nativeQuery = true
+   )
+   public Stream<StatusDetail> findAllByCategory(String categoryName);
+   @Query(
+      value = """
+         SELECT NULL AS order_nr, s.*, u.name AS urlname, p.name AS providergroupname, c.origin, uc.expected_mime_type
+            FROM status s 
+            INNER JOIN url u ON s.url_id = u.id 
+            INNER JOIN url_context uc ON uc.url_id = u.id
+            INNER JOIN context c ON c.id = uc.context_id
+            INNER JOIN providergroup p ON p.id = c.providergroup_id
+            WHERE s.category = ?2
+            AND p.name = ?1
+            AND uc.active = true       
+            """,
+      nativeQuery = true
+   )
+   public Stream<StatusDetail> findAllByProvidergroupnameAndCategory(String providergroupname, String categoryName);
 
    public Stream<StatusDetail> findByOrderNrLessThanEqual(Long orderNr);
 

src/main/java/eu/clarin/linkchecker/persistence/repository/StatusRepository.java

@@ -1,5 +1,6 @@
 package eu.clarin.linkchecker.persistence.repository;
 
+import java.time.LocalDateTime;
 import java.util.Optional;
 import java.util.stream.Stream;
 
@@ -51,6 +52,6 @@ INNER JOIN (context c)
          nativeQuery = true
       )
    @Modifying
-   public void saveStatusLinksOlderThan(int periodOfDays);
+   public void saveStatusLinksOlderThan(LocalDateTime dateTime);
 
 }

src/main/java/eu/clarin/linkchecker/persistence/repository/UrlContextRepository.java

@@ -6,6 +6,9 @@
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.CrudRepository;
+import org.springframework.data.repository.query.Param;
+
+import org.springframework.transaction.annotation.Transactional;
 
 import eu.clarin.linkchecker.persistence.model.Context;
 import eu.clarin.linkchecker.persistence.model.Url;
@@ -14,6 +17,17 @@
 public interface UrlContextRepository extends CrudRepository<UrlContext, Long> {
 
    public Optional<UrlContext> findByUrlAndContextAndExpectedMimeType(Url url, Context context, String expectedMimeType);
+   @Modifying()
+   @Transactional()
+   @Query(
+         value = """
+               INSERT INTO url_context(url_id, context_id, expected_mime_type, active, ingestion_date)
+               VALUES(:urlId, :contextId, :expectedMimeType, TRUE, :ingestionDate)
+               ON DUPLICATE KEY UPDATE active=TRUE, ingestion_date=:ingestionDate
+            """,
+         nativeQuery = true
+      )
+   public void insertOrUpdate(@Param("urlId") Long urlId, @Param("contextId") Long contextId, @Param("expectedMimeType") String expectedMimeType, @Param("ingestionDate") LocalDateTime ingestionDate);
 
    @Modifying(clearAutomatically = true, flushAutomatically = true)
    @Query("UPDATE UrlContext uc SET uc.active = false WHERE uc.active = true AND uc.ingestionDate < ?1")