diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..e096db6 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,17 @@ +# version 0.0.1-alpha2 +- modifications in schema.sql +-- enlarging field context.origin from VARCHAR(256) to VARCHAR(512) +-- changing type of obsolete.client_name from INT to VARCHAR(256) +-- renaming aggregated_status.number to aggregated_status.number_id and adding field aggregated_status.number_duration +- corrections in script /add-on/transferDB.sql +- corrections in model +-- adding index definition to UrlContext.class +-- adapting AggregatedStatus.class to modified schema +- corrections, modifications and additions in repository +-- changing parameter of HistoryRepository.saveHistoryLinksOlderThan from int periodOfDays to LocalDateTime +-- using custom queries in StatusDetailRepository for methods findAllByCategory and findAllByProvidergroupnameAndCategory +-- changing parameter of StatusRepository.saveStatusLinksOlderThan from int periodOfDays to LocalDateTime +-- using custom query UrlConext.findByUrlAndContextAndExpectedMimeType +- corrections, modifications and additions in service +-- adding methods StatusService.findAllDetail(Category category) and StatusService.findAllDetail(String providergroupname, Category category) +-- modifications in LinkService.class for better performance \ No newline at end of file diff --git a/add-on/transferDB.sh b/add-on/transferDB.sh index 56da613..8ad33c9 100644 --- a/add-on/transferDB.sh +++ b/add-on/transferDB.sh @@ -24,7 +24,7 @@ echo "removing dump file ${DUMP_FILE}" rm ${DUMP_FILE} echo "done removing" echo "dropping database ${MYSQL_DATABASE}" -mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "DROP DATABASE IF EXISTS ;" +mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" echo "done dropping" echo "creating new database ${MYSQL_DATABASE}" mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "CREATE DATABASE ${MYSQL_DATABASE} CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;" diff --git a/pom.xml b/pom.xml index edd415d..f1c647d 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,3 @@ - 4.0.0 @@ -9,12 +8,13 @@ eu.clarin linkchecker-persistence - 0.0.1-alpha1 + 0.0.2 Linkchecker Persistence API UTF-8 17 - 17 + 17 + 4.11.2 @@ -39,7 +39,7 @@ eu.clarin.cmdi vlo-commons - 4.11.1 + ${vlo.version} diff --git a/src/main/java/eu/clarin/linkchecker/persistence/model/AggregatedStatus.java b/src/main/java/eu/clarin/linkchecker/persistence/model/AggregatedStatus.java index e8f793b..84b4c36 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/model/AggregatedStatus.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/model/AggregatedStatus.java @@ -34,6 +34,8 @@ public class AggregatedStatus { @Nullable private Long maxDuration; - private Long number; + private Long numberId; + + private Long numberDuration; } diff --git a/src/main/java/eu/clarin/linkchecker/persistence/model/UrlContext.java b/src/main/java/eu/clarin/linkchecker/persistence/model/UrlContext.java index 7373789..b04a465 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/model/UrlContext.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/model/UrlContext.java @@ -6,6 +6,7 @@ import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; +import javax.persistence.Index; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.Table; @@ -20,7 +21,7 @@ @RequiredArgsConstructor @Data @Entity -@Table(name = "url_context") +@Table(name = "url_context", indexes = @Index(columnList = "url_id, context_id, expectedMimeType", unique = true)) public class UrlContext { @Id diff --git a/src/main/java/eu/clarin/linkchecker/persistence/repository/HistoryRepository.java b/src/main/java/eu/clarin/linkchecker/persistence/repository/HistoryRepository.java index 5cfaf7c..1757fd5 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/repository/HistoryRepository.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/repository/HistoryRepository.java @@ -1,5 +1,7 @@ package eu.clarin.linkchecker.persistence.repository; +import java.time.LocalDateTime; + import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; @@ -9,23 +11,25 @@ public interface HistoryRepository extends PagingAndSortingRepository { @Query( - value = "INSERT INTO obsolete (url_name, client_name, providergroup_name, origin, expected_mime_type, ingestion_date, status_code, message, category, method, content_type, content_length, duration, checking_date, redirect_count) " - + "SELECT u.name, cl.name, p.name, c.origin, uc.expected_mime_type, uc.ingestion_date, h.status_code, h.message, h.category, h.method, h.content_type, h.content_length, h.duration, h.checking_date, h.redirect_count " - + "FROM url_context uc " - + "INNER JOIN (url u) " - + "ON u.id=uc.url_id " - + "INNER JOIN (context c) " - + "ON c.id=uc.context_id " - + "INNER JOIN providergroup p " - + "ON p.id=c.providergroup_id " - + "INNER JOIN history h " - + "ON h.url_id=u.id " - + "INNER JOIN client cl " - + "ON cl.id=c.client_id " - + "WHERE uc.ingestion_date < ?1", + value = """ + INSERT INTO obsolete (url_name, client_name, providergroup_name, origin, expected_mime_type, ingestion_date, status_code, message, category, method, content_type, content_length, duration, checking_date, redirect_count, deletion_date) + SELECT u.name, cl.name, p.name, c.origin, uc.expected_mime_type, uc.ingestion_date, h.status_code, h.message, h.category, h.method, h.content_type, h.content_length, h.duration, h.checking_date, h.redirect_count, NOW() + FROM url_context uc + INNER JOIN (url u) + ON u.id=uc.url_id + INNER JOIN (context c) + ON c.id=uc.context_id + INNER JOIN providergroup p + ON p.id=c.providergroup_id + INNER JOIN history h + ON h.url_id=u.id + INNER JOIN client cl + ON cl.id=c.client_id + WHERE uc.ingestion_date < ?1 + """, nativeQuery = true ) @Modifying - public void saveHistoryLinksOlderThan(int persiodOfDays); + public void saveHistoryLinksOlderThan(LocalDateTime dateTime); } diff --git a/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusDetailRepository.java b/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusDetailRepository.java index b30635f..490c435 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusDetailRepository.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusDetailRepository.java @@ -6,20 +6,47 @@ import java.util.stream.Stream; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import eu.clarin.linkchecker.persistence.model.StatusDetail; import eu.clarin.linkchecker.persistence.model.StatusDetailId; -import eu.clarin.linkchecker.persistence.utils.Category; + /** * */ public interface StatusDetailRepository extends CrudRepository{ - public Stream findAllByCategory(Category category); - - public Stream findAllByProvidergroupnameAndCategory(String providergroupname, Category category); + @Query( + value = """ + SELECT NULL AS order_nr, s.*, u.name AS urlname, p.name AS providergroupname, c.origin, uc.expected_mime_type + FROM status s + INNER JOIN url u ON s.url_id = u.id + INNER JOIN url_context uc ON uc.url_id = u.id + INNER JOIN context c ON c.id = uc.context_id + INNER JOIN providergroup p ON p.id = c.providergroup_id + WHERE s.category = ?1 + AND uc.active = true + """, + nativeQuery = true + ) + public Stream findAllByCategory(String categoryName); + @Query( + value = """ + SELECT NULL AS order_nr, s.*, u.name AS urlname, p.name AS providergroupname, c.origin, uc.expected_mime_type + FROM status s + INNER JOIN url u ON s.url_id = u.id + INNER JOIN url_context uc ON uc.url_id = u.id + INNER JOIN context c ON c.id = uc.context_id + INNER JOIN providergroup p ON p.id = c.providergroup_id + WHERE s.category = ?2 + AND p.name = ?1 + AND uc.active = true + """, + nativeQuery = true + ) + public Stream findAllByProvidergroupnameAndCategory(String providergroupname, String categoryName); public Stream findByOrderNrLessThanEqual(Long orderNr); diff --git a/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusRepository.java b/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusRepository.java index ed1df00..379dbf3 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusRepository.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/repository/StatusRepository.java @@ -1,5 +1,6 @@ package eu.clarin.linkchecker.persistence.repository; +import java.time.LocalDateTime; import java.util.Optional; import java.util.stream.Stream; @@ -51,6 +52,6 @@ INNER JOIN (context c) nativeQuery = true ) @Modifying - public void saveStatusLinksOlderThan(int periodOfDays); + public void saveStatusLinksOlderThan(LocalDateTime dateTime); } diff --git a/src/main/java/eu/clarin/linkchecker/persistence/repository/UrlContextRepository.java b/src/main/java/eu/clarin/linkchecker/persistence/repository/UrlContextRepository.java index 4f8c4a3..9343846 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/repository/UrlContextRepository.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/repository/UrlContextRepository.java @@ -6,6 +6,9 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.query.Param; + +import org.springframework.transaction.annotation.Transactional; import eu.clarin.linkchecker.persistence.model.Context; import eu.clarin.linkchecker.persistence.model.Url; @@ -14,6 +17,17 @@ public interface UrlContextRepository extends CrudRepository { public Optional findByUrlAndContextAndExpectedMimeType(Url url, Context context, String expectedMimeType); + @Modifying() + @Transactional() + @Query( + value = """ + INSERT INTO url_context(url_id, context_id, expected_mime_type, active, ingestion_date) + VALUES(:urlId, :contextId, :expectedMimeType, TRUE, :ingestionDate) + ON DUPLICATE KEY UPDATE active=TRUE, ingestion_date=:ingestionDate + """, + nativeQuery = true + ) + public void insertOrUpdate(@Param("urlId") Long urlId, @Param("contextId") Long contextId, @Param("expectedMimeType") String expectedMimeType, @Param("ingestionDate") LocalDateTime ingestionDate); @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE UrlContext uc SET uc.active = false WHERE uc.active = true AND uc.ingestionDate < ?1") diff --git a/src/main/java/eu/clarin/linkchecker/persistence/service/LinkService.java b/src/main/java/eu/clarin/linkchecker/persistence/service/LinkService.java index d72e73b..cf66eef 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/service/LinkService.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/service/LinkService.java @@ -1,14 +1,19 @@ package eu.clarin.linkchecker.persistence.service; import java.time.LocalDateTime; +import java.util.Collection; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import javax.transaction.Transactional; - import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Transactional; import eu.clarin.linkchecker.persistence.model.*; import eu.clarin.linkchecker.persistence.repository.ContextRepository; @@ -23,6 +28,7 @@ import lombok.extern.slf4j.Slf4j; @Service +@Scope("prototype") @Slf4j public class LinkService { @@ -41,10 +47,35 @@ public class LinkService { @Autowired private HistoryRepository hRep; + private Map providergroupMap = new ConcurrentHashMap(); + //locks + private Set urlLock = ConcurrentHashMap.newKeySet(); + private Set contextLock = ConcurrentHashMap.newKeySet(); + + public void save(Client client, String urlString, String origin, String providerGroupName, String expectedMimeType) { save(client, urlString, origin, providerGroupName, expectedMimeType, LocalDateTime.now()); } + @Transactional(isolation = Isolation.READ_UNCOMMITTED) + public void savePerOrigin(Client client, String providergroupName, String origin, Collection> urlMimes) { + log.trace("insert or update providergroup"); + Providergroup providergroup = (providergroupName == null)?null: getProvidergroup(providergroupName); + log.trace("insert or update context"); + Context context = getContext(origin, providergroup, client); + + urlMimes.forEach(urlMime -> { + String urlName = urlMime.getFirst().trim(); + + ValidationResult validation = UrlValidator.validate(urlName); + log.trace("insert or update url"); + Url url = getUrl(urlName, validation, LocalDateTime.now()); + log.trace("insert or update url_context"); + insertOrUpdateUrlContext(url.getId(), context.getId(), urlMime.getSecond(), LocalDateTime.now()); + }); + log.trace("done insert or update url_context"); + } + @Transactional public void save(Client client, String urlName, String origin, String providergroupName, String expectedMimeType, LocalDateTime ingestionDate) { @@ -52,58 +83,82 @@ public void save(Client client, String urlName, String origin, String providergr ValidationResult validation = UrlValidator.validate(urlName); - Url url = getUrl(urlName, validation); - + log.trace("insert or update url"); + Url url = getUrl(urlName, validation, ingestionDate); + log.trace("insert or update providergroup"); Providergroup providergroup = (providergroupName == null)?null: getProvidergroup(providergroupName); - + log.trace("insert or update context"); Context context = getContext(origin, providergroup, client); - - getUrlContext(url, context, expectedMimeType, ingestionDate); - - - if(!validation.isValid()) { //create a status entry if Url is not valid - Status status = new Status(url, Category.Invalid_URL, validation.getMessage(), ingestionDate); - - sService.save(status); - } - } - - private synchronized Url getUrl(String urlName, ValidationResult validation) { - - return uRep.findByName(urlName) - .orElseGet(() -> uRep.save(new Url(urlName, validation.getHost(), validation.isValid()))); - + log.trace("insert or update url_context"); + insertOrUpdateUrlContext(url.getId(), context.getId(), expectedMimeType, ingestionDate); + log.trace("done insert or update url_context"); } - private synchronized Providergroup getProvidergroup(String providergroupName) { - - return pRep.findByName(providergroupName) - .orElseGet(() -> pRep.save(new Providergroup(providergroupName))); - + private Url getUrl(String urlName, ValidationResult validation, LocalDateTime ingestionDate) { + try { + while(!this.urlLock.add(urlName)) { + + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + + log.error("InterruptedException while waiting for unlock"); + } + } + return uRep.findByName(urlName) + .orElseGet(() -> { + Url url = uRep.save(new Url(urlName, validation.getHost(), validation.isValid())); + + if(!validation.isValid()) { //create a status entry if Url is not valid + Status status = new Status(url, Category.Invalid_URL, validation.getMessage(), ingestionDate); + sService.save(status); + } + + return url; + }); + } + finally{ + this.urlLock.remove(urlName); + } + } - - private synchronized Context getContext(String origin, Providergroup providergroup, Client client){ - - return cRep.findByOriginAndProvidergroupAndClient(origin, providergroup, client) - .orElseGet(() -> cRep.save(new Context(origin, providergroup, client))); - + + private Providergroup getProvidergroup(String providergroupName) { + + return this.providergroupMap.computeIfAbsent(providergroupName, + k-> pRep.findByName(providergroupName).orElseGet(() -> pRep.save(new Providergroup(providergroupName)))); } - private synchronized UrlContext getUrlContext(Url url, Context context, String expectedMimeType, LocalDateTime ingestionDate) { - - return ucRep.findByUrlAndContextAndExpectedMimeType(url, context, expectedMimeType) - .or(() -> Optional.of(new UrlContext(url, context, ingestionDate, true))) - .map(urlContext -> { - urlContext.setExpectedMimeType(expectedMimeType); - urlContext.setIngestionDate(ingestionDate); - urlContext.setActive(true); + private Context getContext(String origin, Providergroup providergroup, Client client){ + + try { + while(!this.contextLock.add(origin)) { + + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { - return ucRep.save(urlContext); - }) - .get(); + log.error("InterruptedException while waiting for unlock"); + } + } + + return cRep.findByOriginAndProvidergroupAndClient(origin, providergroup, client) + .orElseGet(() -> cRep.save(new Context(origin, providergroup, client))); + + } + finally { + + this.contextLock.remove(origin); + } } + private void insertOrUpdateUrlContext(Long urlId, Long contextId, String expectedMimeType, LocalDateTime ingestionDate) { + ucRep.insertOrUpdate(urlId, contextId, expectedMimeType, ingestionDate); + } + @Transactional public void deactivateLinksOlderThan(int periodInDays) { @@ -113,20 +168,22 @@ public void deactivateLinksOlderThan(int periodInDays) { @Transactional public void deleteLinksOderThan(int periodInDays) { - log.info("multi step deletion of links older then {} days", periodInDays); + LocalDateTime oldestDate = LocalDateTime.now().minusDays(periodInDays); + + log.info("multi step deletion of links older then {} days (date: {})", periodInDays, oldestDate); int step = 1; log.info("step {}: saving status records", step); - sRep.saveStatusLinksOlderThan(periodInDays); + sRep.saveStatusLinksOlderThan(oldestDate); log.info("step {}: done", step++); log.info("step {}: saving history records", step); - hRep.saveHistoryLinksOlderThan(periodInDays); + hRep.saveHistoryLinksOlderThan(oldestDate); log.info("step {}: done", step++); log.info("step {}: deleting url_context records", step); - ucRep.deleteOlderThan(LocalDateTime.now().minusDays(periodInDays)); + ucRep.deleteOlderThan(oldestDate); log.info("step {}: done", step++); log.info("step {}: deleting url records with delete cascade to status and history records", step); diff --git a/src/main/java/eu/clarin/linkchecker/persistence/service/StatusService.java b/src/main/java/eu/clarin/linkchecker/persistence/service/StatusService.java index ce5d976..ff54f2d 100644 --- a/src/main/java/eu/clarin/linkchecker/persistence/service/StatusService.java +++ b/src/main/java/eu/clarin/linkchecker/persistence/service/StatusService.java @@ -3,6 +3,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; import javax.transaction.Transactional; @@ -12,7 +13,9 @@ import eu.clarin.cmdi.vlo.PIDUtils; import eu.clarin.linkchecker.persistence.model.*; import eu.clarin.linkchecker.persistence.repository.HistoryRepository; +import eu.clarin.linkchecker.persistence.repository.StatusDetailRepository; import eu.clarin.linkchecker.persistence.repository.StatusRepository; +import eu.clarin.linkchecker.persistence.utils.Category; @Service @Transactional @@ -21,6 +24,8 @@ public class StatusService { StatusRepository sRep; @Autowired HistoryRepository hRep; + @Autowired + StatusDetailRepository sdRep; @Transactional @@ -59,4 +64,12 @@ public Map getStatus(String... urlNames){ return map; } + @Transactional + public Stream findAllDetail(Category category){ + return sdRep.findAllByCategory(category.name()); + } + @Transactional + public Stream findAllDetail(String providergroupname, Category category){ + return sdRep.findAllByProvidergroupnameAndCategory(providergroupname, category.name()); + } } diff --git a/src/main/resources/schema.sql b/src/main/resources/schema.sql index 64678f6..da0ff9d 100644 --- a/src/main/resources/schema.sql +++ b/src/main/resources/schema.sql @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS `client` ( CREATE TABLE IF NOT EXISTS `context` ( `id` INT NOT NULL AUTO_INCREMENT, `client_id` INT NOT NULL, - `origin` VARCHAR(256) NOT NULL, + `origin` VARCHAR(512) NOT NULL, `providergroup_id` INT DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY (`origin`, `providergroup_id`, `client_id`), @@ -105,7 +105,7 @@ CREATE TABLE IF NOT EXISTS `history` ( CREATE TABLE IF NOT EXISTS `obsolete` ( `id` INT NOT NULL AUTO_INCREMENT, `url_name` VARCHAR(512) NOT NULL, - `client_name` INT DEFAULT NULL, + `client_name` VARCHAR(256) DEFAULT NULL, `providergroup_name` VARCHAR(256) DEFAULT NULL, `origin` VARCHAR(256) DEFAULT NULL, `expected_mime_type` VARCHAR(256) DEFAULT NULL, @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS `obsolete` ( `category` VARCHAR(25) DEFAULT NULL, `method` VARCHAR(10) DEFAULT NULL, `content_type` VARCHAR(256) DEFAULT NULL, - `content_length` bigint DEFAULT NULL, + `content_length` BIGINT DEFAULT NULL, `duration` INT DEFAULT NULL, `checking_date` DATETIME DEFAULT NULL, `redirect_count` INT DEFAULT NULL, @@ -125,7 +125,7 @@ CREATE TABLE IF NOT EXISTS `obsolete` ( CREATE VIEW IF NOT EXISTS `aggregated_status` AS - SELECT p.name, s.category, COUNT(s.id) AS number, AVG(s.duration) AS avg_duration, MAX(s.duration) AS max_duration + SELECT p.name, s.category, COUNT(s.id) AS number_id, COUNT(s.duration) AS number_duration, AVG(s.duration) AS avg_duration, MAX(s.duration) AS max_duration FROM url_context uc JOIN (status s) ON (uc.url_id=s.url_id) diff --git a/src/test/java/eu/clarin/linkchecker/persistence/repositories/AggregatedStatusRepositoryTests.java b/src/test/java/eu/clarin/linkchecker/persistence/repositories/AggregatedStatusRepositoryTests.java index 5a20eca..6998521 100644 --- a/src/test/java/eu/clarin/linkchecker/persistence/repositories/AggregatedStatusRepositoryTests.java +++ b/src/test/java/eu/clarin/linkchecker/persistence/repositories/AggregatedStatusRepositoryTests.java @@ -88,7 +88,7 @@ void findAllByProviderGroupName() { stream.forEach(ag -> { assertEquals( this.statusMap.get(pgName).stream().filter(s -> s.getCategory() == ag.getCategory()).count(), - ag.getNumber() + ag.getNumberId() ); }); } diff --git a/src/test/java/eu/clarin/linkchecker/persistence/services/LinkServiceTests.java b/src/test/java/eu/clarin/linkchecker/persistence/services/LinkServiceTests.java index 7abb75f..7dd1d9b 100644 --- a/src/test/java/eu/clarin/linkchecker/persistence/services/LinkServiceTests.java +++ b/src/test/java/eu/clarin/linkchecker/persistence/services/LinkServiceTests.java @@ -3,6 +3,8 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationContext; +import org.springframework.data.util.Pair; import eu.clarin.linkchecker.persistence.model.Client; import eu.clarin.linkchecker.persistence.model.Role; @@ -13,9 +15,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.time.LocalDateTime; +import java.util.Collection; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; import java.util.stream.IntStream; @SpringBootTest @@ -24,16 +28,18 @@ class LinkServiceTests extends RepositoryTests{ @Autowired - private LinkService lService; + ApplicationContext ctx; @Test void save() { + + LinkService lService = ctx.getBean(LinkService.class); Client client = usRep.save(new Client("wowasa", UUID.randomUUID().toString(), Role.ADMIN)); IntStream.range(0, 3).forEach(i -> { - lService.save(client, "http://www.wowasa.com?page=0", "origin0", null, null); + lService.save(client, "http://www.wowasa.com?page=0", "origin0", null, "application/xml"); }); @@ -75,7 +81,7 @@ void save() { @Test void multithreadedSave() { - + LinkService lService = ctx.getBean(LinkService.class); Client client = usRep.save(new Client("wowasa", "xxxxxxxx", Role.ADMIN)); @@ -111,10 +117,71 @@ void multithreadedSave() { assertEquals(15, cRep.count()); } + + @Test + void savePerOrigin() { + + LinkService lService = ctx.getBean(LinkService.class); + + Client client = usRep.save(new Client("wowasa", UUID.randomUUID().toString(), Role.ADMIN)); + + Collection> urlMimes = IntStream + .range(0, 5) + .mapToObj(i -> Pair.of("http://www.wowasa.com?page=" +i, "application/jpg")) + .collect(Collectors.toList()); + + IntStream.range(0, 3).forEach(originNr -> { + + lService.savePerOrigin(client, "pg1", "origin" + originNr, urlMimes); + + }); + + assertEquals(5, uRep.count()); + assertEquals(1, pRep.count()); + assertEquals(3, cRep.count()); + } + + @Test + void multithreadedSavePerOrigin() { + + LinkService lService = ctx.getBean(LinkService.class); + + Client client = usRep.save(new Client("wowasa", UUID.randomUUID().toString(), Role.ADMIN)); + + Collection> urlMimes = IntStream + .range(0, 5) + .mapToObj(i -> Pair.of("http://www.wowasa.com?page=" +i, "application/jpg")) + .collect(Collectors.toList()); + + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(30); + + IntStream.range(0, 30).forEach(threadNr -> { + + executor.submit(() -> { + lService.savePerOrigin(client, "pg1", "origin" + threadNr, urlMimes); + }); + }); + + while(executor.getActiveCount() > 0) { + try { + Thread.sleep(5000); + } + catch (InterruptedException e) { + + log.error("", e); + } + } + + assertEquals(5, uRep.count()); + assertEquals(1, pRep.count()); + assertEquals(30, cRep.count()); + } @Test void deactivateLinksOlderThan() { + LinkService lService = ctx.getBean(LinkService.class); + Client client = usRep.save(new Client("wowasa", "xxxxxxxx", Role.ADMIN)); IntStream.range(0, 15).forEach(i -> { @@ -133,6 +200,8 @@ void deactivateLinksOlderThan() { @Test void deleteLinksOlderThan() { + LinkService lService = ctx.getBean(LinkService.class); + Client client = usRep.save(new Client("wowasa", "xxxxxxxx", Role.ADMIN)); IntStream.range(0, 100).forEach(i -> {