Skip to content

Commit

Permalink
WIP: add tracing to debug the CI issue
Browse files Browse the repository at this point in the history
  • Loading branch information
@Pfeil
Pfeil committed Jan 10, 2025
1 parent 04bff17 commit 7349a0c
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 19 deletions.
2 changes: 1 addition & 1 deletion src/main/java/edu/kit/datamanager/pit/Application.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public class Application {
protected static final String ERROR_COMMUNICATION = "Communication error: {}";
protected static final String ERROR_CONFIGURATION = "Configuration error: {}";

public static final Executor EXECUTOR = Executors.newVirtualThreadPerTaskExecutor();
public static final Executor EXECUTOR = Executors.newSingleThreadExecutor();


@Bean
Expand Down
17 changes: 16 additions & 1 deletion src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ public void validate(PIDRecord pidRecord)
.map(this.typeRegistry::queryAttributeInfo)
// validate values using schema
.map(attributeInfoFuture -> attributeInfoFuture.thenApply(attributeInfo -> {
LOG.trace("Validating attribute {} with {} values.",
attributeInfo.pid(),
pidRecord.getPropertyValues(attributeInfo.pid()).length);
for (String value : pidRecord.getPropertyValues(attributeInfo.pid())) {
LOG.trace("Validating value {} for attribute {}.", value, attributeInfo.pid());
boolean isValid = attributeInfo.validate(value);
LOG.trace("Value {} for attribute {} is valid: {}.", value, attributeInfo.pid(), isValid);
if (!isValid) {
throw new RecordValidationException(
pidRecord,
Expand All @@ -68,12 +73,18 @@ public void validate(PIDRecord pidRecord)
}))
// resolve profiles and apply their validation
.map(attributeInfoFuture -> attributeInfoFuture.thenApply(attributeInfo -> {
LOG.trace("Checking if attribute {} is a profile key.", attributeInfo.pid());
boolean indicatesProfileValue = this.profileKeys.contains(attributeInfo.pid());
if (indicatesProfileValue) {
LOG.trace("Attribute {} is a profile key.", attributeInfo.pid());
Arrays.stream(pidRecord.getPropertyValues(attributeInfo.pid()))
.map(this.typeRegistry::queryAsProfile)
.map(profilePid -> {
return this.typeRegistry.queryAsProfile(profilePid);
})
.forEach(registeredProfileFuture -> registeredProfileFuture.thenApply(registeredProfile -> {
LOG.trace("Validating profile {}.", registeredProfile.pid());
registeredProfile.validateAttributes(pidRecord, this.alwaysAcceptAdditionalAttributes);
LOG.trace("Profile {} is valid.", registeredProfile.pid());
return registeredProfile;
}));
}
Expand All @@ -83,9 +94,13 @@ public void validate(PIDRecord pidRecord)


try {
LOG.trace("Waiting for all attribute info futures to complete.");
CompletableFuture.allOf(attributeInfoFutures.toArray(new CompletableFuture<?>[0])).join();
LOG.trace("All attribute info futures completed.");
} catch (CompletionException e) {
LOG.trace("Exception occurred during validation. Unpack Exception, if required.", e);
unpackAsyncExceptions(e);
LOG.trace("Exception was not unpacked. Rethrowing.", e);
throw new ExternalServiceException(this.typeRegistry.getRegistryIdentifier());
} catch (CancellationException e) {
unpackAsyncExceptions(e);
Expand Down
25 changes: 20 additions & 5 deletions src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@
import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.web.client.RestClient;

import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.http.HttpClient;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -52,7 +54,14 @@ public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGen
} catch (URISyntaxException e) {
throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);
}
this.http = RestClient.builder().baseUrl(baseUri).build();
HttpClient httpClient = HttpClient.newBuilder()
.followRedirects(HttpClient.Redirect.NORMAL)
.connectTimeout(Duration.ofSeconds(10))
.build();
this.http = RestClient.builder()
.baseUrl(baseUri)
.requestFactory(new JdkClientHttpRequestFactory(httpClient))
.build();

// TODO better name caching properties (and consider extending them)
int maximumSize = properties.getMaximumSize();
Expand All @@ -77,10 +86,10 @@ public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGen
.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
.removalListener((key, value, cause) ->
LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
)
.buildAsync(attributePid -> {
LOG.trace("Loading profile {} to cache.", attributePid);
LOG.trace("Loading attribute {} to cache.", attributePid);
return this.queryAttribute(attributePid);
});
}
Expand Down Expand Up @@ -187,12 +196,18 @@ protected RegisteredProfile extractProfileInformation(String profilePid, JsonNod

@Override
public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
return this.attributeCache.get(attributePid);
LOG.trace("Querying attribute info for {}", attributePid);
CompletableFuture<AttributeInfo> attributeInfoFuture = this.attributeCache.get(attributePid);;
LOG.trace("Finished querying attribute info for {}", attributePid);
return attributeInfoFuture;
}

@Override
public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
return this.profileCache.get(profilePid);
LOG.trace("Querying profile for {}", profilePid);
CompletableFuture<RegisteredProfile> result = this.profileCache.get(profilePid);
LOG.trace("Finished querying profile for {}", profilePid);
return result;
}

@Override
Expand Down
40 changes: 28 additions & 12 deletions src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,9 @@
import java.util.Optional;
import java.util.stream.Stream;

import edu.kit.datamanager.pit.common.PidAlreadyExistsException;
import edu.kit.datamanager.pit.common.PidNotFoundException;
import edu.kit.datamanager.pit.common.*;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.configuration.PidGenerationProperties;
import edu.kit.datamanager.pit.common.RecordValidationException;
import edu.kit.datamanager.pit.domain.PIDRecord;
import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticRepository;
import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticWrapper;
Expand Down Expand Up @@ -90,32 +88,37 @@ public ResponseEntity<PIDRecord> createPID(
final UriComponentsBuilder uriBuilder
) throws IOException {
LOG.info("Creating PID");
LOG.trace("tracing works");

if (dryrun) {
pidRecord.setPid("dryrun");
} else {
setPid(pidRecord);
}

LOG.trace("Start validation of PID record.");
this.typingService.validate(pidRecord);

if (dryrun) {
// dryrun only does validation. Stop now and return as we would later on.
return ResponseEntity.status(HttpStatus.OK).eTag(quotedEtag(pidRecord)).body(pidRecord);
}

LOG.trace("Registering PID.");
String pid = this.typingService.registerPid(pidRecord);
pidRecord.setPid(pid);

if (applicationProps.getStorageStrategy().storesModified()) {
LOG.trace("Storing PID locally.");
storeLocally(pid, true);
}
PidRecordMessage message = PidRecordMessage.creation(
pid,
"", // TODO parameter is depricated and will be removed soon.
"", // TODO parameter is deprecated and will be removed soon.
AuthenticationHelper.getPrincipal(),
ControllerUtils.getLocalHostname());
try {
LOG.trace("Sending message to messaging service.");
this.messagingService.send(message);
} catch (Exception e) {
LOG.error("Could not notify messaging service about the following message: {}", message);
Expand All @@ -133,27 +136,39 @@ private void setPid(PIDRecord pidRecord) throws IOException {
boolean allowsCustomPids = pidGenerationProperties.isCustomClientPidsEnabled();

if (allowsCustomPids && hasCustomPid) {
// in this only case, we do not have to generate a PID
LOG.trace("Custom PID is allowed and given. No need to generate a PID.");
// in this only case, we do not have to generate a PID,
// but we have to check if the PID is already registered and return an error if so
String prefix = this.typingService.getPrefix().orElseThrow(() -> new IOException("No prefix configured."));
String prefix = this.typingService.getPrefix()
.orElseThrow(() -> new InvalidConfigException("No prefix configured."));
String maybeSuffix = pidRecord.getPid();
String pid = PidSuffix.asPrefixedChecked(maybeSuffix, prefix);
LOG.trace("Prefix: {}, Suffix: {}, PID: {}", prefix, maybeSuffix, pid);
boolean isRegisteredPid = this.typingService.isPidRegistered(pid);
if (isRegisteredPid) {
throw new PidAlreadyExistsException(pidRecord.getPid());
}
} else {
LOG.trace("Custom PID is not allowed or not given. Generating PID.");
// In all other (usual) cases, we have to generate a PID.
// We store only the suffix in the pid field.
// The registration at the PID service will preprend the prefix.

// The registration at the PID service will prepend the prefix.
long maximumGeneratedSuffixes = 100;
Stream<PidSuffix> suffixStream = suffixGenerator.infiniteStream();
Optional<PidSuffix> maybeSuffix = Streams.failableStream(suffixStream)
// With failible streams, we can throw exceptions.
.filter(suffix -> !this.typingService.isPidRegistered(suffix))
.filter(suffix -> {
LOG.trace("Checking if PID suffix is registered: {}", suffix);
return !this.typingService.isPidRegistered(suffix);
})
.stream() // back to normal java streams
.findFirst(); // as the stream is infinite, we should always find a prefix.
PidSuffix suffix = maybeSuffix.orElseThrow(() -> new IOException("Could not generate PID suffix."));
.limit(maximumGeneratedSuffixes)
.findFirst();
PidSuffix suffix = maybeSuffix
.orElseThrow(() -> new ExternalServiceException(
"Could not generate PID suffix which did not exist yet. Tried "
+ maximumGeneratedSuffixes
+ " times."
));
pidRecord.setPid(suffix.get());
}
}
Expand Down Expand Up @@ -263,6 +278,7 @@ public ResponseEntity<PIDRecord> getRecord(
}

private void saveToElastic(PIDRecord rec) {
LOG.trace("Saving PID record to elastic search.");
this.elastic.ifPresent(
database -> database.save(
new PidRecordElasticWrapper(rec, typingService.getOperations())
Expand Down
1 change: 1 addition & 0 deletions src/test/resources/test/application-test.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ management.endpoints.web.exposure.include: *
#logging.level.root: ERROR
#logging.level.edu.kit.datamanager.doip:TRACE
logging.level.edu.kit: DEBUG
logging.level.edu.kit.datamanager.pit: TRACE
#logging.level.org.springframework.transaction: TRACE
logging.level.org.springframework: WARN
logging.level.org.springframework.amqp: WARN
Expand Down

0 comments on commit 7349a0c

Please sign in to comment.