Skip to content

Commit

Permalink
Refactor IncrementalBuilder and Indexer.
Browse files Browse the repository at this point in the history 
Refactor of the incremental builder and indexer to make it easier to
write custom builders that calculate the affected resources separately
from indexing the changed and deleted sources in the build request.

Signed-off-by: Andrew Lamb <[email protected]>
  • Loading branch information
@andrewL-avlq
andrewL-avlq committed Oct 27, 2023
1 parent 346e1f7 commit c871b8b
Show file tree
Hide file tree
Showing 2 changed files with 131 additions and 70 deletions.
148 changes: 84 additions & 64 deletions org.eclipse.xtext/src/org/eclipse/xtext/build/IncrementalBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.resource.Resource;
Expand Down Expand Up @@ -44,6 +45,7 @@
import org.eclipse.xtext.resource.XtextResourceSet;
import org.eclipse.xtext.resource.clustering.DisabledClusteringPolicy;
import org.eclipse.xtext.resource.clustering.IResourceClusteringPolicy;
import org.eclipse.xtext.resource.impl.ResourceDescriptionsData;
import org.eclipse.xtext.resource.persistence.IResourceStorageFacade;
import org.eclipse.xtext.resource.persistence.SerializableResourceDescription;
import org.eclipse.xtext.resource.persistence.SourceLevelURIsAdapter;
Expand Down Expand Up @@ -227,80 +229,43 @@ protected void unloadResource(URI uri, Predicate<Resource> condition) {
}
}

/**
* Unload the given uris that are not already in the unloaded set.
*
* @param uriStream
* the uris to unload.
* @param unloaded
* the set of unloaded uris, to which the given uris to unload will be added.
* @since 2.33
*/
protected void unloadResources(final Stream<URI> uriStream, final Set<URI> unloaded) {
uriStream.forEach(uri -> {
if (unloaded.add(uri)) {
unloadResource(uri);
}
});
}

public IncrementalBuilder.Result launch() {
Source2GeneratedMapping newSource2GeneratedMapping = request.getState().getFileMappings();
Set<URI> unloaded = new HashSet<>();
for (URI deleted : request.getDeletedFiles()) {
if (unloaded.add(deleted)) {
unloadResource(deleted);
}
}
for (URI dirty : request.getDirtyFiles()) {
if (unloaded.add(dirty)) {
unloadResource(dirty);
}
}
for (URI source : request.getDeletedFiles()) {
request.getAfterValidate().afterValidate(source, Collections.emptyList());
Map<URI, String> outputConfigs = newSource2GeneratedMapping.deleteSourceAndGetOutputConfigs(source);
for (URI generated : outputConfigs.keySet()) {
IResourceServiceProvider serviceProvider = context.getResourceServiceProvider(source);
XtextResourceSet resourceSet = request.getResourceSet();
Set<OutputConfiguration> configs = serviceProvider
.get(IContextualOutputConfigurationProvider2.class).getOutputConfigurations(resourceSet);
String configName = outputConfigs.get(generated);
OutputConfiguration config = FluentIterable.from(configs)
.firstMatch(it -> it.getName().equals(configName)).orNull();
if (config != null && config.isCleanUpDerivedResources()) {
try {
resourceSet.getURIConverter().delete(generated, Collections.emptyMap());
request.getAfterDeleteFile().apply(generated);
} catch (IOException e) {
throw new RuntimeIOException(e);
}
}
}
}
unloadResources(request.getDeletedFiles().stream(), unloaded);
unloadResources(request.getDirtyFiles().stream(), unloaded);

Source2GeneratedMapping newSource2GeneratedMapping = request.getState().getFileMappings();
deleteGeneratedSources(getRequest().getDeletedFiles(), newSource2GeneratedMapping);
Indexer.IndexResult result = indexer.computeAndIndexAffected(request, context);
operationCanceledManager.checkCanceled(request.getCancelIndicator());

List<IResourceDescription.Delta> resolvedDeltas = new ArrayList<>();
for (IResourceDescription.Delta delta : result.getResourceDeltas()) {
URI uri = delta.getUri();
if (delta.getOld() != null && unloaded.add(uri)) {
unloadResource(uri);
}
if (delta.getNew() == null) {
resolvedDeltas.add(delta);
}
}
result.getResourceDeltas().stream().filter(delta -> delta.getNew() == null).forEach(resolvedDeltas::add);
unloadResources(result.getResourceDeltas().stream().filter(it -> it.getOld() != null).map(Delta::getUri),
unloaded);
List<URI> toBeBuilt = result.getResourceDeltas().stream().filter((it) -> it.getNew() != null)
.map(Delta::getUri).collect(Collectors.toList());

installSourceLevelURIs(toBeBuilt);
Iterable<IResourceDescription.Delta> deltas = context.executeClustered(toBeBuilt,
(resource) -> {
CancelIndicator cancelIndicator = request.getCancelIndicator();
operationCanceledManager.checkCanceled(cancelIndicator);
// trigger init
resource.getContents();
EcoreUtil2.resolveLazyCrossReferences(resource, CancelIndicator.NullImpl);
operationCanceledManager.checkCanceled(cancelIndicator);
IResourceServiceProvider serviceProvider = getResourceServiceProvider(resource);
IResourceDescription.Manager manager = serviceProvider.getResourceDescriptionManager();
IResourceDescription description = manager.getResourceDescription(resource);
IResourceDescription copiedDescription = getSerializableResourceDescription(description);
result.getNewIndex().addDescription(resource.getURI(), copiedDescription);
operationCanceledManager.checkCanceled(cancelIndicator);
if (!request.isIndexOnly() && validate(resource) && serviceProvider.get(IShouldGenerate.class)
.shouldGenerate(resource, CancelIndicator.NullImpl)) {
operationCanceledManager.checkCanceled(cancelIndicator);
generate(resource, request, newSource2GeneratedMapping);
}
IResourceDescription old = context.getOldState().getResourceDescriptions()
.getResourceDescription(resource.getURI());
return manager.createDelta(old, copiedDescription);
});
resource -> buildResource(resource, result.getNewIndex(), newSource2GeneratedMapping));

Iterables.addAll(resolvedDeltas, deltas);
return new IncrementalBuilder.Result(request.getState(), resolvedDeltas);
Expand Down Expand Up @@ -332,6 +297,61 @@ protected Indexer getIndexer() {
return indexer;
}

/**
* @since 2.33
*/
protected void deleteGeneratedSources(final List<URI> deletedSources,
final Source2GeneratedMapping newSource2GeneratedMapping) {
for (URI source : deletedSources) {
request.getAfterValidate().afterValidate(source, Collections.emptyList());
Map<URI, String> outputConfigs = newSource2GeneratedMapping.deleteSourceAndGetOutputConfigs(source);
for (URI generated : outputConfigs.keySet()) {
IResourceServiceProvider serviceProvider = context.getResourceServiceProvider(source);
XtextResourceSet resourceSet = request.getResourceSet();
Set<OutputConfiguration> configs = serviceProvider
.get(IContextualOutputConfigurationProvider2.class).getOutputConfigurations(resourceSet);
String configName = outputConfigs.get(generated);
OutputConfiguration config = FluentIterable.from(configs)
.firstMatch(it -> it.getName().equals(configName)).orNull();
if (config != null && config.isCleanUpDerivedResources()) {
try {
resourceSet.getURIConverter().delete(generated, Collections.emptyMap());
request.getAfterDeleteFile().apply(generated);
} catch (IOException e) {
throw new RuntimeIOException(e);
}
}
}
}
}

/**
* @since 2.33
*/
protected IResourceDescription.Delta buildResource(final Resource resource,
final ResourceDescriptionsData newIndex, final Source2GeneratedMapping newSource2GeneratedMapping) {
CancelIndicator cancelIndicator = request.getCancelIndicator();
operationCanceledManager.checkCanceled(cancelIndicator);
// trigger init
resource.getContents();
EcoreUtil2.resolveLazyCrossReferences(resource, CancelIndicator.NullImpl);
operationCanceledManager.checkCanceled(cancelIndicator);
IResourceServiceProvider serviceProvider = getResourceServiceProvider(resource);
IResourceDescription.Manager manager = serviceProvider.getResourceDescriptionManager();
IResourceDescription description = manager.getResourceDescription(resource);
IResourceDescription copiedDescription = getSerializableResourceDescription(description);
newIndex.addDescription(resource.getURI(), copiedDescription);
operationCanceledManager.checkCanceled(cancelIndicator);
if (!request.isIndexOnly() && validate(resource)
&& serviceProvider.get(IShouldGenerate.class).shouldGenerate(resource, CancelIndicator.NullImpl)) {
operationCanceledManager.checkCanceled(cancelIndicator);
generate(resource, request, newSource2GeneratedMapping);
}
IResourceDescription old = context.getOldState().getResourceDescriptions()
.getResourceDescription(resource.getURI());
return manager.createDelta(old, copiedDescription);
}

/**
* @since 2.28
*/
Expand Down
53 changes: 47 additions & 6 deletions org.eclipse.xtext/src/org/eclipse/xtext/build/Indexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,12 @@ public String toString() {
private OperationCanceledManager operationCanceledManager;

/**
* Compute an updated index.
* Compute an initial index. Explicit calls to {@link #computeAndIndexAffected(Collection, IndexResult, BuildRequest, BuildContext)}
* are needed to complete indexing.
*
* @since 2.33
*/
public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildContext context) {
public Indexer.IndexResult initialIndex(final BuildRequest request, final BuildContext context) {
ResourceDescriptionsData previousIndex = context.getOldState().getResourceDescriptions();
ResourceDescriptionsData newIndex = request.getState().getResourceDescriptions();
List<IResourceDescription.Delta> deltas = new ArrayList<>();
Expand All @@ -119,16 +122,30 @@ public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildCo
for (IResourceDescription.Delta delta : deltas) {
newIndex.register(delta);
}
return new Indexer.IndexResult(deltas, newIndex);
}

/**
* Compute an updated index based on new deltas and a previously calculated {@link #initialIndex(BuildRequest, BuildContext)}.
*
* @since 2.33
*/
public List<IResourceDescription.Delta> computeAndIndexAffected(
final Collection<IResourceDescription.Delta> newDeltas, final Indexer.IndexResult result,
final BuildRequest request, final BuildContext context) {
List<IResourceDescription.Delta> deltas = result.getResourceDeltas();
Set<IResourceDescription.Delta> allDeltas = new HashSet<>(deltas);
allDeltas.addAll(request.getExternalDeltas());
Set<URI> deltaSet = FluentIterable.from(deltas).transform(Delta::getUri).toSet();
ResourceDescriptionsData previousIndex = context.getOldState().getResourceDescriptions();
List<URI> allAffected = FluentIterable.from(previousIndex.getAllResourceDescriptions())
.transform(IResourceDescription::getURI).filter(it -> !deltaSet.contains(it)).filter(it -> {
IResourceServiceProvider resourceServiceProvider = context.getResourceServiceProvider(it);
if (resourceServiceProvider != null) {
IResourceDescription.Manager manager = resourceServiceProvider.getResourceDescriptionManager();
IResourceDescription resourceDescription = previousIndex.getResourceDescription(it);
return isAffected(resourceDescription, manager, allDeltas, allDeltas, newIndex);
return isAffected(resourceDescription, manager, newDeltas, allDeltas,
result.getNewIndex());
} else {
IResourceDescription.Delta delta = getDeltaForDeletedResource(it, previousIndex);
if (delta != null) {
Expand All @@ -137,8 +154,19 @@ public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildCo
return false;
}
}).toList();
deltas.addAll(getDeltasForChangedResources(allAffected, previousIndex, context));
return new Indexer.IndexResult(deltas, newIndex);
List<IResourceDescription.Delta> affectedDeltas = getDeltasForChangedResources(allAffected, previousIndex,
context);
deltas.addAll(affectedDeltas);
return affectedDeltas;
}

/**
* Compute an updated index.
*/
public Indexer.IndexResult computeAndIndexAffected(BuildRequest request, BuildContext context) {
Indexer.IndexResult result = initialIndex(request, context);
computeAndIndexAffected(result.getResourceDeltas(), result, request, context);
return result;
}

/**
Expand All @@ -152,7 +180,7 @@ protected List<IResourceDescription.Delta> getDeltasForDeletedResources(BuildReq
IResourceServiceProvider resourceServiceProvider = context.getResourceServiceProvider(deleted);
if (resourceServiceProvider != null) {
operationCanceledManager.checkCanceled(context.getCancelIndicator());
IResourceDescription.Delta delta = getDeltaForDeletedResource(deleted, oldIndex);
IResourceDescription.Delta delta = getDeltaForDeletedResource(deleted, oldIndex, resourceServiceProvider.getResourceDescriptionManager());
if (delta != null) {
deltas.add(delta);
}
Expand All @@ -169,8 +197,21 @@ protected List<IResourceDescription.Delta> getDeltasForDeletedResources(BuildReq
*
*/
protected IResourceDescription.Delta getDeltaForDeletedResource(URI uri, ResourceDescriptionsData oldIndex) {
return getDeltaForDeletedResource(uri, oldIndex, null);
}

/**
* Gets a delta for a resource that shall be deleted using the resource description manager if provided.
*
* @since 2.33
*
*/
protected IResourceDescription.Delta getDeltaForDeletedResource(URI uri, ResourceDescriptionsData oldIndex, IResourceDescription.Manager manager) {
IResourceDescription oldDescription = oldIndex.getResourceDescription(uri);
if (oldDescription != null) {
if (manager != null) {
return manager.createDelta(oldDescription, null);
}
return new DefaultResourceDescriptionDelta(oldDescription, null);
}
return null;
Expand Down

0 comments on commit c871b8b

Please sign in to comment.