Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lazy loading of extension attributes + powsybl-ws-dependencies 2.15.0 #49

Merged
merged 12 commits into from
Sep 26, 2024
Merged

Lazy loading of extension attributes + powsybl-ws-dependencies 2.15.0 #49

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f0547c3
Lazy loading of extension attributes
antoinebhs Jul 17, 2024
ea37e6f
Add test remove extension
antoinebhs Jul 17, 2024
1b029ea
Fix queries
antoinebhs Jul 18, 2024
cd95320
Merge branch 'main' of https://github.com/powsybl/powsybl-network-sto…
antoinebhs Aug 1, 2024
147fa5c
Use generic TopLevelDocument
antoinebhs Sep 2, 2024
b8a52f0
Merge branch 'main' of https://github.com/powsybl/powsybl-network-sto…
antoinebhs Sep 17, 2024
7c7cb3d
Merge branch 'main' of https://github.com/powsybl/powsybl-network-sto…
antoinebhs Sep 23, 2024
65d3275
Change after update of network-store
antoinebhs Sep 23, 2024
6f9f0bf
Add test for removed equipment
antoinebhs Sep 23, 2024
ca6ceb8
Fix URI after review comments
antoinebhs Sep 23, 2024
bc789bc
Upgrade powsybl-ws-dependencies
antoinebhs Sep 25, 2024
dd70625
Add controller integration tests
antoinebhs Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 33 additions & 5 deletions ...on-test/src/test/java/com/powsybl/network/store/integration/NetworkStoreExtensionsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,7 @@
import com.powsybl.computation.local.LocalComputationManager;
import com.powsybl.iidm.network.*;
import com.powsybl.iidm.network.extensions.*;
import com.powsybl.iidm.network.test.BatteryNetworkFactory;
import com.powsybl.iidm.network.test.EurostagTutorialExample1Factory;
import com.powsybl.iidm.network.test.HvdcTestNetwork;
import com.powsybl.iidm.network.test.SvcTestCaseFactory;
import com.powsybl.iidm.network.test.ThreeWindingsTransformerNetworkFactory;
import com.powsybl.iidm.network.test.*;
import com.powsybl.network.store.client.NetworkStoreService;
import com.powsybl.network.store.iidm.impl.NetworkImpl;
import com.powsybl.network.store.model.BaseVoltageSourceAttribute;
Expand Down Expand Up @@ -1570,4 +1566,36 @@ public void test3wtToBeEstimated() {
assertTrue(threeWindingsTransformerToBeEstimated.shouldEstimatePhaseTapChanger3());
}
}

@Test
public void testRemoveExtension() {
antoinebhs marked this conversation as resolved.
Show resolved Hide resolved
try (NetworkStoreService service = createNetworkStoreService(randomServerPort)) {
Network network = EurostagTutorialExample1Factory.create(service.getNetworkFactory());
Generator gen = network.getGenerator("GEN");
gen.newExtension(ActivePowerControlAdder.class)
.withParticipate(true)
.withDroop(6.3f)
.add();
service.flush(network);
}

try (NetworkStoreService service = createNetworkStoreService(randomServerPort)) {
Network network = service.getNetwork(service.getNetworkIds().keySet().iterator().next());
Generator gen = network.getGenerator("GEN");
ActivePowerControl<Generator> activePowerControl = gen.getExtension(ActivePowerControl.class);
assertNotNull(activePowerControl);

gen.removeExtension(ActivePowerControl.class);
assertNull(gen.getExtensionByName("activePowerControl"));
assertNull(gen.getExtension(ActivePowerControl.class));
service.flush(network);
}

try (NetworkStoreService service = createNetworkStoreService(randomServerPort)) {
Network network = service.getNetwork(service.getNetworkIds().keySet().iterator().next());
Generator gen = network.getGenerator("GEN");
assertNull(gen.getExtensionByName("activePowerControl"));
assertNull(gen.getExtension(ActivePowerControl.class));
}
}
}
174 changes: 116 additions & 58 deletions network-store-server/src/main/java/com/powsybl/network/store/server/ExtensionHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.powsybl.network.store.model.*;
import com.powsybl.network.store.model.ExtensionAttributes;
import com.powsybl.network.store.model.IdentifiableAttributes;
import com.powsybl.network.store.model.NetworkAttributes;
import com.powsybl.network.store.model.Resource;
import com.powsybl.network.store.server.dto.OwnerInfo;
import com.powsybl.network.store.server.exceptions.UncheckedSqlException;
import org.springframework.stereotype.Component;
Expand Down Expand Up @@ -66,78 +69,108 @@ public void insertExtensions(Map<OwnerInfo, Map<String, ExtensionAttributes>> ex
}
}

public Map<OwnerInfo, Map<String, ExtensionAttributes>> getExtensionsWithInClause(UUID networkUuid, int variantNum, String columnNameForWhereClause, List<String> valuesForInClause) {
if (valuesForInClause.isEmpty()) {
return Collections.emptyMap();
}
public Optional<ExtensionAttributes> getExtensionAttributes(UUID networkUuid, int variantNum, String identifiableId, String extensionName) {
try (var connection = dataSource.getConnection()) {
var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildGetExtensionsWithInClauseQuery(columnNameForWhereClause, valuesForInClause.size()));
var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildGetExtensionsQuery());
preparedStmt.setObject(1, networkUuid);
preparedStmt.setInt(2, variantNum);
for (int i = 0; i < valuesForInClause.size(); i++) {
preparedStmt.setString(3 + i, valuesForInClause.get(i));
}
preparedStmt.setString(3, identifiableId);
preparedStmt.setString(4, extensionName);

return innerGetExtensions(preparedStmt);
return innerGetExtensionAttributes(preparedStmt);
} catch (SQLException e) {
throw new UncheckedSqlException(e);
}
}

public Map<OwnerInfo, Map<String, ExtensionAttributes>> getExtensions(UUID networkUuid, int variantNum, String columnNameForWhereClause, String valueForWhereClause) {
private Optional<ExtensionAttributes> innerGetExtensionAttributes(PreparedStatement preparedStmt) throws SQLException {
try (ResultSet resultSet = preparedStmt.executeQuery()) {
if (resultSet.next()) {
return Optional.of(mapper.readValue(resultSet.getString(1), ExtensionAttributes.class));
}
return Optional.empty();
} catch (JsonProcessingException e) {
throw new UncheckedIOException(e);
}
}

public Map<String, ExtensionAttributes> getAllExtensionsAttributesByResourceTypeAndExtensionName(UUID networkUuid, int variantNum, String resourceType, String extensionName) {
try (var connection = dataSource.getConnection()) {
var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildGetExtensionsQuery(columnNameForWhereClause));
var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildGetAllExtensionsAttributesByResourceTypeAndExtensionName());
preparedStmt.setObject(1, networkUuid);
preparedStmt.setInt(2, variantNum);
preparedStmt.setString(3, valueForWhereClause);

return innerGetExtensions(preparedStmt);
preparedStmt.setString(3, resourceType);
preparedStmt.setString(4, extensionName);
return innerGetAllExtensionsAttributesByResourceTypeAndExtensionName(preparedStmt);
} catch (SQLException e) {
throw new UncheckedSqlException(e);
}
}

private Map<OwnerInfo, Map<String, ExtensionAttributes>> innerGetExtensions(PreparedStatement preparedStmt) throws SQLException {
private Map<String, ExtensionAttributes> innerGetAllExtensionsAttributesByResourceTypeAndExtensionName(PreparedStatement preparedStmt) throws SQLException {
try (ResultSet resultSet = preparedStmt.executeQuery()) {
Map<OwnerInfo, Map<String, ExtensionAttributes>> map = new HashMap<>();
Map<String, ExtensionAttributes> map = new HashMap<>();
while (resultSet.next()) {
String equipmentId = resultSet.getString(1);
ExtensionAttributes extensionValue = mapper.readValue(resultSet.getString(2), ExtensionAttributes.class);
map.put(equipmentId, extensionValue);
}
return map;
} catch (JsonProcessingException e) {
throw new UncheckedIOException(e);
}
}

OwnerInfo owner = new OwnerInfo();
owner.setEquipmentId(resultSet.getString(1));
owner.setEquipmentType(ResourceType.valueOf(resultSet.getString(2)));
owner.setNetworkUuid(UUID.fromString(resultSet.getString(3)));
owner.setVariantNum(resultSet.getInt(4));

String extensionName = resultSet.getString(5);
ExtensionAttributes extensionValue = mapper.readValue(resultSet.getString(6), ExtensionAttributes.class);
public Map<String, ExtensionAttributes> getAllExtensionsAttributesByIdentifiableId(UUID networkUuid, int variantNum, String identifiableId) {
try (var connection = dataSource.getConnection()) {
var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildGetAllExtensionsAttributesByIdentifiableId());
preparedStmt.setObject(1, networkUuid);
preparedStmt.setInt(2, variantNum);
preparedStmt.setString(3, identifiableId);
return innerGetAllExtensionsAttributesByIdentifiableId(preparedStmt);
} catch (SQLException e) {
throw new UncheckedSqlException(e);
}
}

map.computeIfAbsent(owner, k -> new HashMap<>());
map.get(owner).put(extensionName, extensionValue);
private Map<String, ExtensionAttributes> innerGetAllExtensionsAttributesByIdentifiableId(PreparedStatement preparedStmt) throws SQLException {
try (ResultSet resultSet = preparedStmt.executeQuery()) {
Map<String, ExtensionAttributes> map = new HashMap<>();
while (resultSet.next()) {
String extensionName = resultSet.getString(1);
ExtensionAttributes extensionValue = mapper.readValue(resultSet.getString(2), ExtensionAttributes.class);
map.put(extensionName, extensionValue);
}
return map;
} catch (JsonProcessingException e) {
throw new UncheckedIOException(e);
}
}

public <T extends IdentifiableAttributes> void insertExtensionsInIdentifiables(UUID networkUuid, List<Resource<T>> equipments, Map<OwnerInfo, Map<String, ExtensionAttributes>> extensions) {
if (!extensions.isEmpty() && !equipments.isEmpty()) {
for (Resource<T> equipmentAttributesResource : equipments) {
OwnerInfo owner = new OwnerInfo(
equipmentAttributesResource.getId(),
equipmentAttributesResource.getType(),
networkUuid,
equipmentAttributesResource.getVariantNum()
);
if (extensions.containsKey(owner)) {
T attributes = equipmentAttributesResource.getAttributes();
extensions.get(owner).forEach((key, value) -> {
if (attributes != null && attributes.getExtensionAttributes() != null) {
attributes.getExtensionAttributes().put(key, value);
}
});
}
public Map<String, Map<String, ExtensionAttributes>> getAllExtensionsAttributesByResourceType(UUID networkUuid, int variantNum, String resourceType) {
try (var connection = dataSource.getConnection()) {
var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildGetAllExtensionsAttributesByResourceType());
preparedStmt.setObject(1, networkUuid);
preparedStmt.setInt(2, variantNum);
preparedStmt.setString(3, resourceType);
return innerGetAllExtensionsAttributesByResourceType(preparedStmt);
} catch (SQLException e) {
throw new UncheckedSqlException(e);
}
}

private Map<String, Map<String, ExtensionAttributes>> innerGetAllExtensionsAttributesByResourceType(PreparedStatement preparedStmt) throws SQLException {
try (ResultSet resultSet = preparedStmt.executeQuery()) {
Map<String, Map<String, ExtensionAttributes>> map = new HashMap<>();
while (resultSet.next()) {
String equipmentId = resultSet.getString(1);
String extensionName = resultSet.getString(2);
ExtensionAttributes extensionValue = mapper.readValue(resultSet.getString(3), ExtensionAttributes.class);
map.computeIfAbsent(equipmentId, k -> new HashMap<>()).put(extensionName, extensionValue);
}
return map;
} catch (JsonProcessingException e) {
throw new UncheckedIOException(e);
}
}

Expand All @@ -160,10 +193,35 @@ public void deleteExtensionsFromIdentifiables(UUID networkUuid, int variantNum,
}
}

public void deleteExtensionsFromIdentifiables(UUID networkUuid, int variantNum, Map<String, Set<String>> extensionNamesByIdentifiableId) {
try (var connection = dataSource.getConnection()) {
int totalParameters = extensionNamesByIdentifiableId.values().stream().mapToInt(Set::size).sum();
if (totalParameters > 0) {
try (var preparedStmt = connection.prepareStatement(QueryExtensionCatalog.buildDeleteExtensionsVariantByIdentifiableIdAndExtensionsNameINQuery(totalParameters))) {
preparedStmt.setObject(1, networkUuid);
preparedStmt.setInt(2, variantNum);

int paramIndex = 3;
for (Map.Entry<String, Set<String>> entry : extensionNamesByIdentifiableId.entrySet()) {
String equipmentId = entry.getKey();
for (String extensionName : entry.getValue()) {
preparedStmt.setString(paramIndex++, equipmentId);
preparedStmt.setString(paramIndex++, extensionName);
}
}

preparedStmt.executeUpdate();
}
}
} catch (SQLException e) {
throw new UncheckedSqlException(e);
}
}

/**
* Extensions do not always exist in the variant and can be added by a
* modification for example. Instead of implementing an UPSERT, we delete then insert
* the extensions.
* Delete extension attributes loaded in the resource and insert the updated extension afterward.
* We can't delete all the extensions attributes associated with this resource here as we are not sure that all
* extension attributes have been modified/loaded in the resource.
*/
public <T extends IdentifiableAttributes> void updateExtensionsFromEquipments(UUID networkUuid, List<Resource<T>> resources) {
deleteExtensionsFromEquipments(networkUuid, resources);
Expand Down Expand Up @@ -191,20 +249,20 @@ public <T extends IdentifiableAttributes> Map<OwnerInfo, Map<String, ExtensionAt
}

public <T extends IdentifiableAttributes> void deleteExtensionsFromEquipments(UUID networkUuid, List<Resource<T>> resources) {
Map<Integer, List<String>> resourceIdsByVariant = new HashMap<>();
Map<Integer, Map<String, Set<String>>> extensionsAttributesByVariant = new HashMap<>();
for (Resource<T> resource : resources) {
List<String> resourceIds = resourceIdsByVariant.get(resource.getVariantNum());
if (resourceIds != null) {
resourceIds.add(resource.getId());
} else {
resourceIds = new ArrayList<>();
resourceIds.add(resource.getId());
}
resourceIdsByVariant.put(resource.getVariantNum(), resourceIds);
extensionsAttributesByVariant
.computeIfAbsent(resource.getVariantNum(), k -> new HashMap<>())
.put(resource.getId(), resource.getAttributes().getExtensionAttributes().keySet());
}
resourceIdsByVariant.forEach((k, v) -> deleteExtensionsFromIdentifiables(networkUuid, k, v));
extensionsAttributesByVariant.forEach((k, v) -> deleteExtensionsFromIdentifiables(networkUuid, k, v));
}

/**
* Delete extension attributes loaded in the resource and insert the updated extension afterward.
* We can't delete all the extensions attributes associated with this resource here as we are not sure that all
* extension attributes have been modified/loaded in the resource.
*/
public void updateExtensionsFromNetworks(List<Resource<NetworkAttributes>> resources) {
deleteExtensionsFromNetworks(resources);
insertExtensions(getExtensionsFromNetworks(resources));
Expand Down Expand Up @@ -232,7 +290,7 @@ public Map<OwnerInfo, Map<String, ExtensionAttributes>> getExtensionsFromNetwork

private void deleteExtensionsFromNetworks(List<Resource<NetworkAttributes>> resources) {
for (Resource<NetworkAttributes> resource : resources) {
deleteExtensionsFromIdentifiable(resource.getAttributes().getUuid(), resource.getVariantNum(), resource.getId());
deleteExtensionsFromIdentifiables(resource.getAttributes().getUuid(), resource.getVariantNum(), Map.of(resource.getId(), resource.getAttributes().getExtensionAttributes().keySet()));
}
}
}
Loading
Loading