Skip to content

Commit

Permalink
Merge release branch 4.20 to main
Browse files Browse the repository at this point in the history 
* 4.20:
  log name change after merge forward
  check tags while fetching storage pool for importing vm (#9764)
  UI: Add cluster arch type to the zone creation wizard (#10080)
  • Loading branch information
@DaanHoogland
DaanHoogland committed Jan 3, 2025
2 parents 5bae118 + cfafcae commit 30b2588
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 33 deletions.
134 changes: 101 additions & 33 deletions server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ private Map<String, Network.IpAddresses> getNicIpAddresses(final List<UnmanagedI
return nicIpAddresses;
}

private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final DataCenter zone, final Cluster cluster) {
private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final DataCenter zone, final Cluster cluster, String diskOfferingTags) {
StoragePool storagePool = null;
final String dsHost = disk.getDatastoreHost();
final String dsPath = disk.getDatastorePath();
Expand All @@ -548,7 +548,8 @@ private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final Da
List<StoragePoolVO> pools = primaryDataStoreDao.listPoolByHostPath(dsHost, dsPath);
for (StoragePool pool : pools) {
if (pool.getDataCenterId() == zone.getId() &&
(pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId()))) {
(pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId())) &&
volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOfferingTags)) {
storagePool = pool;
break;
}
Expand All @@ -560,7 +561,8 @@ private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final Da
pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId()));
for (StoragePool pool : pools) {
String searchPoolParam = StringUtils.isNotBlank(dsPath) ? dsPath : dsName;
if (StringUtils.contains(pool.getPath(), searchPoolParam)) {
if (StringUtils.contains(pool.getPath(), searchPoolParam) &&
volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOfferingTags)) {
storagePool = pool;
break;
}
Expand Down Expand Up @@ -623,7 +625,8 @@ private void checkUnmanagedDiskAndOfferingForImport(String instanceName, Unmanag
if (diskOffering != null && !diskOffering.isCustomized() && diskOffering.getDiskSize() < disk.getCapacity()) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Size of disk offering(ID: %s) %dGB is found less than the size of disk(ID: %s) %dGB during VM import", diskOffering.getUuid(), (diskOffering.getDiskSize() / Resource.ResourceType.bytesToGiB), disk.getDiskId(), (disk.getCapacity() / (Resource.ResourceType.bytesToGiB))));
}
StoragePool storagePool = getStoragePool(disk, zone, cluster);
diskOffering = diskOffering != null ? diskOffering : diskOfferingDao.findById(serviceOffering.getDiskOfferingId());
StoragePool storagePool = getStoragePool(disk, zone, cluster, diskOffering != null ? diskOffering.getTags() : null);
if (diskOffering != null && !migrateAllowed && !storagePoolSupportsDiskOffering(storagePool, diskOffering)) {
throw new InvalidParameterValueException(String.format("Disk offering: %s is not compatible with storage pool: %s of unmanaged disk: %s", diskOffering.getUuid(), storagePool.getUuid(), disk.getDiskId()));
}
Expand Down Expand Up @@ -860,7 +863,7 @@ private Pair<DiskProfile, StoragePool> importDisk(UnmanagedInstanceTO.Disk disk,
diskInfo.setDiskChain(new String[]{disk.getImagePath()});
chainInfo = gson.toJson(diskInfo);
}
StoragePool storagePool = getStoragePool(disk, zone, cluster);
StoragePool storagePool = getStoragePool(disk, zone, cluster, diskOffering != null ? diskOffering.getTags() : null);
DiskProfile profile = volumeManager.importVolume(type, name, diskOffering, diskSize,
minIops, maxIops, vm.getDataCenterId(), vm.getHypervisorType(), vm, template, owner, deviceId, storagePool.getId(), path, chainInfo);

Expand Down Expand Up @@ -1645,7 +1648,7 @@ protected UserVm importUnmanagedInstanceFromVmwareToKvm(DataCenter zone, Cluster

temporaryConvertLocation = selectInstanceConversionTemporaryLocation(
destinationCluster, convertHost, convertStoragePoolId);
List<StoragePoolVO> convertStoragePools = findInstanceConversionStoragePoolsInCluster(destinationCluster);
List<StoragePoolVO> convertStoragePools = findInstanceConversionStoragePoolsInCluster(destinationCluster, serviceOffering, dataDiskOfferingMap);
long importStartTime = System.currentTimeMillis();
Pair<UnmanagedInstanceTO, Boolean> sourceInstanceDetails = getSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password, clusterName, sourceHostName, sourceVMName);
sourceVMwareInstance = sourceInstanceDetails.first();
Expand All @@ -1661,15 +1664,18 @@ protected UserVm importUnmanagedInstanceFromVmwareToKvm(DataCenter zone, Cluster
if (cmd.getForceMsToImportVmFiles() || !conversionSupportAnswer.isOvfExportSupported()) {
// Uses MS for OVF export to temporary conversion location
int noOfThreads = UnmanagedVMsManager.ThreadsOnMSToImportVMwareVMFiles.value();
ovfTemplateOnConvertLocation = createOvfTemplateOfSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password,
clusterName, sourceHostName, sourceVMwareInstance.getName(), temporaryConvertLocation, noOfThreads);
ovfTemplateOnConvertLocation = createOvfTemplateOfSourceVmwareUnmanagedInstance(
vcenter, datacenterName, username, password, clusterName, sourceHostName,
sourceVMwareInstance.getName(), temporaryConvertLocation, noOfThreads);
convertedInstance = convertVmwareInstanceToKVMWithOVFOnConvertLocation(sourceVMName,
sourceVMwareInstance, convertHost, importHost, convertStoragePools,
temporaryConvertLocation, ovfTemplateOnConvertLocation);
sourceVMwareInstance, convertHost, importHost, convertStoragePools,
serviceOffering, dataDiskOfferingMap, temporaryConvertLocation,
ovfTemplateOnConvertLocation);
} else {
// Uses KVM Host for OVF export to temporary conversion location, through ovftool
convertedInstance = convertVmwareInstanceToKVMAfterExportingOVFToConvertLocation(
sourceVMName, sourceVMwareInstance, convertHost, importHost, convertStoragePools,
sourceVMName, sourceVMwareInstance, convertHost, importHost,
convertStoragePools, serviceOffering, dataDiskOfferingMap,
temporaryConvertLocation, vcenter, username, password, datacenterName);
}

Expand Down Expand Up @@ -1759,9 +1765,9 @@ private void sanitizeConvertedInstance(UnmanagedInstanceTO convertedInstance, Un
convertedInstance.setPowerState(UnmanagedInstanceTO.PowerState.PowerOff);
List<UnmanagedInstanceTO.Disk> convertedInstanceDisks = convertedInstance.getDisks();
List<UnmanagedInstanceTO.Disk> sourceVMwareInstanceDisks = sourceVMwareInstance.getDisks();
for (int i = 0; i < convertedInstanceDisks.size(); i++) {
UnmanagedInstanceTO.Disk disk = convertedInstanceDisks.get(i);
disk.setDiskId(sourceVMwareInstanceDisks.get(i).getDiskId());
for (UnmanagedInstanceTO.Disk sourceVMwareInstanceDisk : sourceVMwareInstanceDisks) {
UnmanagedInstanceTO.Disk convertedDisk = convertedInstanceDisks.get(sourceVMwareInstanceDisk.getPosition());
convertedDisk.setDiskId(sourceVMwareInstanceDisk.getDiskId());
}
List<UnmanagedInstanceTO.Nic> convertedInstanceNics = convertedInstance.getNics();
List<UnmanagedInstanceTO.Nic> sourceVMwareInstanceNics = sourceVMwareInstance.getNics();
Expand Down Expand Up @@ -1945,16 +1951,16 @@ private CheckConvertInstanceAnswer checkConversionSupportOnHost(HostVO convertHo
}

private UnmanagedInstanceTO convertVmwareInstanceToKVMWithOVFOnConvertLocation(
String sourceVM, UnmanagedInstanceTO sourceVMwareInstance,
HostVO convertHost, HostVO importHost,
List<StoragePoolVO> convertStoragePools, DataStoreTO temporaryConvertLocation,
String ovfTemplateDirConvertLocation
String sourceVM, UnmanagedInstanceTO sourceVMwareInstance, HostVO convertHost,
HostVO importHost, List<StoragePoolVO> convertStoragePools,
ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap,
DataStoreTO temporaryConvertLocation, String ovfTemplateDirConvertLocation
) {
logger.debug(String.format("Delegating the conversion of instance %s from VMware to KVM to the host %s (%s) using OVF %s on conversion datastore",
sourceVM, convertHost.getId(), convertHost.getName(), ovfTemplateDirConvertLocation));

RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(sourceVM);
List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks());
List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks(), serviceOffering, dataDiskOfferingMap);
ConvertInstanceCommand cmd = new ConvertInstanceCommand(remoteInstanceTO,
Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation, ovfTemplateDirConvertLocation, false, false);
int timeoutSeconds = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
Expand All @@ -1980,16 +1986,17 @@ private UnmanagedInstanceTO convertVmwareInstanceToKVMWithOVFOnConvertLocation(
}

private UnmanagedInstanceTO convertVmwareInstanceToKVMAfterExportingOVFToConvertLocation(
String sourceVM, UnmanagedInstanceTO sourceVMwareInstance,
HostVO convertHost, HostVO importHost, List<StoragePoolVO> convertStoragePools,
DataStoreTO temporaryConvertLocation, String vcenterHost,
String vcenterUsername, String vcenterPassword, String datacenterName
String sourceVM, UnmanagedInstanceTO sourceVMwareInstance, HostVO convertHost,
HostVO importHost, List<StoragePoolVO> convertStoragePools,
ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap,
DataStoreTO temporaryConvertLocation, String vcenterHost, String vcenterUsername,
String vcenterPassword, String datacenterName
) {
logger.debug(String.format("Delegating the conversion of instance %s from VMware to KVM to the host %s (%s) after OVF export through ovftool",
sourceVM, convertHost.getId(), convertHost.getName()));

RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(sourceVMwareInstance.getName(), vcenterHost, vcenterUsername, vcenterPassword, datacenterName);
List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks());
List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks(), serviceOffering, dataDiskOfferingMap);
ConvertInstanceCommand cmd = new ConvertInstanceCommand(remoteInstanceTO,
Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation, null, false, true);
int timeoutSeconds = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
Expand Down Expand Up @@ -2052,12 +2059,31 @@ private UnmanagedInstanceTO convertAndImportToKVM(ConvertInstanceCommand convert
return ((ImportConvertedInstanceAnswer) importAnswer).getConvertedInstance();
}

private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(Cluster destinationCluster) {
private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(
Cluster destinationCluster, ServiceOfferingVO serviceOffering,
Map<String, Long> dataDiskOfferingMap
) {
List<StoragePoolVO> pools = new ArrayList<>();
List<StoragePoolVO> clusterPools = primaryDataStoreDao.findClusterWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem);
pools.addAll(clusterPools);
List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getDataCenterId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem);
pools.addAll(zonePools);
pools.addAll(primaryDataStoreDao.findClusterWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem));
pools.addAll(primaryDataStoreDao.findZoneWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getDataCenterId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem));
List<String> diskOfferingTags = new ArrayList<>();
for (Long diskOfferingId : dataDiskOfferingMap.values()) {
DiskOfferingVO diskOffering = diskOfferingDao.findById(diskOfferingId);
if (diskOffering == null) {
String msg = String.format("Cannot find disk offering with ID %s", diskOfferingId);
logger.error(msg);
throw new CloudRuntimeException(msg);
}
diskOfferingTags.add(diskOffering.getTags());
}
if (serviceOffering.getDiskOfferingId() != null) {
DiskOfferingVO diskOffering = diskOfferingDao.findById(serviceOffering.getDiskOfferingId());
if (diskOffering != null) {
diskOfferingTags.add(diskOffering.getTags());
}
}

pools = getPoolsWithMatchingTags(pools, diskOfferingTags);
if (pools.isEmpty()) {
String msg = String.format("Cannot find suitable storage pools in cluster %s for the conversion", destinationCluster.getName());
logger.error(msg);
Expand All @@ -2066,12 +2092,54 @@ private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(Cluster
return pools;
}

private List<String> selectInstanceConversionStoragePools(List<StoragePoolVO> pools, List<UnmanagedInstanceTO.Disk> disks) {
private List<StoragePoolVO> getPoolsWithMatchingTags(List<StoragePoolVO> pools, List<String> diskOfferingTags) {
if (diskOfferingTags.isEmpty()) {
return pools;
}
List<StoragePoolVO> poolsSupportingTags = new ArrayList<>(pools);
for (String tags : diskOfferingTags) {
boolean tagsMatched = false;
for (StoragePoolVO pool : pools) {
if (volumeApiService.doesTargetStorageSupportDiskOffering(pool, tags)) {
poolsSupportingTags.add(pool);
tagsMatched = true;
}
}
if (!tagsMatched) {
String msg = String.format("Cannot find suitable storage pools for the conversion with disk offering tags %s", tags);
logger.error(msg);
throw new CloudRuntimeException(msg);
}
}
return poolsSupportingTags;
}

private List<String> selectInstanceConversionStoragePools(
List<StoragePoolVO> pools, List<UnmanagedInstanceTO.Disk> disks,
ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap
) {
List<String> storagePools = new ArrayList<>(disks.size());
//TODO: Choose pools by capacity
for (int i = 0; i < disks.size(); i++) {
storagePools.add(null);
}
Set<String> dataDiskIds = dataDiskOfferingMap.keySet();
for (UnmanagedInstanceTO.Disk disk : disks) {
Long capacity = disk.getCapacity();
storagePools.add(pools.get(0).getUuid());
Long diskOfferingId = dataDiskOfferingMap.get(disk.getDiskId());
if (diskOfferingId == null && !dataDiskIds.contains(disk.getDiskId())) {
diskOfferingId = serviceOffering.getDiskOfferingId();
}
//TODO: Choose pools by capacity
if (diskOfferingId == null) {
storagePools.set(disk.getPosition(), pools.get(0).getUuid());
} else {
DiskOfferingVO diskOffering = diskOfferingDao.findById(diskOfferingId);
for (StoragePoolVO pool : pools) {
if (volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOffering.getTags())) {
storagePools.set(disk.getPosition(), pool.getUuid());
break;
}
}
}
}
return storagePools;
}
Expand Down
4 changes: 4 additions & 0 deletions server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ public void setUp() throws Exception {
List<UnmanagedInstanceTO.Disk> instanceDisks = new ArrayList<>();
UnmanagedInstanceTO.Disk instanceDisk = new UnmanagedInstanceTO.Disk();
instanceDisk.setDiskId("1000-1");
instanceDisk.setPosition(0);
instanceDisk.setLabel("DiskLabel");
instanceDisk.setController("scsi");
instanceDisk.setImagePath("[b6ccf44a1fa13e29b3667b4954fa10ee] TestInstance/ROOT-1.vmdk");
Expand Down Expand Up @@ -424,6 +425,7 @@ public void importUnmanagedInstanceTest() {
ImportUnmanagedInstanceCmd importUnmanageInstanceCmd = Mockito.mock(ImportUnmanagedInstanceCmd.class);
when(importUnmanageInstanceCmd.getName()).thenReturn("TestInstance");
when(importUnmanageInstanceCmd.getDomainId()).thenReturn(null);
when(volumeApiService.doesTargetStorageSupportDiskOffering(any(StoragePool.class), any())).thenReturn(true);
try (MockedStatic<UsageEventUtils> ignored = Mockito.mockStatic(UsageEventUtils.class)) {
unmanagedVMsManager.importUnmanagedInstance(importUnmanageInstanceCmd);
}
Expand Down Expand Up @@ -703,6 +705,8 @@ private void baseTestImportVmFromVmwareToKvm(VcenterParameter vcenterParameter,
when(agentManager.send(Mockito.eq(convertHostId), Mockito.any(CheckConvertInstanceCommand.class))).thenReturn(checkConvertInstanceAnswer);
}

when(volumeApiService.doesTargetStorageSupportDiskOffering(any(StoragePool.class), any())).thenReturn(true);

ConvertInstanceAnswer convertInstanceAnswer = mock(ConvertInstanceAnswer.class);
ImportConvertedInstanceAnswer convertImportedInstanceAnswer = mock(ImportConvertedInstanceAnswer.class);
when(convertInstanceAnswer.getConvertedInstance()).thenReturn(instance);
Expand Down
14 changes: 14 additions & 0 deletions ui/src/views/infra/zone/ZoneWizardAddResources.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,13 @@ export default {
placeHolder: 'message.error.cluster.name',
required: true
},
{
title: 'label.arch',
key: 'arch',
required: false,
select: true,
options: this.architectureTypes
},
{
title: 'label.vcenter.host',
key: 'vCenterHost',
Expand Down Expand Up @@ -846,6 +853,13 @@ export default {
primaryStorageScopes: [],
primaryStorageProtocols: [],
primaryStorageProviders: [],
architectureTypes: [{
id: 'x86_64',
description: 'AMD 64 bits (x86_64)'
}, {
id: 'aarch64',
description: 'ARM 64 bits (aarch64)'
}],
storageProviders: [],
currentStep: null,
options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider']
Expand Down
1 change: 1 addition & 0 deletions ui/src/views/infra/zone/ZoneWizardLaunchZone.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1283,6 +1283,7 @@ export default {
if (this.isEdgeZone) {
clusterName = 'Cluster-' + this.stepData.zoneReturned.name
}
params.arch = this.prefillContent?.arch || null
if (hypervisor === 'VMware') {
params.username = this.prefillContent?.vCenterUsername || null
Expand Down

0 comments on commit 30b2588

Please sign in to comment.