diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 3fe8c53fecea..dac2a84d05e2 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -61,6 +61,7 @@ jobs: run: | mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + if: github.repository == 'apache/cloudstack' with: files: ./client/target/site/jacoco-aggregate/jacoco.xml fail_ci_if_error: true diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 275f6d25268d..5eea0f35d647 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -185,7 +185,8 @@ public static enum StoragePoolType { Linstor(true, true, EncryptionSupport.Storage), DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters StorPool(true, true, EncryptionSupport.Hypervisor), - FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + OntapiSCSI(true, true, EncryptionSupport.Unsupported); // NetApp ONTAP iSCSI private final boolean shared; private final boolean overProvisioning; diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 877df55c6d67..67e5555f1d75 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -67,6 +67,10 @@ public interface VmDetailConstants { String CPU_SPEED = "cpuSpeed"; String MEMORY = "memory"; + // VM deployment with custom root disk offering params + String MIN_IOPS = "minIops"; + String MAX_IOPS = "maxIops"; + // Misc details for internal usage (not to be set/changed by user or admin) String CPU_OVER_COMMIT_RATIO = "cpuOvercommitRatio"; String MEMORY_OVER_COMMIT_RATIO = "memoryOvercommitRatio"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java index 2aef856f58f3..784b9e77bdb6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java @@ -19,8 +19,8 @@ import java.net.UnknownHostException; import java.util.Map; - import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; @@ -80,10 +80,10 @@ public class CreateStoragePoolCmd extends BaseCmd { @Parameter(name = ApiConstants.MANAGED, type = CommandType.BOOLEAN, required = false, description = "Whether the storage should be managed by CloudStack") private Boolean managed; - @Parameter(name = ApiConstants.CAPACITY_IOPS, type = CommandType.LONG, required = false, description = "IOPS CloudStack can provision from this storage pool") + @Parameter(name = ApiConstants.CAPACITY_IOPS, type = CommandType.LONG, required = false, description = "IOPS CloudStack can provision from this storage pool", validations = {ApiArgValidator.PositiveNumber}) private Long capacityIops; - @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false, description = "Bytes CloudStack can provision from this storage pool") + @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false, description = "Bytes CloudStack can provision from this storage pool", validations = {ApiArgValidator.PositiveNumber}) private Long capacityBytes; @Parameter(name = ApiConstants.HYPERVISOR, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java index 4b0a6ba00b28..31e72cb2ee7a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java @@ -19,20 +19,20 @@ import java.util.List; import java.util.Map; -import org.apache.cloudstack.api.ApiCommandResourceType; - import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.ObjectUtils; import com.cloud.storage.StoragePool; import com.cloud.user.Account; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.ObjectUtils; @SuppressWarnings("rawtypes") @APICommand(name = "updateStoragePool", description = "Updates a storage pool.", responseObject = StoragePoolResponse.class, since = "3.0.0", @@ -53,10 +53,11 @@ public class UpdateStoragePoolCmd extends BaseCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "Comma-separated list of tags for the storage pool") private List tags; - @Parameter(name = ApiConstants.CAPACITY_IOPS, type = CommandType.LONG, required = false, description = "IOPS CloudStack can provision from this storage pool") + @Parameter(name = ApiConstants.CAPACITY_IOPS, type = CommandType.LONG, required = false, description = "IOPS CloudStack can provision from this storage pool", + validations = {ApiArgValidator.PositiveNumber}) private Long capacityIops; - @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false, description = "Bytes CloudStack can provision from this storage pool") + @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false, description = "Bytes CloudStack can provision from this storage pool",validations={ApiArgValidator.PositiveNumber}) private Long capacityBytes; @Parameter(name = ApiConstants.ENABLED, type = CommandType.BOOLEAN, required = false, description = "False to disable the pool for allocation of new volumes, true to" + diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java index f8f744285c04..60f4f308503d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. package org.apache.cloudstack.api.command.user.volume; -import org.apache.cloudstack.api.BaseAsyncCmd; - import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -60,7 +60,7 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd { @Parameter(name = ApiConstants.MAX_IOPS, type = CommandType.LONG, required = false, description = "New maximum number of IOPS") private Long maxIops; - @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, required = false, description = "New volume size in GB") + @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, required = false, description = "New volume size in GB",validations = {ApiArgValidator.PositiveNumber}) private Long size; @Parameter(name = ApiConstants.SHRINK_OK, type = CommandType.BOOLEAN, required = false, description = "Verify OK to Shrink") diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java index 031d9315cd08..8cde09af1212 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java @@ -117,6 +117,10 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co @Param(description = "Max IOPS of the disk volume") private Long maxIops; + @SerializedName("iscustomizediops") + @Param(description = "True if the volume's disk offering uses custom IOPS, false otherwise") + private Boolean customizedIops; + @SerializedName(ApiConstants.CREATED) @Param(description = "The date the disk volume was created") private Date created; @@ -410,6 +414,10 @@ public void setMaxIops(Long maxIops) { this.maxIops = maxIops; } + public void setCustomizedIops(Boolean customizedIops) { + this.customizedIops = customizedIops; + } + public void setCreated(Date created) { this.created = created; } @@ -712,6 +720,14 @@ public Long getMaxIops() { return maxIops; } + public Boolean isCustomizedIops() { + return customizedIops; + } + + public Boolean getCustomizedIops() { + return customizedIops; + } + public Date getCreated() { return created; } diff --git a/api/src/test/java/com/cloud/storage/StorageTest.java b/api/src/test/java/com/cloud/storage/StorageTest.java index 2bcc28e2b4b6..b9edda569dab 100644 --- a/api/src/test/java/com/cloud/storage/StorageTest.java +++ b/api/src/test/java/com/cloud/storage/StorageTest.java @@ -49,6 +49,7 @@ public void isSharedStoragePool() { Assert.assertTrue(StoragePoolType.ManagedNFS.isShared()); Assert.assertTrue(StoragePoolType.DatastoreCluster.isShared()); Assert.assertTrue(StoragePoolType.Linstor.isShared()); + Assert.assertTrue(StoragePoolType.OntapiSCSI.isShared()); } @Test @@ -73,6 +74,7 @@ public void supportsOverProvisioningTestAllStoragePoolTypes() { Assert.assertFalse(StoragePoolType.ManagedNFS.supportsOverProvisioning()); Assert.assertTrue(StoragePoolType.DatastoreCluster.supportsOverProvisioning()); Assert.assertTrue(StoragePoolType.Linstor.supportsOverProvisioning()); + Assert.assertTrue(StoragePoolType.OntapiSCSI.supportsOverProvisioning()); } @Test diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 71c329796d1b..4056de31bc15 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -251,7 +251,8 @@ public long download(boolean resume, DownloadCompleteCallback callback) { private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException { byte[] buffer = new byte[CHUNK_SIZE]; - long offset = 0; + // Absolute file position for subsequent writes, seeks, and format verification. + long offset = out.getFilePointer(); VerifyFormat verifyFormat = new VerifyFormat(file); status = Status.IN_PROGRESS; while (status != Status.ABORTED) { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 9f6d02cc1234..9a06e7285e4d 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -61,14 +61,12 @@ import com.cloud.vm.NicProfile; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; -import static org.apache.cloudstack.api.ApiConstants.MAX_IOPS; -import static org.apache.cloudstack.api.ApiConstants.MIN_IOPS; - @Component public class CloudOrchestrator implements OrchestrationService { @@ -205,8 +203,8 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String Map userVmDetails = _vmInstanceDetailsDao.listDetailsKeyPairs(vm.getId()); if (userVmDetails != null) { - String minIops = userVmDetails.get(MIN_IOPS); - String maxIops = userVmDetails.get(MAX_IOPS); + String minIops = userVmDetails.get(VmDetailConstants.MIN_IOPS); + String maxIops = userVmDetails.get(VmDetailConstants.MAX_IOPS); rootDiskOfferingInfo.setMinIops(minIops != null && minIops.trim().length() > 0 ? Long.parseLong(minIops) : null); rootDiskOfferingInfo.setMaxIops(maxIops != null && maxIops.trim().length() > 0 ? Long.parseLong(maxIops) : null); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/CloudOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/CloudOrchestratorTest.java new file mode 100644 index 000000000000..fd7deac04548 --- /dev/null +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/CloudOrchestratorTest.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.engine.orchestration; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.offering.DiskOfferingInfo; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.utils.component.ComponentContext; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; +import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntityImpl; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class CloudOrchestratorTest { + + private static final long VM_ID = 1L; + private static final long SERVICE_OFFERING_ID = 2L; + private static final long ROOT_DISK_OFFERING_ID = 3L; + private static final String TEMPLATE_ID = "4"; + + @InjectMocks + private CloudOrchestrator cloudOrchestrator = new CloudOrchestrator(); + + @Mock + private VirtualMachineManager _itMgr; + @Mock + private VMTemplateDao _templateDao; + @Mock + private VMInstanceDao _vmDao; + @Mock + private VMInstanceDetailsDao _vmInstanceDetailsDao; + @Mock + private ServiceOfferingDao _serviceOfferingDao; + @Mock + private DiskOfferingDao _diskOfferingDao; + + @Test + public void createVirtualMachineSetsCustomIopsFromVmDetails() throws Exception { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + DiskOfferingVO rootDiskOffering = Mockito.mock(DiskOfferingVO.class); + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + VirtualMachineEntityImpl vmEntity = Mockito.mock(VirtualMachineEntityImpl.class); + + Mockito.when(_vmDao.findByUuid("vm-uuid")).thenReturn(vm); + Mockito.when(vm.getId()).thenReturn(VM_ID); + Mockito.when(vm.getServiceOfferingId()).thenReturn(SERVICE_OFFERING_ID); + Mockito.when(vm.getInstanceName()).thenReturn("i-1-1-VM"); + Mockito.when(_serviceOfferingDao.findById(VM_ID, SERVICE_OFFERING_ID)).thenReturn(serviceOffering); + Mockito.when(_diskOfferingDao.findById(ROOT_DISK_OFFERING_ID)).thenReturn(rootDiskOffering); + Mockito.when(rootDiskOffering.isCustomizedIops()).thenReturn(true); + Mockito.when(_templateDao.findById(Long.valueOf(TEMPLATE_ID))).thenReturn(template); + + Map details = new HashMap<>(); + details.put(VmDetailConstants.MIN_IOPS, "100"); + details.put(VmDetailConstants.MAX_IOPS, "1000"); + Mockito.when(_vmInstanceDetailsDao.listDetailsKeyPairs(VM_ID)).thenReturn(details); + + try (MockedStatic componentContext = Mockito.mockStatic(ComponentContext.class)) { + componentContext.when(() -> ComponentContext.inject(VirtualMachineEntityImpl.class)).thenReturn(vmEntity); + + cloudOrchestrator.createVirtualMachine("vm-uuid", "owner", TEMPLATE_ID, "host", "display", HypervisorType.KVM.name(), + 1, 1000, 1024, null, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null, + null, null, null, null, ROOT_DISK_OFFERING_ID, null, null, null, null); + } + + ArgumentCaptor rootDiskOfferingInfo = ArgumentCaptor.forClass(DiskOfferingInfo.class); + Mockito.verify(_itMgr).allocate(Mockito.eq("i-1-1-VM"), Mockito.eq(template), Mockito.eq(serviceOffering), rootDiskOfferingInfo.capture(), + Mockito.anyList(), Mockito.anyList(), Mockito.any(LinkedHashMap.class), Mockito.isNull(), Mockito.eq(HypervisorType.KVM), + Mockito.isNull(), Mockito.isNull(), Mockito.isNull(), Mockito.isNull()); + + Assert.assertEquals(Long.valueOf(100), rootDiskOfferingInfo.getValue().getMinIops()); + Assert.assertEquals(Long.valueOf(1000), rootDiskOfferingInfo.getValue().getMaxIops()); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 4cd9a8e23bfc..16b8eb975ac4 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -182,6 +182,13 @@ public interface VolumeDao extends GenericDao, StateDao implements Vol protected static final String SELECT_HYPERTYPE_FROM_CLUSTER_VOLUME = "SELECT c.hypervisor_type from volumes v, storage_pool s, cluster c where v.pool_id = s.id and s.cluster_id = c.id and v.id = ?"; protected static final String SELECT_HYPERTYPE_FROM_ZONE_VOLUME = "SELECT s.hypervisor from volumes v, storage_pool s where v.pool_id = s.id and v.id = ?"; protected static final String SELECT_POOLSCOPE = "SELECT s.scope from storage_pool s, volumes v where s.id = v.pool_id and v.id = ?"; + // Looks for a VM whose Ready root disk is on this pool and whose attached Ready data disk is on another pool. + // LIMIT 1 stops after the first match so we do not load all volumes. + private static final String HAS_MULTI_PRIMARY_STORAGE_POOL_VM = + "SELECT 1 FROM volumes root INNER JOIN volumes data ON data.instance_id = root.instance_id " + + "WHERE root.pool_id = ? AND root.volume_type = 'ROOT' AND root.instance_id IS NOT NULL " + + "AND root.removed IS NULL AND root.state = 'Ready' " + + "AND data.volume_type = 'DATADISK' AND data.pool_id IS NOT NULL AND data.pool_id <> ? " + + "AND data.removed IS NULL AND data.state = 'Ready' LIMIT 1"; private static final String ORDER_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT_PART1 = "SELECT pool.id, SUM(IF(vol.state='Ready' AND vol.account_id = ?, 1, 0)) FROM `cloud`.`storage_pool` pool LEFT JOIN `cloud`.`volumes` vol ON pool.id = vol.pool_id WHERE pool.data_center_id = ? "; private static final String ORDER_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT_PART2 = " GROUP BY pool.id ORDER BY 2 ASC "; @@ -998,6 +1006,21 @@ public boolean existsWithKmsKey(long kmsKeyId) { return findOneBy(sc) != null; } + @Override + @DB + public boolean hasMultiPrimaryStoragePoolVm(long poolId) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(HAS_MULTI_PRIMARY_STORAGE_POOL_VM)) { + pstmt.setLong(1, poolId); + pstmt.setLong(2, poolId); + try (ResultSet rs = pstmt.executeQuery()) { + return rs.next(); + } + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + HAS_MULTI_PRIMARY_STORAGE_POOL_VM, e); + } + } + public VolumeVO findByExternalUuid(String externalUuid) { SearchCriteria sc = ExternalUuidSearch.create(); sc.setParameters("externalUuid", externalUuid); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 8f59321627db..0141c90d45ad 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -226,3 +226,25 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` ( PRIMARY KEY (`id`), CONSTRAINT `fk_instance_boot_group_details__group_id` FOREIGN KEY (`boot_group_id`) REFERENCES `instance_boot_group` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Re-tag NetApp ONTAP iSCSI storage pools to the dedicated 'OntapiSCSI' pool type. +UPDATE `cloud`.`storage_pool` +SET `pool_type` = 'OntapiSCSI' +WHERE `storage_provider_name` = 'NetApp ONTAP' + AND `pool_type` = 'Iscsi'; + +-- The denormalized copy on every volume that lives on such a pool. +UPDATE `cloud`.`volumes` v + JOIN `cloud`.`storage_pool` sp ON sp.`id` = v.`pool_id` + SET v.`pool_type` = 'OntapiSCSI' +WHERE sp.`storage_provider_name` = 'NetApp ONTAP' + AND sp.`pool_type` IN ('Iscsi', 'OntapiSCSI') + AND v.`pool_type` = 'Iscsi'; + +-- Volume format realignment. +UPDATE `cloud`.`volumes` v + JOIN `cloud`.`storage_pool` sp ON sp.`id` = v.`pool_id` + SET v.`format` = 'RAW' +WHERE sp.`storage_provider_name` = 'NetApp ONTAP' + AND sp.`pool_type` IN ('Iscsi', 'OntapiSCSI') + AND (v.`format` IS NULL OR v.`format` <> 'RAW'); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 7674f1ce25a1..dec21c805395 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -616,8 +616,8 @@ private void handleVolumeMigrationFromManagedStorageToNonManagedStorage(VolumeIn private void verifyFormatWithPoolType(ImageFormat imageFormat, StoragePoolType poolType) { if (imageFormat != ImageFormat.VHD && imageFormat != ImageFormat.OVA && imageFormat != ImageFormat.QCOW2 && !(imageFormat == ImageFormat.RAW && (StoragePoolType.PowerFlex == poolType || - StoragePoolType.FiberChannel == poolType))) { - throw new CloudRuntimeException(String.format("Only the following image types are currently supported: %s, %s, %s, %s (for PowerFlex and FiberChannel)", + StoragePoolType.FiberChannel == poolType || StoragePoolType.OntapiSCSI == poolType))) { + throw new CloudRuntimeException(String.format("Only the following image types are currently supported: %s, %s, %s, %s (for PowerFlex, FiberChannel and OntapiSCSI)", ImageFormat.VHD.toString(), ImageFormat.OVA.toString(), ImageFormat.QCOW2.toString(), ImageFormat.RAW.toString())); } } @@ -1787,7 +1787,11 @@ else if (HypervisorType.KVM.equals(snapshotInfo.getHypervisorType())) { VolumeObjectTO newVolume = new VolumeObjectTO(); newVolume.setSize(volumeInfo.getSize()); - newVolume.setPath(volumeInfo.get_iScsiName()); + String path = volumeInfo.get_iScsiName(); + if (path == null) { + path = volumeInfo.getPath(); + } + newVolume.setPath(path); newVolume.setFormat(volumeInfo.getFormat()); copyCmdAnswer = new CopyCmdAnswer(newVolume); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java index 4cb1d0361aa9..ea921d4d7a9e 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java @@ -27,7 +27,6 @@ import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; - import javax.inject.Inject; import com.cloud.utils.StringUtils; diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 69b40f64af19..df3a7cd7e29b 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -765,7 +765,11 @@ protected Void managedCopyBaseImageCallback(AsyncCallbackDispatcher/device/delete — the standard Linux kernel SCSI - * API for removing a single device without tearing down the entire iSCSI session. - * Once the kernel processes the delete, it also removes the by-path symlink. - * - * This is used instead of iscsiadm --logout when other LUNs on the same IQN are still - * active (ONTAP single-IQN-per-SVM model), since logout would tear down ALL LUNs. - */ - private void removeStaleScsiDevice(String host, int port, String iqn, String lun) { - String byPath = getByPath(host, port, "/" + iqn + "/" + lun); - Path byPathLink = Paths.get(byPath); - if (!Files.exists(byPathLink)) { - logger.debug("by-path entry for LUN " + lun + " already gone, nothing to remove"); - return; - } - try { - Path realDevice = byPathLink.toRealPath(); - String devName = realDevice.getFileName().toString(); - File deleteFile = new File("/sys/block/" + devName + "/device/delete"); - if (!deleteFile.exists()) { - logger.warn("sysfs delete entry not found for device " + devName + " — cannot remove stale SCSI device"); - return; - } - try (FileWriter fw = new FileWriter(deleteFile)) { - fw.write("1"); - } - logger.info("Removed stale SCSI device " + devName + " for LUN /" + iqn + "/" + lun + " via sysfs"); - } catch (Exception e) { - logger.warn("Failed to remove stale SCSI device for LUN /" + iqn + "/" + lun + ": " + e.getMessage()); - } - } - private boolean disconnectPhysicalDisk(String host, int port, String iqn, String lun) { - // Check if other LUNs on the same IQN target are still in use. - // ONTAP (and similar) uses a single IQN per SVM with multiple LUNs. - // Doing iscsiadm --logout tears down the ENTIRE target session, - // which would destroy access to ALL LUNs — not just the one being disconnected. - if (hasOtherActiveLuns(host, port, iqn, lun)) { - logger.info("Skipping iSCSI logout for /" + iqn + "/" + lun + - " — other LUNs on the same target are still active. Removing stale SCSI device for this LUN only."); - removeStaleScsiDevice(host, port, iqn, lun); - // After removing this LUN's device, re-check: if no other LUNs remain active, - // If it is the last one then must logout to clean up the iSCSI session entirely. - if (hasOtherActiveLuns(host, port, iqn, lun)) { - logger.info("Other LUNs still active after removing /" + iqn + "/" + lun + " — session kept alive."); - return true; - } - logger.info("No more active LUNs on target after removing /" + iqn + "/" + lun + " — proceeding with iSCSI logout."); - } - - // No other LUNs active on this target — safe to logout and delete the node record. + // use iscsiadm to log out of the iSCSI target and un-discover it // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 --logout Script iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger); @@ -536,8 +387,8 @@ public List listPhysicalDisks(String storagePoolUuid, KVMStorag @Override public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, - ProvisioningType provisioningType, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { + ProvisioningType provisioningType, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { throw new UnsupportedOperationException("Creating a disk from a template is not yet supported for this configuration."); } @@ -571,19 +422,6 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolu try { QemuImg q = new QemuImg(timeout); q.convert(srcFile, destFile); - // Below fix is required when vendor depends on host based copy rather than storage CAN_CREATE_VOLUME_FROM_VOLUME capability - // When host based template copy is triggered , small size template sits in RAM(depending on host memory and RAM) and copy is marked successful and by the time flush to storage is triggered - // disconnectPhysicalDisk would disconnect the lun , hence template staying in RAM is not copied to storage lun. Below does flushing of data to storage and marking - // copy as successful once flush is complete. - Script flushCmd = new Script(true, "blockdev", 0, logger); - flushCmd.add("--flushbufs", destDisk.getPath()); - String flushResult = flushCmd.execute(); - if (flushResult != null) { - logger.warn("iSCSI copyPhysicalDisk: blockdev --flushbufs returned: {}", flushResult); - } - Script syncCmd = new Script(true, "sync", 0, logger); - syncCmd.execute(); - logger.info("iSCSI copyPhysicalDisk: flush/sync completed "); } catch (QemuImgException | LibvirtException ex) { String msg = "Failed to copy data from " + srcDisk.getPath() + " to " + destDisk.getPath() + ". The error was the following: " + ex.getMessage(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index d99847fd921a..3c1318dbec51 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -448,7 +448,7 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) if (type == StoragePoolType.NetworkFilesystem) { _haMonitor.removeStoragePool(uuid); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid);; + boolean deleteStatus = adaptor.deleteStoragePool(uuid); synchronized (_storagePools) { _storagePools.remove(uuid); } @@ -457,10 +457,13 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); + boolean deleteStatus; if (type == StoragePoolType.NetworkFilesystem) { - _haMonitor.removeStoragePool(uuid); + _haMonitor.unregisterStoragePool(uuid); + deleteStatus = adaptor.deleteStoragePool(uuid); + } else { + deleteStatus = adaptor.deleteStoragePool(uuid, details); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); synchronized (_storagePools) { _storagePools.remove(uuid); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 0cb32da318ea..630d02d3c480 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -421,7 +421,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { StoragePoolType.PowerFlex, StoragePoolType.Linstor, StoragePoolType.FiberChannel, - StoragePoolType.CLVM).contains(primaryPool.getType())) { + StoragePoolType.CLVM, + StoragePoolType.OntapiSCSI).contains(primaryPool.getType())) { newTemplate.setFormat(ImageFormat.RAW); } else { newTemplate.setFormat(ImageFormat.QCOW2); @@ -3513,7 +3514,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) { StoragePoolType.PowerFlex, StoragePoolType.Linstor, StoragePoolType.FiberChannel, - StoragePoolType.CLVM).contains(poolType)) { + StoragePoolType.CLVM, + StoragePoolType.OntapiSCSI).contains(poolType)) { return ImageFormat.RAW; } else { return ImageFormat.QCOW2; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/OntapIscsiStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/OntapIscsiStorageAdaptor.java new file mode 100644 index 000000000000..4e07a5023ce0 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/OntapIscsiStorageAdaptor.java @@ -0,0 +1,753 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.storage; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.libvirt.LibvirtException; + +import com.cloud.agent.api.to.DiskTO; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; + +/** + * Serves {@link StoragePoolType#OntapiSCSI} pools: + * + * This must stay in the {@code com.cloud.hypervisor.kvm.storage} package: {@link KVMStoragePoolManager} + * discovers adaptors by a Reflections scan of that package alone, and an unregistered type silently + * falls back to {@link LibvirtStorageAdaptor} rather than failing at startup. + */ +public class OntapIscsiStorageAdaptor implements StorageAdaptor { + + protected Logger logger = LogManager.getLogger(getClass()); + private static final Map MAP_STORAGE_UUID_TO_STORAGE_POOL = new HashMap<>(); + private static final String BY_ID_WWN_PREFIX = "/dev/disk/by-id/wwn-0x"; + + /** + * A LUN WWID is NetApp's NAA prefix (type 6, OUI 00a098, vendor nibble 0) followed by the hex of + * a 12-character serial: 32 hex digits. + * Anchoring on the prefix is what stops this adaptor claiming another vendor's WWN device + * when KVMStoragePoolManager.disconnectPhysicalDiskByPath scans every adaptor. + */ + private static final String NETAPP_NAA_PREFIX = "600a0980"; + private static final Pattern LUN_WWID = Pattern.compile(NETAPP_NAA_PREFIX + "[0-9a-f]{24}"); + + private static final String SYS_BLOCK = "/sys/block"; + private static final String SYS_ISCSI_SESSION = "/sys/class/iscsi_session"; + private static final String SESSION_DIR_PREFIX = "session"; + + /** iscsiadm's ISCSI_ERR_NO_OBJS_FOUND: returned by "-m session" when no session is established. */ + private static final int ISCSI_ERR_NO_OBJS_FOUND = 21; + + /** iscsiadm's ISCSI_ERR_SESS_EXISTS: returned by "--login" when already logged in (e.g. Ubuntu). */ + private static final int ISCSI_SESSION_EXISTS_CODE = 15; + + private static final int DEVICE_WAIT_TRIES = 15; + private static final int DEVICE_WAIT_INTERVAL_MS = 1000; + private static final int DEFAULT_ISCSI_PORT = 3260; + + @Override + public StoragePoolType getStoragePoolType() { + return StoragePoolType.OntapiSCSI; + } + + @Override + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, + StoragePoolType storagePoolType, Map details, boolean isPrimaryStorage) { + IscsiAdmStoragePool storagePool = new IscsiAdmStoragePool(uuid, host, port, storagePoolType, this); + + MAP_STORAGE_UUID_TO_STORAGE_POOL.put(uuid, storagePool); + + return storagePool; + } + + @Override + public KVMStoragePool getStoragePool(String uuid) { + return MAP_STORAGE_UUID_TO_STORAGE_POOL.get(uuid); + } + + @Override + public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { + return MAP_STORAGE_UUID_TO_STORAGE_POOL.get(uuid); + } + + @Override + public boolean deleteStoragePool(String uuid) { + return MAP_STORAGE_UUID_TO_STORAGE_POOL.remove(uuid) != null; + } + + @Override + public boolean deleteStoragePool(KVMStoragePool pool) { + return deleteStoragePool(pool.getUuid()); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { + String devicePath = getDeviceById(getLunWwid(volumeUuid)); + + KVMPhysicalDisk physicalDisk = new KVMPhysicalDisk(devicePath, volumeUuid, pool); + physicalDisk.setFormat(PhysicalDiskFormat.RAW); + + long deviceSize = getDeviceSize(devicePath); + + physicalDisk.setSize(deviceSize); + physicalDisk.setVirtualSize(deviceSize); + + return physicalDisk; + } + + @Override + public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map details, boolean isVMMigrate) { + final String host = pool.getSourceHost(); + final int port = pool.getSourcePort(); + final String iqn = getTargetIqn(volumePath); + + if (!createIscsiNode(host, port, iqn, volumePath)) { + return false; + } + + if (!applyChapCredentials(host, port, iqn, volumePath, details)) { + return false; + } + + if (!loginOrRescanExistingSession(iqn, host, port, volumePath)) { + return false; + } + + // Logging in can return before the kernel has added the device, so the disk is not usable + // the moment iscsiadm succeeds. Waiting on a non-zero size also guards against reporting + // success when the device never appears, which would otherwise let a caller create a plain + // file where the LUN device was expected. + if (!waitForDeviceToAppear(volumePath, pool)) { + logger.warn("iSCSI device for LUN {} on target {} at {}:{} did not become available", + getLunWwid(volumePath), iqn, host, port); + return false; + } + + return true; + } + + @Override + public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { + return disconnectLun(pool.getSourceHost(), pool.getSourcePort(), getTargetIqn(volumePath), getLunWwid(volumePath)); + } + + @Override + public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { + String host = volumeToDisconnect.get(DiskTO.STORAGE_HOST); + String port = volumeToDisconnect.get(DiskTO.STORAGE_PORT); + String path = volumeToDisconnect.get(DiskTO.IQN); + + if (host == null || port == null || path == null) { + return false; + } + + return disconnectLun(host, Integer.parseInt(port), getTargetIqn(path), getLunWwid(path)); + } + + /** + * Claims the by-id devices this adaptor hands out. + * + * The target IQN and portal are not recoverable from a by-id name, so they are read back from + * the device's iSCSI session in sysfs. + * + * Returning false for anything else is required by the {@link StorageAdaptor} contract: + * {@code KVMStoragePoolManager.disconnectPhysicalDiskByPath} scans every adaptor and stops at + * the first one that claims the path. + */ + @Override + public boolean disconnectPhysicalDiskByPath(String localPath) { + if (!isOntapDevicePath(localPath)) { + return false; + } + + String kernelDevice = resolveKernelDevice(localPath); + if (kernelDevice == null) { + logger.info("Device {} is already gone, nothing to disconnect", localPath); + return true; + } + + Integer sessionId = findSessionId(kernelDevice); + String iqn = sessionId == null ? null : readSessionAttribute(sessionId, "targetname"); + if (iqn == null) { + // A real ONTAP LUN always sits under an iSCSI session. If this one does not, it is not + // ours, so leave it alone and let the manager try the next adaptor. + logger.debug("Device {} ({}) is not on a readable iSCSI session, not claiming it", + localPath, kernelDevice); + return false; + } + + return disconnectLun(null, 0, iqn, localPath.substring(BY_ID_WWN_PREFIX.length())); + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) { + return copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolumeUuid, KVMStoragePool destPool, + int timeout, byte[] srcPassphrase, byte[] destPassphrase, ProvisioningType provisioningType) { + KVMStoragePool srcPool = srcDisk.getPool(); + QemuImgFile srcFile = srcPool.getType() == StoragePoolType.RBD + ? new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, srcDisk.getPath()), srcDisk.getFormat()) + : new QemuImgFile(srcDisk.getPath(), srcDisk.getFormat()); + + KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(destVolumeUuid); + QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); + + try { + new QemuImg(timeout).convert(srcFile, destFile); + + // A small template can still be sitting in the page cache when convert returns. The LUN + // is disconnected right after a copy, so without an explicit flush that data would never + // reach the array and the copy would be reported successful while the LUN stayed empty. + flushToDevice(destDisk.getPath()); + } catch (QemuImgException | LibvirtException ex) { + String msg = "Failed to copy data from " + srcDisk.getPath() + " to " + destDisk.getPath() + + ". The error was the following: " + ex.getMessage(); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + + return destPool.getPhysicalDisk(destVolumeUuid); + } + + @Override + public boolean refresh(KVMStoragePool pool) { + return true; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, KVMStoragePool pool, PhysicalDiskFormat format, + ProvisioningType provisioningType, long size, byte[] passphrase) { + throw new UnsupportedOperationException("Creating a physical disk is not supported; ONTAP LUNs are provisioned by the management server."); + } + + @Override + public boolean deletePhysicalDisk(String volumeUuid, KVMStoragePool pool, Storage.ImageFormat format) { + throw new UnsupportedOperationException("Deleting a physical disk is not supported; ONTAP LUNs are removed by the management server."); + } + + @Override + public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { + throw new UnsupportedOperationException("Listing disks is not supported for this configuration."); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, + ProvisioningType provisioningType, long size, KVMStoragePool destPool, + int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Creating a disk from a template is not supported for this configuration."); + } + + @Override + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + throw new UnsupportedOperationException("Creating a template from a disk is not supported for this configuration."); + } + + @Override + public boolean createFolder(String uuid, String path) { + return createFolder(uuid, path, null); + } + + @Override + public boolean createFolder(String uuid, String path, String localPath) { + throw new UnsupportedOperationException("A folder cannot be created in this configuration."); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, + long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + return null; + } + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, + KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + return null; + } + + private boolean createIscsiNode(String host, int port, String iqn, String volumePath) { + String result = runIscsiadmNodeCommand(host, port, iqn, "-o", "new"); + + if (result == null) { + logger.debug("Added iSCSI node for target {}", iqn); + return true; + } + if (result.toLowerCase().contains("exists")) { + logger.debug("iSCSI node already exists for target {}, proceeding", iqn); + return true; + } + logger.warn("Failed to add iSCSI node for {}: {}", volumePath, result); + return false; + } + + private boolean applyChapCredentials(String host, int port, String iqn, String volumePath, Map details) { + if (details == null) { + return true; + } + + String username = details.get(DiskTO.CHAP_INITIATOR_USERNAME); + String secret = details.get(DiskTO.CHAP_INITIATOR_SECRET); + + if (!StringUtils.isNoneBlank(username, secret)) { + return true; + } + + return updateNodeSetting(host, port, iqn, "node.session.auth.authmethod", "CHAP", volumePath) + && updateNodeSetting(host, port, iqn, "node.session.auth.username", username, volumePath) + && updateNodeSetting(host, port, iqn, "node.session.auth.password", secret, volumePath); + } + + private boolean updateNodeSetting(String host, int port, String iqn, String name, String value, String volumePath) { + String result = runIscsiadmNodeCommand(host, port, iqn, "--op", "update", "-n", name, "-v", value); + if (result != null) { + // The value is not logged: for the CHAP settings it is a credential. + logger.warn("Failed to set {} on iSCSI target {} for {}: {}", name, iqn, volumePath, result); + return false; + } + return true; + } + + /** + * Logs in, rescanning instead when the session was already established. + * + * Login is idempotent but its exit status is not portable: re-login exits 0 on some + * distributions and ISCSI_ERR_SESS_EXISTS on others, where it also produces an error message + * that would otherwise read as a failure. The session is therefore checked beforehand and a + * pre-existing session treated as success. + */ + private boolean loginOrRescanExistingSession(String iqn, String host, int port, String volumePath) { + boolean sessionAlreadyActive = isIscsiSessionActive(iqn, host); + + Script login = new Script(true, "iscsiadm", 0, logger); + login.add("-m", "node"); + login.add("-T", iqn); + login.add("-p", host + ":" + port); + login.add("--login"); + + String result = login.execute(); + boolean sessionPreExisted = login.getExitValue() == ISCSI_SESSION_EXISTS_CODE || sessionAlreadyActive; + + if (sessionPreExisted) { + logger.debug("iSCSI session for target {} at {}:{} pre-existed, rescanning", iqn, host, port); + rescanIscsiSession(host, port, iqn); + return true; + } + if (result == null) { + logger.debug("Logged in to iSCSI target {} for {}", iqn, volumePath); + return true; + } + logger.warn("Failed to log in to iSCSI target {} for {}: {}", iqn, volumePath, result); + return false; + } + + /** + * Reports whether a session to this target and portal already exists. + * + * ISCSI_ERR_NO_OBJS_FOUND simply means no session exists, which is a normal outcome here; any + * other non-zero exit is treated as "not confirmed active" so that login is still attempted. + */ + private boolean isIscsiSessionActive(String iqn, String host) { + Script sessionCmd = new Script(true, "iscsiadm", 0, logger); + sessionCmd.add("-m", "session"); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + sessionCmd.executeIgnoreExitValue(parser, ISCSI_ERR_NO_OBJS_FOUND); + + int exitValue = sessionCmd.getExitValue(); + if (exitValue != 0 && exitValue != ISCSI_ERR_NO_OBJS_FOUND) { + logger.warn("Unable to determine iSCSI session state for target {} at {}: 'iscsiadm -m session' exited with {}", + iqn, host, exitValue); + return false; + } + + String sessions = parser.getLines(); + if (StringUtils.isBlank(sessions)) { + return false; + } + for (String line : sessions.split("\n")) { + if (line.contains(iqn) && line.contains(host)) { + return true; + } + } + return false; + } + + private void rescanIscsiSession(String host, int port, String iqn) { + String result = runIscsiadmNodeCommand(host, port, iqn, "--rescan"); + if (result != null) { + logger.warn("iSCSI session rescan of target {} returned: {}", iqn, result); + } + } + + /** + * Releases one LUN, tearing the session down only once it is the last one on the target. + * + * ONTAP presents all of an SVM's LUNs through a single IQN, so {@code iscsiadm --logout} would + * drop every LUN on the session rather than just this one. While siblings remain, the LUN is + * released by deleting its SCSI device through sysfs instead; the kernel does not remove + * devices from a live session on its own when the array unmaps a LUN. + * + * @param host the portal address, or null to read it back from the device's session + */ + private boolean disconnectLun(String host, int port, String iqn, String lunWwid) { + String devicePath = getDeviceById(lunWwid); + String kernelDevice = resolveKernelDevice(devicePath); + + if (kernelDevice == null) { + logger.info("Device {} for LUN {} on target {} is already gone; nothing to disconnect", devicePath, lunWwid, iqn); + return true; + } + + Integer sessionId = findSessionId(kernelDevice); + if (sessionId == null) { + logger.warn("Unable to resolve the iSCSI session id for device {} (LUN {}, target {}); " + + "Removing the SCSI device only ", kernelDevice, lunWwid, iqn); + removeScsiDevice(kernelDevice); + return true; + } + if (hasOtherLunsInSession(sessionId, kernelDevice)) { + logger.info("Skipping iSCSI logout for LUN {} on target {}: other LUNs on the same session are still " + + "active. Removing device {} only.", lunWwid, iqn, kernelDevice); + removeScsiDevice(kernelDevice); + return true; + } + + if (host == null) { + host = readConnectionAttribute(sessionId, "persistent_address"); + port = parsePort(readConnectionAttribute(sessionId, "persistent_port")); + } + if (host == null) { + logger.warn("Unable to determine the portal of target {} even with sessionId={}; removing device {} without logging out", + iqn, sessionId, kernelDevice); + removeScsiDevice(kernelDevice); + return true; + } + + if (!logoutAndForgetTarget(host, port, iqn, lunWwid)) { + return false; + } + + waitForDeviceToDisappear(devicePath); + return true; + } + + private boolean logoutAndForgetTarget(String host, int port, String iqn, String lunWwid) { + String logoutResult = runIscsiadmNodeCommand(host, port, iqn, "--logout"); + if (logoutResult != null) { + logger.warn("Failed to log out of iSCSI target {} while releasing LUN {}: {}", iqn, lunWwid, logoutResult); + return false; + } + logger.debug("Logged out of iSCSI target {} while releasing LUN {}", iqn, lunWwid); + + String deleteResult = runIscsiadmNodeCommand(host, port, iqn, "-o", "delete"); + if (deleteResult != null) { + logger.warn("Failed to delete the iSCSI node record for target {}: {}", iqn, deleteResult); + return false; + } + logger.debug("Deleted the iSCSI node record for target {}", iqn); + + return true; + } + + /** @return null on success, or the command output describing the failure */ + private String runIscsiadmNodeCommand(String host, int port, String iqn, String... operation) { + Script command = new Script(true, "iscsiadm", 0, logger); + command.add("-m", "node"); + command.add("-T", iqn); + command.add("-p", host + ":" + port); + for (String argument : operation) { + command.add(argument); + } + return command.execute(); + } + + // --------------------------------------------------------------------------------------------- + // Device and sysfs handling + // --------------------------------------------------------------------------------------------- + + /** + * Removes a single SCSI device from the kernel, the standard alternative to tearing down the + * whole session. The kernel removes the device's udev aliases once it processes the delete. + */ + private void removeScsiDevice(String kernelDevice) { + File deleteFile = new File(SYS_BLOCK + "/" + kernelDevice + "/device/delete"); + if (!deleteFile.exists()) { + logger.warn("No sysfs delete entry for device {}; cannot remove it", kernelDevice); + return; + } + try (FileWriter writer = new FileWriter(deleteFile)) { + writer.write("1"); + logger.info("Removed SCSI device {} via sysfs", kernelDevice); + } catch (IOException ex) { + logger.warn("Failed to remove SCSI device {}: {}", kernelDevice, ex.getMessage()); + } + } + + /** + * Reports whether the session carries any LUN other than {@code ownDevice}. + * + * Every SCSI disk reachable through an iSCSI session has that session in its sysfs device path, + * so comparing session ids across {@code /sys/block} identifies the siblings without needing to + * know any logical unit numbers. + */ + private boolean hasOtherLunsInSession(int sessionId, String ownDevice) { + File[] blockDevices = new File(SYS_BLOCK).listFiles(); + if (blockDevices == null) { + return false; + } + for (File blockDevice : blockDevices) { + String name = blockDevice.getName(); + if (name.equals(ownDevice)) { + continue; + } + Integer otherSessionId = findSessionId(name); + if (otherSessionId != null && otherSessionId == sessionId) { + logger.debug("Device {} is another LUN on iSCSI session {}", name, sessionId); + return true; + } + } + return false; + } + + /** + * Extracts the iSCSI session id from a block device's sysfs path, which looks like + * {@code /sys/devices/platform/host33/session13/target33:0:0/33:0:0:1/block/sdc}. + * + * @return the session id, or null if the device is not backed by iSCSI + */ + private Integer findSessionId(String kernelDevice) { + try { + Path deviceLink = Paths.get(SYS_BLOCK, kernelDevice, "device"); + if (!Files.exists(deviceLink)) { + return null; + } + for (Path element : deviceLink.toRealPath()) { + String name = element.toString(); + if (name.startsWith(SESSION_DIR_PREFIX)) { + return Integer.parseInt(name.substring(SESSION_DIR_PREFIX.length())); + } + } + } catch (IOException | NumberFormatException ex) { + logger.debug("Unable to determine the iSCSI session of device {}: {}", kernelDevice, ex.getMessage()); + } + return null; + } + + private String readSessionAttribute(int sessionId, String attribute) { + return readSysfsValue(Paths.get(SYS_ISCSI_SESSION, SESSION_DIR_PREFIX + sessionId, attribute)); + } + + /** + * Reads a connection attribute of a session. A connection's own index is not guaranteed to + * match its session's, so the connection directory is discovered under the session rather than + * assumed to be {@code connection:0}. + */ + private String readConnectionAttribute(int sessionId, String attribute) { + Path sessionDevice = Paths.get(SYS_ISCSI_SESSION, SESSION_DIR_PREFIX + sessionId, "device"); + File[] children = sessionDevice.toFile().listFiles(); + if (children == null) { + return null; + } + for (File child : children) { + String name = child.getName(); + if (!name.startsWith("connection")) { + continue; + } + String value = readSysfsValue(sessionDevice.resolve(name).resolve("iscsi_connection").resolve(name).resolve(attribute)); + if (value != null) { + return value; + } + } + return null; + } + + private String readSysfsValue(Path path) { + try { + if (!Files.exists(path)) { + return null; + } + String value = new String(Files.readAllBytes(path)).trim(); + return value.isEmpty() ? null : value; + } catch (IOException ex) { + logger.debug("Unable to read {}: {}", path, ex.getMessage()); + return null; + } + } + + private int parsePort(String port) { + try { + return port != null ? Integer.parseInt(port) : DEFAULT_ISCSI_PORT; + } catch (NumberFormatException ex) { + return DEFAULT_ISCSI_PORT; + } + } + + private String resolveKernelDevice(String devicePath) { + try { + Path link = Paths.get(devicePath); + if (!Files.exists(link)) { + return null; + } + return link.toRealPath().getFileName().toString(); + } catch (IOException ex) { + logger.debug("Unable to resolve {}: {}", devicePath, ex.getMessage()); + return null; + } + } + + private boolean waitForDeviceToAppear(String volumePath, KVMStoragePool pool) { + for (int attempt = 0; attempt < DEVICE_WAIT_TRIES; attempt++) { + if (getPhysicalDisk(volumePath, pool).getSize() > 0) { + return true; + } + if (!sleepBetweenAttempts()) { + return false; + } + } + return getPhysicalDisk(volumePath, pool).getSize() > 0; + } + + private void waitForDeviceToDisappear(String devicePath) { + for (int attempt = 0; attempt < DEVICE_WAIT_TRIES && getDeviceSize(devicePath) > 0; attempt++) { + if (!sleepBetweenAttempts()) { + return; + } + } + } + + private boolean sleepBetweenAttempts() { + try { + Thread.sleep(DEVICE_WAIT_INTERVAL_MS); + return true; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** + * @return the device size in bytes, or 0 if the device is absent or not a block device + */ + private long getDeviceSize(String devicePath) { + Path path = Paths.get(devicePath); + + if (!Files.exists(path)) { + logger.debug("Device does not exist yet: {}", devicePath); + return 0L; + } + if (Files.isRegularFile(path)) { + // A plain file here means something wrote to the device name before the LUN appeared; + // treating it as a disk would silently back a volume with local storage. + logger.warn("Found a regular file at {} where a block device was expected; it must be removed manually", devicePath); + return 0L; + } + + Script command = new Script(true, "blockdev", 0, logger); + command.add("--getsize64", devicePath); + + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = command.execute(parser); + + if (result != null) { + logger.warn("Unable to get the size of device {}: {}", devicePath, result); + return 0L; + } + + try { + return Long.parseLong(parser.getLine().trim()); + } catch (NumberFormatException | NullPointerException ex) { + logger.warn("Unable to parse the size of device {}", devicePath); + return 0L; + } + } + + private void flushToDevice(String devicePath) { + Script flush = new Script(true, "blockdev", 0, logger); + flush.add("--flushbufs", devicePath); + String flushResult = flush.execute(); + if (flushResult != null) { + logger.warn("blockdev --flushbufs on {} returned: {}", devicePath, flushResult); + } + new Script(true, "sync", 0, logger).execute(); + logger.debug("Flushed buffers to {}", devicePath); + } + + // --------------------------------------------------------------------------------------------- + // Volume path handling + // --------------------------------------------------------------------------------------------- + + private boolean isOntapDevicePath(String localPath) { + return localPath != null + && localPath.startsWith(BY_ID_WWN_PREFIX) + && LUN_WWID.matcher(localPath.substring(BY_ID_WWN_PREFIX.length())).matches(); + } + + private String getDeviceById(String lunWwid) { + return BY_ID_WWN_PREFIX + lunWwid; + } + + private String getTargetIqn(String volumePath) { + return getPathComponent(volumePath, 1); + } + + private String getLunWwid(String volumePath) { + return getPathComponent(volumePath, 2); + } + + /** + * Splits a {@code //} volume path. + * + * The WWID occupies the component a logical unit number would hold for other iSCSI vendors, so + * the path keeps the two-component shape every hypervisor resource expects. That is also why + * the path carries the WWID rather than the ONTAP serial it encodes: serials contain '/'. + */ + private String getPathComponent(String volumePath, int index) { + String[] components = volumePath == null ? new String[0] : volumePath.split("/"); + + if (components.length != 3) { + String message = "Wrong format for ONTAP iSCSI path: " + volumePath + + ". It should be formatted as '/targetIQN/lunWwid'."; + logger.warn(message); + throw new CloudRuntimeException(message); + } + + return components[index].trim(); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/OntapIscsiStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/OntapIscsiStorageAdaptorTest.java new file mode 100644 index 000000000000..dc3ced853f5c --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/OntapIscsiStorageAdaptorTest.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.storage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Modifier; +import java.util.Set; + +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.junit.Test; +import org.reflections.Reflections; + +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; + +public class OntapIscsiStorageAdaptorTest { + + @Test + public void getStoragePoolTypeReturnsOntapIscsi() { + assertEquals(StoragePoolType.OntapiSCSI, new OntapIscsiStorageAdaptor().getStoragePoolType()); + } + + @Test + public void createdPoolCarriesOntapIscsiTypeAndRawFormat() { + OntapIscsiStorageAdaptor adaptor = new OntapIscsiStorageAdaptor(); + + KVMStoragePool pool = adaptor.createStoragePool("ontap-iscsi-pool-uuid", "10.0.0.1", 3260, null, null, + StoragePoolType.OntapiSCSI, null, true); + + assertEquals(StoragePoolType.OntapiSCSI, pool.getType()); + // Attach builds a block-based disk off the physical disk format rather than the pool type, + // which is why splitting OntapiSCSI out of Iscsi leaves the generated domain XML unchanged. + assertEquals(PhysicalDiskFormat.RAW, pool.getDefaultFormat()); + assertSame(pool, adaptor.getStoragePool("ontap-iscsi-pool-uuid")); + } + + /** + * KVMStoragePoolManager discovers adaptors by a Reflections scan of its own package, instantiating + * each concrete implementation through a no-arg constructor and keying it on getStoragePoolType(). + * A type with no adaptor silently falls back to LibvirtStorageAdaptor instead of failing at + * startup, so this reproduces the discovery preconditions rather than waiting for the symptom. + * The manager itself is not constructed here because doing so also instantiates + * MultipathSCSIAdapterBase, which requires agent scripts resolvable from the working directory. + */ + @Test + public void adaptorSatisfiesThePoolManagerDiscoveryContract() throws ReflectiveOperationException { + String scannedPackage = KVMStoragePoolManager.class.getPackage().getName(); + Set> discovered = + new Reflections(scannedPackage).getSubTypesOf(StorageAdaptor.class); + + assertTrue("OntapIscsiStorageAdaptor must live in " + scannedPackage + " to be discovered", + discovered.contains(OntapIscsiStorageAdaptor.class)); + assertFalse("An abstract adaptor is skipped by the scan", + Modifier.isAbstract(OntapIscsiStorageAdaptor.class.getModifiers())); + + StorageAdaptor adaptor = OntapIscsiStorageAdaptor.class.getDeclaredConstructor().newInstance(); + assertEquals(StoragePoolType.OntapiSCSI, adaptor.getStoragePoolType()); + assertEquals("IscsiAdmStorageAdaptor must keep serving the other iSCSI vendors", + StoragePoolType.Iscsi, new IscsiAdmStorageAdaptor().getStoragePoolType()); + } + + /** + * The two adaptors are deliberately unrelated: device naming runs through nearly every method + * that does real work, so inheriting IscsiAdmStorageAdaptor coupled them through behaviour + * neither could change safely. Registration only requires implementing StorageAdaptor, and + * KVMStoragePoolManager keys adaptors on getStoragePoolType(), so nothing depends on a shared + * superclass. Asserting it keeps a later "reuse" refactor from quietly restoring by-path + * naming through an un-overridden inherited method. + */ + @Test + public void adaptorDoesNotInheritTheByPathBasedIscsiAdaptor() { + assertTrue("The adaptor must implement StorageAdaptor directly", + StorageAdaptor.class.isAssignableFrom(OntapIscsiStorageAdaptor.class)); + assertFalse("The adaptor must not extend the by-path based iSCSI adaptor", + IscsiAdmStorageAdaptor.class.isAssignableFrom(OntapIscsiStorageAdaptor.class)); + assertEquals("It should sit directly on StorageAdaptor, with no intermediate base class", + Object.class, OntapIscsiStorageAdaptor.class.getSuperclass()); + } + + /** + * The WWID occupies the component the superclass reads a logical unit number from, which is what + * keeps the path at the two components every hypervisor resource's '/targetIQN/LUN' parsing + * demands. ONTAP serial numbers contain '/' (byte 0x2f), so carrying the raw serial here instead + * of its hex WWID would split into four components and throw. + */ + @Test + public void physicalDiskIsNamedByLunWwidRatherThanLogicalUnitNumber() { + OntapIscsiStorageAdaptor adaptor = new OntapIscsiStorageAdaptor(); + KVMStoragePool pool = adaptor.createStoragePool("ontap-iscsi-pool-uuid", "10.196.37.157", 3260, null, null, + StoragePoolType.OntapiSCSI, null, true); + + String targetIqn = "iqn.1992-08.com.netapp:sn.45048fd9b65111f1b106005056bd83cf:vs.3"; + String lunWwid = "600a098078304d2d383f2f6b45734a54"; + + KVMPhysicalDisk disk = adaptor.getPhysicalDisk("/" + targetIqn + "/" + lunWwid, pool); + + assertEquals("/dev/disk/by-id/wwn-0x" + lunWwid, disk.getPath()); + assertFalse("The reported path must carry nothing host-specific", disk.getPath().contains("lun-")); + assertFalse("The reported path must not be derived from by-path", disk.getPath().contains("by-path")); + assertEquals(PhysicalDiskFormat.RAW, disk.getFormat()); + } + + @Test + public void malformedVolumePathIsRejected() { + OntapIscsiStorageAdaptor adaptor = new OntapIscsiStorageAdaptor(); + KVMStoragePool pool = adaptor.createStoragePool("ontap-iscsi-pool-uuid", "10.196.37.157", 3260, null, null, + StoragePoolType.OntapiSCSI, null, true); + + // A raw ONTAP serial such as 'x0M-8?/kEsJT' would land here, splitting into four components. + assertThrows(CloudRuntimeException.class, + () -> adaptor.getPhysicalDisk("/iqn.1992-08.com.netapp:sn.abc:vs.3/x0M-8?/kEsJT", pool)); + assertThrows(CloudRuntimeException.class, + () -> adaptor.getPhysicalDisk("/iqn.1992-08.com.netapp:sn.abc:vs.3", pool)); + } + + /** + * KVMStoragePoolManager.disconnectPhysicalDiskByPath walks every registered adaptor and stops at + * the first one returning true, so an adaptor that over-claims tears down another vendor's + * session. By-path devices still belong to the superclass, which serves the other iSCSI vendors. + */ + @Test + public void disconnectByPathOnlyClaimsTheDevicesThisAdaptorHandsOut() { + OntapIscsiStorageAdaptor adaptor = new OntapIscsiStorageAdaptor(); + + assertFalse("A by-path device is the superclass's to disconnect", adaptor.disconnectPhysicalDiskByPath( + "/dev/disk/by-path/ip-10.0.0.1:3260-iscsi-iqn.1992-08.com.netapp:sn.abc:vs.3-lun-0")); + assertFalse("PowerFlex publishes its own by-id names", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/emc-vol-1235dc1s0-4a2f6b45")); + assertFalse("A partition is not the LUN itself", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/wwn-0x600a098078304d2d383f2f6b45734a51-part1")); + assertFalse("Only the wwn-0x form is handed out, never the scsi-3 alias", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/scsi-3600a098078304d2d383f2f6b45734a51")); + assertFalse("A bare kernel name carries no identity", adaptor.disconnectPhysicalDiskByPath("/dev/sdb")); + assertFalse(adaptor.disconnectPhysicalDiskByPath(null)); + } + + /** + * Other vendors publish the same wwn-0x plus 32 hex shape. Only NetApp's NAA prefix is ours, so a + * different array's LUN or a local RAID volume must fall through to the next adaptor. + */ + @Test + public void disconnectByPathDoesNotClaimOtherVendorsWwnDevices() { + OntapIscsiStorageAdaptor adaptor = new OntapIscsiStorageAdaptor(); + + assertFalse("Pure FlashArray also publishes wwn-0x names", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/wwn-0x624a9370f1f4f5a4b0d3c2e1000113aa")); + assertFalse("HPE Primera also publishes wwn-0x names", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/wwn-0x60002ac0000000000000012300020a1b")); + assertFalse("A local RAID volume also publishes wwn-0x names", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/wwn-0x600508b1001c4d2e8a0f6b1c2d3e4f50")); + assertFalse("NetApp E-Series is not managed by this adaptor", + adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/wwn-0x600a0b800029e3a4000012345f6a7b8c")); + } + + /** + * A claimed device that has already gone reports success: there is no session left to tear down, + * and returning false would send the manager on to adaptors that would mishandle the path. + */ + @Test + public void disconnectByPathSucceedsWhenTheClaimedDeviceIsAlreadyGone() { + OntapIscsiStorageAdaptor adaptor = new OntapIscsiStorageAdaptor(); + + assertTrue(adaptor.disconnectPhysicalDiskByPath("/dev/disk/by-id/wwn-0x600a098078304d2d383f2f6b45734a51")); + } +} diff --git a/plugins/storage/volume/ontap/README.md b/plugins/storage/volume/ontap/README.md index 059bf99e09d8..7521ae890e2c 100644 --- a/plugins/storage/volume/ontap/README.md +++ b/plugins/storage/volume/ontap/README.md @@ -65,7 +65,7 @@ The NetApp ONTAP Storage Plugin provides integration between Apache CloudStack a ### Minimum Volume Size -ONTAP requires a minimum volume size of **1.56 GB** (1,677,721,600 bytes). The plugin will automatically adjust requested sizes below this threshold. +ONTAP requires a minimum volume size of **20 MB** (20,971,520 bytes). Requests below this threshold are rejected. ## Configuration @@ -116,7 +116,7 @@ username=admin;password=secretpass;svmName=svm1;protocol=ISCSI;managementLIF=192 3. **Capacity Errors** - Check aggregate space availability - - Ensure requested volume size meets minimum requirements (1.56 GB) + - Ensure requested volume size meets minimum requirements (20 MB) 4. **Host Connection Issues** - For iSCSI: Verify host IQN is properly configured in host's storage URL diff --git a/plugins/storage/volume/ontap/pom.xml b/plugins/storage/volume/ontap/pom.xml index 0ab54c67d343..c18bf559f92a 100644 --- a/plugins/storage/volume/ontap/pom.xml +++ b/plugins/storage/volume/ontap/pom.xml @@ -86,6 +86,11 @@ cloud-engine-storage-volume ${project.version} + + org.apache.cloudstack + cloud-framework-cluster + ${project.version} + io.swagger swagger-annotations diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/asup/OntapAsupManager.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/asup/OntapAsupManager.java new file mode 100644 index 000000000000..5ab2aa18833e --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/asup/OntapAsupManager.java @@ -0,0 +1,673 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.asup; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.event.EventTypes; +import com.cloud.server.ManagementService; +import com.cloud.storage.Volume; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.net.NetUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; +import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.utils.OntapConfigurationManager; +import org.apache.cloudstack.storage.utils.OntapStorageConstants; +import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.time.Duration; +import java.time.Instant; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Periodic ASUP (AutoSupport) telemetry pusher for the NetApp ONTAP plugin. + * + *

Each run pushes, for every ONTAP-backed primary storage pool, two minimal EMS + * application-log messages to the backing ONTAP cluster:

+ * + *
    + *
  • event-id 0 (heartbeat): identifies the CloudStack deployment (CloudStack + * version, management host) connected to the ONTAP cluster (ONTAP cluster version).
    • + *
  • event-id 1 (pool): maps the CloudStack storage pool to its backing ONTAP + * volume - protocol (NFS/iSCSI), ONTAP FlexVolume UUID, SVM, disk usage, and + * snapshot telemetry (counts by state, total provisioned size).
    • + *
+ * + *

Runs are booked one at a time: after each cycle starts, the next run is scheduled for + * {@link OntapConfigurationManager#AsupIntervalHours} after that start, so production + * intervals (hours) are start-to-start. REST work sits inside the interval; it is not added + * after it. Editing the ONTAP ASUP interval re-books the pending run immediately, with no + * management-server restart.

+ */ +public class OntapAsupManager extends ManagerBase { + private static final int ASUP_LOCK_TIMEOUT_SECONDS = 5; + + /** + * Volume states that guarantee a physical object exists on the ONTAP FlexVolume. + * States like {@link Volume.State#Allocated} have a CloudStack DB row pointing to this + * pool but ONTAP provisioning has not been called yet — they must be excluded to avoid + * inflating disk counts and provisioned-size totals. Upload-family states live on + * secondary storage, not on the primary ONTAP volume, so they are also excluded. + */ + private static final Set CS_VOLUME_STATES = EnumSet.of( + Volume.State.Ready, + Volume.State.Snapshotting, + Volume.State.RevertSnapshotting, + Volume.State.Attaching, + Volume.State.Restoring, + Volume.State.Expunging, + Volume.State.Destroying + ); + + /** Serializes the structured event-description payloads to JSON. */ + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Timestamp of the last ASUP cycle start (or {@link Instant#EPOCH} if none has run). + * The next run is due {@link OntapConfigurationManager#AsupIntervalHours} after this + * instant. {@code volatile} so the scheduler thread's write is visible without extra locking. + */ + volatile Instant lastPushTime = Instant.EPOCH; + + /** Single daemon thread that runs the ASUP pushes; owned by this manager. */ + private ScheduledExecutorService asupScheduler; + + /** The one pending run. Cancelled and re-booked whenever the schedule changes. */ + private ScheduledFuture pendingRun; + + @Inject + private PrimaryDataStoreDao storagePoolDao; + @Inject + private StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + private VolumeDao volumeDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMSnapshotDao vmSnapshotDao; + @Inject + private ManagementService managementService; + @Inject + private ManagementServerHostDao managementServerHostDao; + @Inject + private MessageBus messageBus; + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + // React to edits of ontap.autosupport.interval so a new value re-books the pending run + // instead of waiting for it to fire on the old schedule. + messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, this::onAsupConfigEdited); + return true; + } + + @Override + public boolean start() { + asupScheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("OntapAsup")); + logger.info("OntapAsupManager started; ASUP telemetry interval={}h", + getAsupIntervalHours(OntapConfigurationManager.AsupIntervalHours.value())); + scheduleNextRun(); + return super.start(); + } + + @Override + public boolean stop() { + if (asupScheduler != null) { + asupScheduler.shutdownNow(); + } + return super.stop(); + } + + /** + * Books the single pending run for the moment the configured interval elapses, cancelling + * whatever was booked before. Called at start-up, after every run, and whenever + * {@code ontap.autosupport.interval} is edited, so the live config always decides the next run. + * + *

When the interval is {@link OntapStorageConstants#ASUP_DISABLED_INTERVAL_HOURS} + * nothing is booked; setting a non-zero interval publishes a configuration-edit event, + * which books a run again.

+ */ + private synchronized void scheduleNextRun() { + if (asupScheduler == null || asupScheduler.isShutdown()) { + return; + } + if (pendingRun != null) { + pendingRun.cancel(false); + pendingRun = null; + } + int intervalHours = getAsupIntervalHours(OntapConfigurationManager.AsupIntervalHours.value()); + if (intervalHours == OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS) { + logger.debug("ONTAP ASUP: telemetry is disabled ({}={}); no run scheduled.", + OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY, + OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS); + return; + } + long delayMs = millisUntilNextPush(); + pendingRun = asupScheduler.schedule(new OntapAsupTask(), delayMs, TimeUnit.MILLISECONDS); + logger.debug("ONTAP ASUP: next telemetry push scheduled in {} ms.", delayMs); + } + + /** + * Milliseconds remaining until {@link #lastPushTime} plus the live configured interval. + * Zero when a push is already overdue — for example after the interval is shortened, the + * previous cycle ran longer than the interval, or on the very first run when + * {@link #lastPushTime} is still {@link Instant#EPOCH}. + */ + long millisUntilNextPush() { + Duration configuredInterval = Duration.ofHours( + getAsupIntervalHours(OntapConfigurationManager.AsupIntervalHours.value())); + Duration remaining = configuredInterval.minus(Duration.between(lastPushTime, Instant.now())); + return remaining.isNegative() ? 0L : remaining.toMillis(); + } + + /** + * CloudStack publishes this after invalidating the config cache, so the new value is already + * readable. Re-books the pending run: shortening the interval past the due time schedules + * with zero delay, which pushes right away on the scheduler thread rather than blocking the + * API thread that served {@code updateConfiguration}. + */ + @SuppressWarnings("unchecked") + private void onAsupConfigEdited(String senderAddress, String subject, Object args) { + if (!(args instanceof Ternary)) { + return; + } + String updatedKey = ((Ternary) args).first(); + if (!OntapConfigurationManager.AsupIntervalHours.key().equals(updatedKey)) { + return; + } + logger.debug("ONTAP ASUP: [{}] was updated; re-booking the next push.", updatedKey); + scheduleNextRun(); + } + + /** + * One ASUP run inside a managed CloudStack context, which the DAO calls made during the push + * require. + * + *

{@link #lastPushTime} is stamped at the start of the cycle so the next wait is the + * remainder of the interval (work does not get added on top). The next run is booked again + * in {@code finally} so the schedule never stalls.

+ */ + protected class OntapAsupTask extends ManagedContextRunnable { + @Override + protected void runInContext() { + try { + lastPushTime = Instant.now(); + pushAsupTelemetry(); + } catch (Exception e) { + logger.warn("ONTAP ASUP: unexpected error during periodic push: {}", e.getMessage()); + } finally { + scheduleNextRun(); + } + } + } + + /** + * Iterates all ONTAP-backed primary storage pools and pushes ASUP telemetry for each. + * + *

Guarded by a {@link GlobalLock} so that, in a multi-management-server deployment, + * only one node emits per cycle.

+ */ + protected void pushAsupTelemetry() { + List pools = storagePoolDao.findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME); + if (CollectionUtils.isEmpty(pools)) { + logger.debug("ONTAP ASUP: no ONTAP-backed storage pools found; nothing to push."); + return; + } + + GlobalLock lock = GlobalLock.getInternLock(OntapStorageConstants.ASUP_GLOBAL_LOCK_NAME); + try { + if (!lock.lock(ASUP_LOCK_TIMEOUT_SECONDS)) { + logger.debug("ONTAP ASUP: another management server holds the ASUP lock; skipping this cycle."); + return; + } + try { + logger.debug("ONTAP ASUP: pushing telemetry for {} pool(s) [CloudStack version={}]", + pools.size(), getCloudStackVersion()); + Map clientsByStorageIp = new HashMap<>(); + for (StoragePoolVO pool : pools) { + pushAsupForStoragePool(pool, clientsByStorageIp); + } + } finally { + lock.unlock(); + } + } finally { + lock.releaseRef(); + } + } + + /** + * Pushes the heartbeat (event-id 0) and pool (event-id 1) ASUP messages for a single pool. + * + *

{@code clientsByStorageIp} is the per-cycle cache keyed by management {@code storageIP}. + * A miss builds the strategy, GETs cluster info and node model, and sends event-0. A hit + * reuses that client and skips those calls and event-0. Event-1 is always sent. SVM name + * in the payload still comes from this pool's details.

+ * + *

Best-effort: any failure is logged and swallowed.

+ */ + protected void pushAsupForStoragePool(StoragePoolVO pool, Map clientsByStorageIp) { + try { + Map details = storagePoolDetailsDao.listDetailsKeyPairs(pool.getId()); + if (details == null || details.isEmpty()) { + logger.warn("ONTAP ASUP: storage pool [{}] has no details; skipping.", pool.getId()); + return; + } + + String storageIp = details.get(OntapStorageConstants.STORAGE_IP); + AsupClusterClient asupClusterClient = StringUtils.isNotBlank(storageIp) + ? clientsByStorageIp.get(storageIp) : null; + boolean sendHeartbeat = asupClusterClient == null; + if (asupClusterClient == null) { + StorageStrategy strategy = OntapStorageUtils.resolveStrategyFromPoolDetails(details); + Cluster cluster = strategy.getClusterInfo(); + asupClusterClient = new AsupClusterClient(strategy, cluster); + if (StringUtils.isNotBlank(storageIp)) { + clientsByStorageIp.put(storageIp, asupClusterClient); + } + } + + StorageStrategy strategy = asupClusterClient.strategy; + Cluster cluster = asupClusterClient.cluster; + String ontapVersion = strategy.getClusterVersion(cluster); + String clusterUuid = cluster != null ? cluster.getUuid() : null; + String cloudStackVersion = getCloudStackVersion(); + String computerName = getComputerName(); + + if (sendHeartbeat) { + EmsApplicationLog heartbeat = buildBaseMessage(computerName, cloudStackVersion); + heartbeat.setEventId(OntapStorageConstants.ASUP_EVENT_ID_HEARTBEAT); + heartbeat.setEventDescription(buildHeartbeatDescription( + cloudStackVersion, ontapVersion, cluster)); + strategy.sendAsupMessage(heartbeat); + } else { + logger.debug("ONTAP ASUP: heartbeat already sent this cycle for storage IP [{}]; skipping for pool [{}]", + storageIp, pool.getId()); + } + + // event-id 1: CloudStack storage pool -> backing ONTAP volume mapping, once per pool. + // The description also includes disk usage and snapshot telemetry + EmsApplicationLog poolMessage = buildBaseMessage(computerName, cloudStackVersion); + poolMessage.setEventId(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL); + poolMessage.setEventDescription(buildPoolDescription(pool, details, clusterUuid)); + strategy.sendAsupMessage(poolMessage); + + logger.debug("ONTAP ASUP: pushed telemetry for pool [{}] (ONTAP version={})", + pool.getId(), defaultUnknown(ontapVersion)); + } catch (Exception e) { + // Best-effort telemetry; never propagate. + logger.warn("ONTAP ASUP: failed to push telemetry for pool [{}]: {}", pool.getId(), e.getMessage()); + } + } + + /** + * Builds the heartbeat (event-id 0) description as a JSON object carrying the CloudStack and + * ONTAP versions, cluster hardware model, the management-server operating system platform, + * and the ONTAP cluster UUID. + * Example: {@code {"message":"CloudStack connected to ONTAP cluster","cloudstackVersion": + * "4.23.0.0","platform":"Linux 5.15.0-91-generic (amd64)","ontapVersion":"9.17.1", + * "ontapClusterModel":"AFF-A400","ontapPlatformType":"performance","clusterUuid":"...", + * "managementServerCount":2}} + */ + private String buildHeartbeatDescription(String cloudStackVersion, String ontapVersion, Cluster cluster) { + Map payload = new LinkedHashMap<>(); + payload.put(OntapStorageConstants.ASUP_MESSAGE, OntapStorageConstants.ASUP_HEARTBEAT_MESSAGE); + payload.put(OntapStorageConstants.ASUP_CLOUDSTACK_VERSION, defaultUnknown(cloudStackVersion)); + payload.put(OntapStorageConstants.ASUP_PLATFORM, getOperatingSystem()); + payload.put(OntapStorageConstants.ASUP_ONTAP_VERSION, defaultUnknown(ontapVersion)); + payload.put(OntapStorageConstants.ASUP_ONTAP_CLUSTER_MODEL, + defaultUnknown(cluster != null ? cluster.getModel() : null)); + payload.put(OntapStorageConstants.ASUP_ONTAP_PLATFORM_TYPE, + defaultUnknown(cluster != null ? cluster.getPlatformType() : null)); + payload.put(OntapStorageConstants.ASUP_CLUSTER_UUID, + defaultUnknown(cluster != null ? cluster.getUuid() : null)); + payload.put(OntapStorageConstants.ASUP_MANAGEMENT_SERVER_COUNT, getManagementServerCount()); + return toJson(payload); + } + + /** + * Builds the pool description (event-id 1) as a JSON object combining the backing-volume + * mapping, disk usage, and snapshot telemetry into a single EMS message. + * + *

Example: {@code {"message":"CloudStack storage pool backed by ONTAP volume", + * "poolName":"...","poolStatus":"Up","protocol":"nfs","clusterUuid":"...","svm":"...", + * "ontapVolumeUuid":"...","rootDiskCount":12,"dataDiskCount":18, + * "totalLogicalSizeBytes":322122547200,"multiPrimaryStoragePoolVm":false, + * "volumeSnapshotCount":5,"vmSnapshotCount":3}}

+ */ + private String buildPoolDescription(StoragePoolVO pool, Map details, + String clusterUuid) { + Map payload = new LinkedHashMap<>(); + payload.put(OntapStorageConstants.ASUP_MESSAGE, OntapStorageConstants.ASUP_POOL_MESSAGE); + payload.put(OntapStorageConstants.ASUP_POOL_NAME, defaultUnknown(pool.getName())); + payload.put(OntapStorageConstants.ASUP_POOL_STATUS, + pool.getStatus() == null ? OntapStorageConstants.ASUP_UNKNOWN : pool.getStatus().toString()); + payload.put(OntapStorageConstants.ASUP_PROTOCOL, defaultUnknown(details.get(OntapStorageConstants.PROTOCOL))); + payload.put(OntapStorageConstants.ASUP_CLUSTER_UUID, defaultUnknown(clusterUuid)); + payload.put(OntapStorageConstants.ASUP_SVM, defaultUnknown(details.get(OntapStorageConstants.SVM_NAME))); + payload.put(OntapStorageConstants.ASUP_ONTAP_VOLUME_UUID, defaultUnknown(details.get(OntapStorageConstants.VOLUME_UUID))); + List volumes = null; + try { + volumes = loadNonDestroyedVolumes(pool); + } catch (Exception e) { + logger.error("ONTAP ASUP: failed to load volumes for pool [{}]: {}", pool.getId(), e.getMessage()); + } + if (volumes != null) { + addStoragePoolUsage(pool, payload, volumes); + addSnapshotMetrics(pool, payload, volumes); + } + hasMultiPrimaryStoragePoolVm(pool, payload); + return toJson(payload); + } + + /** + * Single CloudStack DB read of non-destroyed volumes on this pool. Shared by usage and + * snapshot counts so the list is not queried three times. Never returns null. + */ + private List loadNonDestroyedVolumes(StoragePoolVO pool) { + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); + return volumes == null ? java.util.Collections.emptyList() : volumes; + } + + /** + * Computes pool usage from CloudStack's volume records and adds it to the payload: + *
    + *
  • {@code rootDiskCount} - number of ROOT (boot) disks physically on this pool
    • + *
  • {@code dataDiskCount} - number of DATADISK disks physically on this pool
    • + *
  • {@code totalLogicalSizeBytes} - sum of those volumes' provisioned (logical) sizes + * in bytes; for thin-provisioned volumes this is the logical size requested at + * creation time, not the physical space consumed on ONTAP
    • + *
+ * {@code volumes} is the non-destroyed list for this pool (already loaded). Only + * {@link #CS_VOLUME_STATES} are counted. Best-effort: any failure leaves the usage + * fields out and never breaks telemetry. + */ + private void addStoragePoolUsage(StoragePoolVO pool, Map payload, List volumes) { + try { + // Only count volumes that definitely have a physical object on the ONTAP FlexVolume. + // "Allocated" volumes have a pool_id row in the CS DB but ONTAP provisioning has not + // yet been called, so including them would inflate counts and provisioned size. + long rootDiskCount = 0L; + long dataDiskCount = 0L; + long totalLogicalSizeBytes = 0L; + for (VolumeVO volume : volumes) { + if (!CS_VOLUME_STATES.contains(volume.getState())) { + continue; + } + if (Volume.Type.ROOT.equals(volume.getVolumeType())) { + rootDiskCount++; + } else if (Volume.Type.DATADISK.equals(volume.getVolumeType())) { + dataDiskCount++; + } + totalLogicalSizeBytes += volume.getSize() != null ? volume.getSize() : 0L; + } + payload.put(OntapStorageConstants.ASUP_ROOT_DISK_COUNT, rootDiskCount); + payload.put(OntapStorageConstants.ASUP_DATA_DISK_COUNT, dataDiskCount); + payload.put(OntapStorageConstants.ASUP_TOTAL_LOGICAL_SIZE_BYTES, totalLogicalSizeBytes); + } catch (Exception e) { + logger.error("ONTAP ASUP: failed to compute usage for pool [{}]: {}", pool.getId(), e.getMessage()); + } + } + + /** + * Adds {@code hasMultiPrimaryStoragePoolVm}: true when at least one VM with a Ready ROOT + * on this pool also has an attached Ready DATADISK on a different primary storage pool. + * Uses a single {@code LIMIT 1} existence query. + */ + private void hasMultiPrimaryStoragePoolVm(StoragePoolVO pool, Map payload) { + try { + payload.put(OntapStorageConstants.ASUP_MULTI_PRIMARY_STORAGE_POOL_VM, + volumeDao.hasMultiPrimaryStoragePoolVm(pool.getId())); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to compute multiPrimaryStoragePoolVm for pool [{}]: {}", + pool.getId(), e.getMessage()); + } + } + + /** + * Computes and adds two groups of snapshot telemetry to the pool description payload. + * + *

Volume-snapshot metrics ({@code volumeSnapshotCount}): counts all + * non-destroyed CloudStack volume-level snapshots for volumes on this pool.

+ * + *

VM-snapshot metrics ({@code vmSnapshotCount}): counts all active + * (non-expunging, non-removed) VM snapshots for VMs whose ROOT disk is on this pool. + * A data disk on another pool does not duplicate the count.

+ * + *

Best-effort: any failure leaves the fields out without breaking telemetry.

+ */ + private void addSnapshotMetrics(StoragePoolVO pool, Map payload, List volumes) { + addVmSnapshotMetrics(pool, payload, volumes); + addVolumeSnapshotMetrics(pool, payload, volumes); + } + + /** + * Adds {@code volumeSnapshotCount} to the payload. + * Counts all non-destroyed CloudStack volume-level snapshots for volumes on this pool. + */ + private void addVolumeSnapshotMetrics(StoragePoolVO pool, Map payload, List volumes) { + try { + if (volumes.isEmpty()) { + payload.put(OntapStorageConstants.ASUP_VOLUME_SNAPSHOT_COUNT, 0); + return; + } + + List volumeIds = volumes.stream() + .map(VolumeVO::getId) + .collect(java.util.stream.Collectors.toList()); + + List snapshots = snapshotDao.searchByVolumes(volumeIds); + long snapCount = snapshots == null ? 0L : snapshots.stream() + .filter(snap -> !com.cloud.storage.Snapshot.State.Destroyed.equals(snap.getState())) + .count(); + + payload.put(OntapStorageConstants.ASUP_VOLUME_SNAPSHOT_COUNT, snapCount); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to compute volume-snapshot metrics for pool [{}]: {}", + pool.getId(), e.getMessage()); + } + } + + /** + * Adds {@code vmSnapshotCount} to the payload. + * Counts all active (non-expunging, non-removed) VM snapshots for VMs whose ROOT + * disk is on this pool, so a multi-pool VM is counted once (on the root pool). + */ + private void addVmSnapshotMetrics(StoragePoolVO pool, Map payload, List volumes) { + try { + if (volumes.isEmpty()) { + payload.put(OntapStorageConstants.ASUP_VM_SNAPSHOT_COUNT, 0); + return; + } + + java.util.Set vmIds = volumes.stream() + .filter(v -> Volume.Type.ROOT.equals(v.getVolumeType())) + .map(VolumeVO::getInstanceId) + .filter(java.util.Objects::nonNull) + .collect(java.util.stream.Collectors.toSet()); + + if (vmIds.isEmpty()) { + payload.put(OntapStorageConstants.ASUP_VM_SNAPSHOT_COUNT, 0); + return; + } + + List vmSnapshots = vmSnapshotDao.searchByVms(new java.util.ArrayList<>(vmIds)); + long vmSnapCount = vmSnapshots == null ? 0L : vmSnapshots.stream() + .filter(vmSnap -> !VMSnapshot.State.Expunging.equals(vmSnap.getState()) + && vmSnap.getRemoved() == null) + .count(); + + payload.put(OntapStorageConstants.ASUP_VM_SNAPSHOT_COUNT, vmSnapCount); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to compute VM-snapshot metrics for pool [{}]: {}", + pool.getId(), e.getMessage()); + } + } + + /** + * Serializes a payload map to a JSON string. Falls back to the map's {@code toString()} if + * serialization unexpectedly fails, so telemetry is still emitted (best-effort). + */ + private String toJson(Map payload) { + try { + return objectMapper.writeValueAsString(payload); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to serialize event description to JSON: {}", e.getMessage()); + return String.valueOf(payload); + } + } + + /** Builds the common EMS message envelope shared by all ASUP messages. */ + private EmsApplicationLog buildBaseMessage(String computerName, String appVersion) { + EmsApplicationLog message = new EmsApplicationLog(); + message.setComputerName(computerName); + message.setEventSource(OntapStorageConstants.ASUP_EVENT_SOURCE); + message.setAppVersion(appVersion); + message.setCategory(OntapStorageConstants.ASUP_CATEGORY); + message.setSeverity(OntapStorageConstants.ASUP_SEVERITY); + message.setAutosupportRequired(Boolean.FALSE); + return message; + } + + /** + * CloudStack version of this management server, same source as {@link ManagementService#getVersion()} + * / {@code listCapabilities}. Falls back to "unknown" when the server JAR has no manifest + * (for example running from an IDE). + */ + protected String getCloudStackVersion() { + String version = managementService != null ? managementService.getVersion() : null; + return StringUtils.isBlank(version) ? OntapStorageConstants.ASUP_UNKNOWN : version; + } + + /** + * Number of management servers registered in {@code mshost} (not removed), including nodes + * that are not currently {@code Up}. + */ + protected int getManagementServerCount() { + try { + List hosts = managementServerHostDao.listAll(); + return hosts == null ? 0 : hosts.size(); + } catch (Exception e) { + logger.debug("ONTAP ASUP: unable to count management servers: {}", e.getMessage()); + return 0; + } + } + + /** Resolves the management server hostname for the EMS computer-name field. */ + protected String getComputerName() { + String hostName = NetUtils.getCanonicalHostName(); + return StringUtils.isBlank(hostName) ? OntapStorageConstants.ASUP_UNKNOWN : hostName; + } + + /** + * Resolves the management server operating system (name, version and architecture) from JVM + * system properties, e.g. {@code "Linux 5.15.0-91-generic (amd64)"}. Falls back to "unknown". + */ + protected String getOperatingSystem() { + String osName = System.getProperty("os.name"); + String osVersion = System.getProperty("os.version"); + String osArch = System.getProperty("os.arch"); + if (StringUtils.isBlank(osName)) { + return OntapStorageConstants.ASUP_UNKNOWN; + } + StringBuilder sb = new StringBuilder(osName); + if (StringUtils.isNotBlank(osVersion)) { + sb.append(' ').append(osVersion); + } + if (StringUtils.isNotBlank(osArch)) { + sb.append(" (").append(osArch).append(')'); + } + return sb.toString(); + } + + private String defaultUnknown(String value) { + return StringUtils.isBlank(value) ? OntapStorageConstants.ASUP_UNKNOWN : value; + } + + /** + * Returns a usable interval in hours. {@link OntapStorageConstants#ASUP_DISABLED_INTERVAL_HOURS} + * means telemetry is off. Out-of-range or missing DB values (for example set outside the API) + * fall back to the default so ASUP is not sent on an unintended cadence. The scheduler + * converts a non-zero value to seconds via {@link Duration#ofHours(long)}. + */ + int getAsupIntervalHours(Integer configured) { + if (configured == null) { + return OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS; + } + if (configured == OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS) { + return OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS; + } + if (configured < OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS + || configured > OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS) { + logger.warn("ONTAP ASUP: {} value [{}] is outside [{}-{}]; using default [{}]", + OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY, configured, + OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS, + OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS, + OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS); + return OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS; + } + return configured; + } + + /** + * One ONTAP cluster HTTP client plus the cluster GET and node-model GET for this ASUP cycle, + * keyed by {@code storageIP}. Discarded when the cycle ends. + */ + static final class AsupClusterClient { + final StorageStrategy strategy; + final Cluster cluster; + + AsupClusterClient(StorageStrategy strategy, Cluster cluster) { + this.strategy = strategy; + this.cluster = cluster; + } + } + +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriver.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriver.java index ece29f7cd0ac..2bc06c12ca6d 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriver.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriver.java @@ -23,17 +23,25 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ResizeVolumePayload; import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; +import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.utils.Pair; @@ -44,23 +52,31 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.feign.client.SnapshotFeignClient; +import org.apache.cloudstack.storage.feign.model.FileInfo; import org.apache.cloudstack.storage.feign.model.FlexVolSnapshot; import org.apache.cloudstack.storage.feign.model.Lun; +import org.apache.cloudstack.storage.feign.model.LunSpace; +import org.apache.cloudstack.storage.feign.model.Svm; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import org.apache.cloudstack.storage.service.SANStrategy; import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.service.UnifiedNASStrategy; import org.apache.cloudstack.storage.service.UnifiedSANStrategy; import org.apache.cloudstack.storage.service.model.AccessGroup; import org.apache.cloudstack.storage.service.model.CloudStackVolume; @@ -74,9 +90,13 @@ import javax.inject.Inject; import java.util.ArrayList; +import java.util.Arrays; +import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Primary datastore driver for NetApp ONTAP storage systems. @@ -91,6 +111,8 @@ public class OntapPrimaryDatastoreDriver implements PrimaryDataStoreDriver { @Inject private VolumeDao volumeDao; @Inject private VolumeDetailsDao volumeDetailsDao; @Inject private SnapshotDetailsDao snapshotDetailsDao; + @Inject private SnapshotDao snapshotDao; + @Inject private VMTemplatePoolDao vmTemplatePoolDao; @Override public Map getCapabilities() { @@ -98,6 +120,15 @@ public Map getCapabilities() { Map mapCapabilities = new HashMap<>(); mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.TRUE.toString()); mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString()); + mapCapabilities.put(DataStoreCapabilities.CAN_REVERT_VOLUME_TO_SNAPSHOT.toString(), Boolean.TRUE.toString()); + // Enables the framework to cache a template on the FlexVolume once and serve every later + // deployment with an array-side clone instead of another copy from secondary storage. + mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString(), Boolean.TRUE.toString()); + // createTemplate(snapshotid) reads the snapshot from this pool instead of backing it up to secondary first. + mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString()); + // Must be present and false: StorageSystemDataMotionStrategy then clones the snapshot into a + // temporary volume (see copyAsync) and copies that volume to secondary storage. + mapCapabilities.put(OntapStorageConstants.CAN_DIRECT_ATTACH_SNAPSHOT, Boolean.FALSE.toString()); return mapCapabilities; } @@ -115,6 +146,18 @@ public boolean volumesRequireGrantAccessWhenUsed() { return true; } + @Override + public boolean zoneWideVolumesAvailableWithoutClusterMotion(){ + logger.trace("zoneWideVolumesAvailableWithoutClusterMotion invoked"); + return true; + } + + @Override + public boolean requiresAccessForMigration(DataObject dataObject) { + logger.trace("requiresAccessForMigration invoked"); + return true; + } + /** * Creates a volume on the ONTAP storage system. */ @@ -144,6 +187,7 @@ public void createAsync(DataStore dataStore, DataObject dataObject, AsyncComplet } Map details = storagePoolDetailsDao.listDetailsKeyPairs(dataStore.getId()); + validateProtocol(details, dataStore); if (dataObject.getType() == DataObjectType.VOLUME) { VolumeInfo volInfo = (VolumeInfo) dataObject; @@ -151,35 +195,44 @@ public void createAsync(DataStore dataStore, DataObject dataObject, AsyncComplet // Update CloudStack volume record with storage pool association and protocol-specific details VolumeVO volumeVO = volumeDao.findById(volInfo.getId()); if (volumeVO != null) { - // Create the backend storage object (LUN for iSCSI, no-op for NFS) - CloudStackVolume created = createCloudStackVolume(storagePool, volInfo, details); - - volumeVO.setPoolType(storagePool.getPoolType()); - volumeVO.setPoolId(storagePool.getId()); - - if (ProtocolType.ISCSI.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { - String lunName = created != null && created.getLun() != null ? created.getLun().getName() : null; - if (lunName == null) { - throw new CloudRuntimeException("Missing LUN name for volume " + volInfo.getId()); - } - - // Persist LUN details for future operations (delete, grant/revoke access) - volumeDetailsDao.addDetail(volInfo.getId(), OntapStorageConstants.LUN_DOT_UUID, created.getLun().getUuid(), false); - volumeDetailsDao.addDetail(volInfo.getId(), OntapStorageConstants.LUN_DOT_NAME, lunName, false); - if (created.getLun().getUuid() != null) { - volumeVO.setFolder(created.getLun().getUuid()); - } - - logger.info("createAsync: Created LUN [{}] for volume [{}]. LUN mapping will occur during grantAccess() to per-host igroup.", - lunName, volumeVO.getId()); - createCmdResult = new CreateCmdResult(lunName, new Answer(null, true, null)); - } else if (ProtocolType.NFS3.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { - createCmdResult = new CreateCmdResult(volInfo.getUuid(), new Answer(null, true, null)); - logger.info("createAsync: Managed NFS volume [{}] with path [{}] associated with pool {}", - volumeVO.getId(), volInfo.getUuid(), storagePool.getId()); + /* + * Create-volume combinations on ONTAP primary (v1): + * + * 1) cloneOfSnapshot — StorageSystemDataMotionStrategy sets volume_details.cloneOfSnapshot + * when createVolume(snapshotid) targets a managed backend snapshot. + * - Same primary pool / FlexVol only (PRIMARY_POOL_ID must match dataStore). + * - DATA or ROOT snapshot → new attachable data volume (ROOT is never bootable + * via this path; bootable recovery is createTemplate(snapshotid) → deploy). + * - Backend: iSCSI → POST /api/storage/luns (clone.source in .snapshot/); + * NFS → POST /api/storage/file/clone with source_path + * ./.snapshot//. + * - IOPS: the new volume's min/max (snapshot values, or the root offering when one + * is supplied). Applied here; the temporary template clone does not get a policy. + * + * 2) cloneOfTemplate — deploy / create from cached template on this pool. + * + * 3) else — blank LUN (iSCSI) or qcow2 (NFS). + * + * Mutually exclusive from the motion/orchestrator layer; snapshot is checked first + * (SolidFire-style) so a create-from-snapshot never accidentally falls through to blank create. + */ + Long cloneOfSnapshotId = getSnapshotIdForCloning(volInfo.getId()); + Long cloneOfTemplateId = getTemplateIdForCloning(volInfo.getId()); + CloudStackVolume clonedCloudStackVolume; + if (cloneOfSnapshotId != null) { + clonedCloudStackVolume = cloneCloudStackVolumeFromSnapshot( + storagePool, volInfo, details, cloneOfSnapshotId, true); + } else if (cloneOfTemplateId != null) { + clonedCloudStackVolume = cloneCloudStackVolumeFromTemplate( + storagePool, volInfo, details, cloneOfTemplateId); + } else { + clonedCloudStackVolume = createCloudStackVolume(storagePool, volInfo, details); } - volumeDao.update(volumeVO.getId(), volumeVO); + + createCmdResult = recordCreatedVolume(storagePool, volInfo, volumeVO, details, clonedCloudStackVolume); } + } else if (dataObject.getType() == DataObjectType.TEMPLATE) { + createCmdResult = createTemplateOnPrimary(storagePool, (TemplateInfo) dataObject, details); } else { errMsg = "Invalid DataObjectType (" + dataObject.getType() + ") passed to createAsync"; logger.error(errMsg); @@ -189,7 +242,7 @@ public void createAsync(DataStore dataStore, DataObject dataObject, AsyncComplet errMsg = e.getMessage(); logger.error("createAsync: Failed for dataObject name [{}]: {}", dataObject.getName(), errMsg); createCmdResult = new CreateCmdResult(null, new Answer(null, false, errMsg)); - createCmdResult.setResult(e.toString()); + createCmdResult.setResult(errMsg); } finally { if (createCmdResult != null && createCmdResult.isSuccess()) { logger.info("createAsync: Operation completed successfully for {}", dataObject.getType()); @@ -198,20 +251,548 @@ public void createAsync(DataStore dataStore, DataObject dataObject, AsyncComplet } } + /** + * Records pool association, image format and protocol-specific identity (LUN name/uuid for iSCSI) + * of a volume that was just created or cloned on ONTAP, and returns the create result whose path + * the framework stores on the volume. + */ + private CreateCmdResult recordCreatedVolume(StoragePoolVO storagePool, VolumeInfo volInfo, VolumeVO volumeVO, + Map details, CloudStackVolume createdCloudStackVolume) { + CreateCmdResult createCmdResult = null; + volumeVO.setPoolType(storagePool.getPoolType()); + volumeVO.setPoolId(storagePool.getId()); + volumeVO.setFormat(getImageFormat(storagePool)); + logger.info("createAsync: Volume format set to [{}] for pool type [{}]", volumeVO.getFormat(), storagePool.getPoolType()); + + if (ProtocolType.ISCSI.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { + // createCloudStackVolume validates the Feign response (LUN name + uuid) before returning + Lun createdLun = createdCloudStackVolume.getLun(); + String lunName = createdLun.getName(); + + // Persist LUN details for future operations (delete, grant/revoke access) + volumeDetailsDao.addDetail(volInfo.getId(), OntapStorageConstants.LUN_DOT_UUID, createdLun.getUuid(), false); + volumeDetailsDao.addDetail(volInfo.getId(), OntapStorageConstants.LUN_DOT_NAME, lunName, false); + volumeVO.setFolder(createdLun.getUuid()); + + logger.info("createAsync: Created LUN [{}] for volume [{}]. LUN mapping will occur during grantAccess() to per-host igroup.", + lunName, volumeVO.getId()); + createCmdResult = new CreateCmdResult(lunName, new Answer(null, true, null)); + } else if (ProtocolType.NFS3.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { + createCmdResult = new CreateCmdResult(volInfo.getUuid(), new Answer(null, true, null)); + logger.info("createAsync: Managed NFS volume [{}] with path [{}] associated with pool {}", + volumeVO.getId(), volInfo.getUuid(), storagePool.getId()); + } + volumeDao.update(volumeVO.getId(), volumeVO); + return createCmdResult; + } + /** * Creates a volume on the ONTAP backend. */ private CloudStackVolume createCloudStackVolume(StoragePoolVO storagePool, VolumeInfo volumeObject, Map details) { + verifySufficientIopsForStoragePool(storagePool, volumeObject.getMinIops(), volumeObject.getId()); + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + VolumeQosPolicy qosPolicy = null; + Volume.Type volumeType = volumeObject.getVolumeType(); + if (volumeType == Volume.Type.DATADISK || volumeType == Volume.Type.ROOT) { + qosPolicy = createQosPolicyIfNeeded(storageStrategy, details, + volumeObject.getMinIops(), volumeObject.getMaxIops(), storagePool.getId()); + } + CloudStackVolume request = createCloudStackVolumeRequestByProtocol( + storagePool, details, volumeObject, qosPolicy); + try { + CloudStackVolume created = storageStrategy.createCloudStackVolume(request); + persistQosPolicyDetails(volumeObject.getId(), qosPolicy); + return created; + } catch (RuntimeException e) { + if (qosPolicy != null) { + storageStrategy.deleteVolumeQosPolicy(qosPolicy.getUuid()); + } + throw e; + } + } + + private VolumeQosPolicy createQosPolicyIfNeeded(StorageStrategy storageStrategy, Map details, + Long minIops, Long maxIops, Long poolId) { + if (!validateIops(storageStrategy, details, poolId, minIops, maxIops)) { + return null; + } + String policyName = getQosPolicyName(details.get(OntapStorageConstants.SVM_NAME), minIops, maxIops); + return storageStrategy.createVolumeQosPolicy(policyName, minIops, maxIops); + } + + private boolean validateIops(StorageStrategy storageStrategy, Map details, + Long poolId, Long minIops, Long maxIops) { + long min = minIops == null ? 0 : minIops; + long max = maxIops == null ? 0 : maxIops; + if (min <= 0 && max <= 0) { + return false; + } + if (min > 0 && max > 0 && min > max) { + throw new CloudRuntimeException("Minimum IOPS cannot be greater than maximum IOPS"); + } + if (min > 0) { + String isAff = details.get(OntapStorageConstants.IS_AFF); + if (StringUtils.isBlank(isAff)) { + try { + isAff = Boolean.toString(storageStrategy.isAff()); + details.put(OntapStorageConstants.IS_AFF, isAff); + storagePoolDetailsDao.addDetail(poolId, OntapStorageConstants.IS_AFF, isAff, false); + } catch (Exception e) { + logger.error("Unable to determine whether storage pool [{}] is AFF or FAS: {}", + poolId, e.getMessage()); + return true; + } + } + if (!Boolean.parseBoolean(isAff)) { + throw new CloudRuntimeException(OntapStorageConstants.MIN_IOPS_NOT_SUPPORTED_ON_FAS); + } + } + return true; + } + + /** + * Builds a reusable SVM-scoped QoS policy name: cs_{min}_to_{max}_iops_{svmName}. + * Dots in the SVM name are replaced with underscores (ONTAP QoS names cannot contain '.'). + */ + private String getQosPolicyName(String svmName, Long minIops, Long maxIops) { + String sanitizedSvmName = svmName == null ? "" : svmName.replace(".", OntapStorageConstants.UNDERSCORE); + long min = minIops != null && minIops > 0 ? minIops : 0; + long max = maxIops != null && maxIops > 0 ? maxIops : 0; + return OntapStorageConstants.QOS_POLICY_NAME_PREFIX + min + OntapStorageConstants.UNDERSCORE + + OntapStorageConstants.QOS_POLICY_NAME_TO + max + OntapStorageConstants.UNDERSCORE + + OntapStorageConstants.QOS_POLICY_NAME_IOPS + sanitizedSvmName; + } + + private void persistQosPolicyDetails(long volumeId, VolumeQosPolicy qosPolicy) { + volumeDetailsDao.removeDetail(volumeId, OntapStorageConstants.QOS_POLICY_UUID); + if (qosPolicy == null) { + return; + } + volumeDetailsDao.addDetail(volumeId, OntapStorageConstants.QOS_POLICY_UUID, qosPolicy.getUuid(), false); + } + + /** + * Creates the backend object that caches a template on this pool's FlexVolume. + * + *

Protocol-specific work is delegated to {@link StorageStrategy#createTemplateCache}. + * This method maps the result to {@link CreateCmdResult} and records SAN identity on + * {@code template_spool_ref} ({@code local_download_path} = LUN uuid).

+ */ + private CreateCmdResult createTemplateOnPrimary(StoragePoolVO storagePool, TemplateInfo templateInfo, Map details) { + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + long sizeInBytes = getDataObjectSizeIncludingHypervisorSnapshotReserve(templateInfo, storagePool); + CloudStackVolume created = null; + try { + created = storageStrategy.createTemplateCache(storagePool, templateInfo, details, sizeInBytes); + String path = recordTemplateCacheOnSpoolRef(storagePool, templateInfo, created, sizeInBytes); + return new CreateCmdResult(path, new Answer(null, true, null)); + } catch (Exception e) { + // Compensating delete for SAN: strategy cleans create-time LUN failures; this covers + // post-create failures (e.g. template_spool_ref update) after a LUN was returned. + if (created != null && created.getLun() != null) { + bestEffortDeleteTemplateCacheLun(storageStrategy, details.get(OntapStorageConstants.SVM_NAME), + created.getLun().getName(), created.getLun().getUuid()); + } + if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException) e; + } + throw new CloudRuntimeException("Failed to create template cache for template [" + templateInfo.getId() + + "]: " + e.getMessage(), e); + } + } + + /** + * Persists SAN cache identity on {@code template_spool_ref} when a LUN was created. + * NFS returns the template uuid as the create path; {@code install_path} is filled later. + */ + private String recordTemplateCacheOnSpoolRef(StoragePoolVO storagePool, TemplateInfo templateInfo, + CloudStackVolume created, long sizeInBytes) { + if (created == null || created.getLun() == null) { + return templateInfo.getUuid(); + } + Lun lun = created.getLun(); + VMTemplateStoragePoolVO templatePoolRef = findTemplatePoolRef(storagePool.getId(), templateInfo.getId()); + templatePoolRef.setLocalDownloadPath(lun.getUuid()); + templatePoolRef.setTemplateSize(sizeInBytes); + vmTemplatePoolDao.update(templatePoolRef.getId(), templatePoolRef); + return lun.getName(); + } + + /** + * Clones the cached template into a new volume on the same FlexVolume. + * + *

Invoked when {@code StorageSystemDataMotionStrategy} has recorded a + * {@code cloneOfTemplate} detail on the volume.

+ */ + private CloudStackVolume cloneCloudStackVolumeFromTemplate(StoragePoolVO storagePool, VolumeInfo volumeInfo, + Map details, long templateId) { + VMTemplateStoragePoolVO templatePoolRef = findTemplatePoolRef(storagePool.getId(), templateId); + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + boolean iscsi = isIscsi(details); + + verifySufficientIopsForStoragePool(storagePool, volumeInfo.getMinIops(), volumeInfo.getId()); + VolumeQosPolicy qosPolicy = null; + Volume.Type volumeType = volumeInfo.getVolumeType(); + if (volumeType == Volume.Type.DATADISK || volumeType == Volume.Type.ROOT) { + qosPolicy = createQosPolicyIfNeeded(storageStrategy, details, + volumeInfo.getMinIops(), volumeInfo.getMaxIops(), storagePool.getId()); + } + + CloudStackVolume request = iscsi + ? createCloneLunRequest(storagePool, details, volumeInfo, templatePoolRef, templateId, qosPolicy) + : createCloneFileRequest(storagePool, volumeInfo, templatePoolRef, templateId); + + try { + CloudStackVolume cloned = storageStrategy.cloneCloudStackVolume(request); + if (cloned == null) { + throw new CloudRuntimeException("ONTAP returned nothing when cloning template [" + templateId + + "] for volume [" + volumeInfo.getId() + "]"); + } + + logger.info("cloneCloudStackVolumeFromTemplate: Cloned template [{}] for volume [{}] on pool [{}]", + templateId, volumeInfo.getId(), storagePool.getId()); + + long requestedSize = getDataObjectSizeIncludingHypervisorSnapshotReserve(volumeInfo, storagePool); + if (requestedSize > templatePoolRef.getTemplateSize()) { + logger.info("cloneCloudStackVolumeFromTemplate: Growing clone of template [{}] from {} to {} bytes for volume [{}]", + templateId, templatePoolRef.getTemplateSize(), requestedSize, volumeInfo.getId()); + resizeClonedVolumeOrCleanup(storageStrategy, cloned, requestedSize, + "cloneCloudStackVolumeFromTemplate template [" + templateId + "] volume [" + volumeInfo.getId() + "]"); + } + + if (!iscsi && qosPolicy != null) { + CloudStackVolume cloudStackVolume = createCloudStackVolumeRequestByProtocol( + storagePool, details, volumeInfo, qosPolicy); + storageStrategy.updateCloudStackVolume(cloudStackVolume); + } + persistQosPolicyDetails(volumeInfo.getId(), qosPolicy); + return cloned; + } catch (Exception e) { + if (qosPolicy != null) { + storageStrategy.deleteVolumeQosPolicy(qosPolicy.getUuid()); + } + throw e; + } + } + + /** + * Returns the CloudStack template id the volume should be cloned from, or null for a blank volume. + * + *

{@code StorageSystemDataMotionStrategy} persists this detail immediately before calling + * {@code createAsync} and removes it right after, so it is only visible during creation.

+ */ + private Long getTemplateIdForCloning(long volumeId) { + VolumeDetailVO detail = volumeDetailsDao.findDetail(volumeId, OntapStorageConstants.CLONE_OF_TEMPLATE); + if (detail == null || detail.getValue() == null || detail.getValue().isEmpty()) { + return null; + } + return Long.valueOf(detail.getValue()); + } + + /** + * Returns the CloudStack snapshot id to clone from when {@code volume_details.cloneOfSnapshot} + * is set, or null when this create is not a create-volume-from-snapshot. + * + *

Set by {@code StorageSystemDataMotionStrategy.handleCreateManagedVolumeFromManagedSnapshot} + * for the duration of {@code createAsync} only (same pattern as {@link #getTemplateIdForCloning}).

+ */ + private Long getSnapshotIdForCloning(long volumeId) { + VolumeDetailVO detail = volumeDetailsDao.findDetail(volumeId, OntapStorageConstants.CLONE_OF_SNAPSHOT); + if (detail == null || detail.getValue() == null || detail.getValue().isEmpty()) { + return null; + } + return Long.valueOf(detail.getValue()); + } + + /** + * Creates a new volume on this pool by cloning a file/LUN from a CloudStack volume snapshot + * that already lives on the same FlexVolume. + * + *

Combinations (product + plugin v1):

+ *
    + *
  • Same pool only — {@code snapshot_details.PRIMARY_POOL_ID} must equal this + * {@code storagePool}. Creating the volume on a different pool is descoped; use migrate later if needed.
    • + *
  • DATA snapshot → attachable data disk (disk offering usually inherited).
    • + *
  • ROOT snapshot → still a data disk here (not bootable). Bootable path remains + * {@code createTemplate(snapshotid)} then deploy.
    • + *
  • iSCSI — {@code POST /api/storage/luns} with + * {@code clone.source.name=/vol/<fv>/.snapshot/<snap>/<lun>}
    • + *
  • NFS3 — {@code POST /api/storage/file/clone} with + * {@code source_path=./.snapshot/<snapshot-name>/<source-file>}
    • + *
  • Not this path — in-place revert ({@code revertSnapshot}); VM/instance snapshots; + * secondary-storage archive restore.
    • + *
+ * + *

Optional grow when the disk offering is larger than the snapshot size (same pattern as + * clone-from-template).

+ * + *

When {@code applyQos} is true, the new volume's min and max IOPS are applied the same way + * as a template clone: iSCSI carries the policy on the clone request, NFS attaches it after + * the file exists. The temporary volume used for {@code createTemplate(snapshotid)} passes + * false and is left without a policy.

+ */ + private CloudStackVolume cloneCloudStackVolumeFromSnapshot(StoragePoolVO storagePool, VolumeInfo volumeInfo, + Map details, long csSnapshotId, + boolean applyQos) { + String snapshotName = requireSnapshotDetail(csSnapshotId, OntapStorageConstants.ONTAP_SNAP_NAME); + String volumePath = requireSnapshotDetail(csSnapshotId, OntapStorageConstants.VOLUME_PATH); + String primaryPoolId = requireSnapshotDetail(csSnapshotId, OntapStorageConstants.PRIMARY_POOL_ID); + String snapProtocol = requireSnapshotDetail(csSnapshotId, OntapStorageConstants.PROTOCOL); + + // Same-pool / same-protocol gate (v1). Fail before any ONTAP call. + if (!String.valueOf(storagePool.getId()).equals(primaryPoolId)) { + throw new CloudRuntimeException("Create volume from snapshot [" + csSnapshotId + + "] requires the snapshot's primary pool [" + primaryPoolId + + "]; requested pool is [" + storagePool.getId() + "] (creating a volume from a snapshot on a different pool is not supported)"); + } + String poolProtocol = details.get(OntapStorageConstants.PROTOCOL); + if (poolProtocol == null || !poolProtocol.equalsIgnoreCase(snapProtocol)) { + throw new CloudRuntimeException("Create volume from snapshot [" + csSnapshotId + + "] protocol mismatch: snapshot=[" + snapProtocol + "], pool=[" + poolProtocol + "]"); + } + + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + boolean iscsi = isIscsi(details); + VolumeQosPolicy qosPolicy = null; + + logger.info("cloneCloudStackVolumeFromSnapshot: Cloning from CS snapshot [{}] (ONTAP snap [{}], path [{}]) " + + "for volume [{}] on pool [{}] protocol [{}]", + csSnapshotId, snapshotName, volumePath, volumeInfo.getId(), storagePool.getId(), poolProtocol); + + try { + if (applyQos) { + verifySufficientIopsForStoragePool(storagePool, volumeInfo.getMinIops(), volumeInfo.getId()); + Volume.Type volumeType = volumeInfo.getVolumeType(); + if (volumeType == Volume.Type.DATADISK || volumeType == Volume.Type.ROOT) { + qosPolicy = createQosPolicyIfNeeded(storageStrategy, details, + volumeInfo.getMinIops(), volumeInfo.getMaxIops(), storagePool.getId()); + } + } + + CloudStackVolume request = iscsi + ? createCloneLunFromSnapshotRequest(storagePool, details, volumeInfo, volumePath, snapshotName, qosPolicy) + : createCloneFileFromSnapshotRequest(storagePool, details, volumeInfo, volumePath, snapshotName); + CloudStackVolume cloned = storageStrategy.cloneCloudStackVolume(request); + if (cloned == null) { + throw new CloudRuntimeException("ONTAP returned nothing when cloning snapshot [" + csSnapshotId + + "] for volume [" + volumeInfo.getId() + "]"); + } + + long requestedSize = getDataObjectSizeIncludingHypervisorSnapshotReserve(volumeInfo, storagePool); + long snapshotSize = resolveSnapshotSizeBytes(csSnapshotId); + if (snapshotSize > 0 && requestedSize > snapshotSize) { + logger.info("cloneCloudStackVolumeFromSnapshot: Growing clone of snapshot [{}] from {} to {} bytes for volume [{}]", + csSnapshotId, snapshotSize, requestedSize, volumeInfo.getId()); + resizeClonedVolumeOrCleanup(storageStrategy, cloned, requestedSize, + "cloneCloudStackVolumeFromSnapshot snapshot [" + csSnapshotId + "] volume [" + volumeInfo.getId() + "]"); + } + + if (!iscsi && qosPolicy != null) { + CloudStackVolume cloudStackVolume = createCloudStackVolumeRequestByProtocol( + storagePool, details, volumeInfo, qosPolicy); + storageStrategy.updateCloudStackVolume(cloudStackVolume); + } + if (applyQos) { + persistQosPolicyDetails(volumeInfo.getId(), qosPolicy); + } + return cloned; + } catch (Exception e) { + if (qosPolicy != null) { + storageStrategy.deleteVolumeQosPolicy(qosPolicy.getUuid()); + } + throw e; + } + } + + /** + * Grows a just-created ONTAP clone to the requested size. If resize fails, deletes the clone + * so the CloudStack create-failure path does not leave an orphaned LUN/file on the array. + */ + private void resizeClonedVolumeOrCleanup(StorageStrategy storageStrategy, CloudStackVolume cloned, + long requestedSize, String context) { + try { + storageStrategy.resizeCloudStackVolume(cloned, requestedSize); + } catch (RuntimeException resizeEx) { + logger.error("{}: Resize to {} bytes failed; deleting orphaned ONTAP clone: {}", + context, requestedSize, resizeEx.getMessage()); + bestEffortDeleteOrphanedClone(storageStrategy, cloned); + throw resizeEx; + } + } + + private void bestEffortDeleteOrphanedClone(StorageStrategy storageStrategy, CloudStackVolume cloned) { + try { + storageStrategy.deleteCloudStackVolume(cloned); + logger.info("Deleted orphaned ONTAP clone after resize failure"); + } catch (Exception cleanupEx) { + logger.warn("Failed to delete orphaned ONTAP clone after resize failure: {}", cleanupEx.getMessage()); + } + } + + private String requireSnapshotDetail(long csSnapshotId, String key) { + String value = getSnapshotDetail(csSnapshotId, key); + if (value == null || value.isEmpty()) { + throw new CloudRuntimeException("Missing snapshot_details [" + key + "] for snapshot [" + csSnapshotId + + "]; cannot create volume from snapshot"); + } + return value; + } + + private long resolveSnapshotSizeBytes(long csSnapshotId) { + SnapshotVO snapshotVO = snapshotDao.findById(csSnapshotId); + if (snapshotVO == null || snapshotVO.getSize() <= 0) { + return 0L; + } + return snapshotVO.getSize(); + } + + private VMTemplateStoragePoolVO findTemplatePoolRef(long poolId, long templateId) { + VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId, null); + if (templatePoolRef == null) { + throw new CloudRuntimeException("No template_spool_ref row for template [" + templateId + "] on pool [" + poolId + "]"); + } + return templatePoolRef; + } + + /** + * Deletes the LUN caching a template on this pool, invoked by template eviction + * ({@code TemplateManagerImpl.evictTemplateFromStoragePool}). + * + *

Volumes previously cloned from this LUN are unaffected: an ONTAP sis-clone shares blocks + * with its source through reference counting rather than depending on it, so the source can be + * removed while its clones stay online.

+ * + *

{@code deleteCloudStackVolume} already unmaps as it deletes ({@code allow_delete_while_mapped}) + * and treats a missing LUN as success.

+ */ + private void deleteTemplateOnPrimary(DataStore store, TemplateInfo templateInfo) { + StoragePoolVO storagePool = storagePoolDao.findById(store.getId()); + if (storagePool == null) { + throw new CloudRuntimeException("Storage Pool not found for id: " + store.getId()); + } + + Map details = storagePoolDetailsDao.listDetailsKeyPairs(store.getId()); + VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePool.getId(), templateInfo.getId(), null); + if (templatePoolRef == null) { + logger.warn("deleteTemplateOnPrimary: No template_spool_ref for template [{}] on pool [{}]; nothing to delete", + templateInfo.getId(), storagePool.getId()); + return; + } + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); - CloudStackVolume cloudStackVolumeRequest = OntapStorageUtils.createCloudStackVolumeRequestByProtocol(storagePool, details, volumeObject); - return storageStrategy.createCloudStackVolume(cloudStackVolumeRequest); + if (isIscsi(details)) { + deleteIscsiTemplateCache(storagePool, templateInfo, templatePoolRef, storageStrategy, details); + } else { + deleteNfsTemplateCache(details, templateInfo, templatePoolRef, storageStrategy); + } + } + + private void deleteIscsiTemplateCache(StoragePoolVO storagePool, TemplateInfo templateInfo, + VMTemplateStoragePoolVO templatePoolRef, StorageStrategy storageStrategy, + Map details) { + String lunName = getTemplateLunName(storagePool, templateInfo.getId()); + String lunUuid = templatePoolRef.getLocalDownloadPath(); + deleteTemplateCacheLun(storageStrategy, details.get(OntapStorageConstants.SVM_NAME), lunName, lunUuid); + logger.info("deleteIscsiTemplateCache: Deleted template cache LUN [{}] for template [{}] on pool [{}]", + lunName, templateInfo.getId(), storagePool.getId()); + } + + /** + * Deletes a template-cache LUN. When {@code lunUuid} is missing, resolves the LUN by its + * deterministic name so eviction and create-time rollback still work. + */ + private void deleteTemplateCacheLun(StorageStrategy storageStrategy, String svmName, String lunName, String lunUuid) { + String uuid = lunUuid; + if (uuid == null || uuid.isEmpty()) { + CloudStackVolume existing = getCloudStackVolumeByName(storageStrategy, svmName, lunName); + if (existing == null || existing.getLun() == null || existing.getLun().getUuid() == null) { + logger.warn("deleteTemplateCacheLun: LUN [{}] not found on SVM [{}]; nothing to delete", lunName, svmName); + return; + } + uuid = existing.getLun().getUuid(); + } + + Lun lun = new Lun(); + lun.setUuid(uuid); + lun.setName(lunName); + + CloudStackVolume deleteRequest = new CloudStackVolume(); + deleteRequest.setLun(lun); + storageStrategy.deleteCloudStackVolume(deleteRequest); + } + + /** + * Best-effort cleanup of a template-cache LUN after a failed create. Never masks the original + * create failure. + */ + private void bestEffortDeleteTemplateCacheLun(StorageStrategy storageStrategy, String svmName, String lunName, + String lunUuid) { + try { + deleteTemplateCacheLun(storageStrategy, svmName, lunName, lunUuid); + logger.info("bestEffortDeleteTemplateCacheLun: Removed leftover template cache LUN [{}] on SVM [{}]", + lunName, svmName); + } catch (Exception cleanupEx) { + logger.warn("bestEffortDeleteTemplateCacheLun: Failed to remove leftover template cache LUN [{}] on SVM [{}]: {}", + lunName, svmName, cleanupEx.getMessage()); + } + } + + private void deleteNfsTemplateCache(Map details, TemplateInfo templateInfo, + VMTemplateStoragePoolVO templatePoolRef, StorageStrategy storageStrategy) { + String filePath = templatePoolRef.getInstallPath(); + if (filePath == null || filePath.isEmpty()) { + logger.warn("deleteNfsTemplateCache: No install_path recorded for template [{}]; nothing to delete", + templateInfo.getId()); + return; + } + String flexVolUuid = details.get(OntapStorageConstants.VOLUME_UUID); + if (flexVolUuid == null || flexVolUuid.isEmpty()) { + // Misconfigured pool detail — fail eviction rather than calling ONTAP with a null + // volume UUID (which would hit /api/storage/volumes/null and still look like success + // upstream if we swallowed the error). + throw new CloudRuntimeException("FlexVolume UUID (volumeUUID) is missing from storage pool details; " + + "cannot delete NFS template cache file [" + filePath + "] for template [" + + templateInfo.getId() + "]"); + } + ((UnifiedNASStrategy) storageStrategy).deleteFileByPath(flexVolUuid, filePath); + logger.info("deleteNfsTemplateCache: Deleted template cache file [{}] for template [{}]", + filePath, templateInfo.getId()); + } + + /** + * Rejects the request when the minimum IOPS being asked for would push the pool past its + * configured IOPS capacity. Pools without an IOPS capacity enforce no ceiling. + * + * Used IOPS include volumes still in {@link Volume.State#Creating} so a second overlapping + * create sees the first reservation. The volume being checked is omitted so its own min IOPS + * are not counted twice (CloudStack has already moved it to Creating before this runs). + */ + private void verifySufficientIopsForStoragePool(StoragePoolVO storagePool, Long requestedMinIops, long excludeVolumeId) { + Long capacityIops = storagePool.getCapacityIops(); + if (capacityIops == null || requestedMinIops == null || requestedMinIops <= 0) { + return; + } + + long requestedTotalIops = getAllocatedMinIops(storagePool, excludeVolumeId) + requestedMinIops; + if (requestedTotalIops > capacityIops) { + throw new CloudRuntimeException(String.format( + "Insufficient IOPS capacity on storage pool %s: requested total of %d IOPS exceeds the pool IOPS capacity of %d", + storagePool.getName(), requestedTotalIops, capacityIops)); + } } /** * Deletes a volume or snapshot from the ONTAP storage system. * - *

For volumes, deletes the backend storage object (LUN for iSCSI, no-op for NFS). - * For snapshots, deletes the FlexVolume snapshot from ONTAP that was created by takeSnapshot.

+ *

For volumes, deletes the backend storage object (LUN for iSCSI, file for NFS) via + * {@link StorageStrategy#deleteCloudStackVolume}.

+ * + *

For volume snapshots, this driver is invoked by the standard CloudStack delete chain + * ({@code StorageSystemSnapshotStrategy} → {@code SnapshotServiceImpl.deleteSnapshot} → + * {@code deleteAsync}). It reads ONTAP metadata from {@code snapshot_details} and delegates + * the actual FlexVol snapshot delete to {@link StorageStrategy} (NFS or iSCSI implementation). + * ONTAP REST/delete-job logic must not live here — keep it in the storage-strategy layer.

*/ @Override public void deleteAsync(DataStore store, DataObject data, AsyncCompletionCallback callback) { @@ -231,14 +812,39 @@ public void deleteAsync(DataStore store, DataObject data, AsyncCompletionCallbac StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); logger.info("createCloudStackVolumeForTypeVolume: Connection to Ontap SVM [{}] successful, preparing CloudStackVolumeRequest", details.get(OntapStorageConstants.SVM_NAME)); VolumeInfo volumeInfo = (VolumeInfo) data; - CloudStackVolume cloudStackVolumeRequest = createDeleteCloudStackVolumeRequest(storagePool, details, volumeInfo); - storageStrategy.deleteCloudStackVolume(cloudStackVolumeRequest); + VolumeDetailVO qosPolicyDetail = volumeDetailsDao.findDetail(volumeInfo.getId(), OntapStorageConstants.QOS_POLICY_UUID); + + // NFS file delete leaves the QoS assignment in place unless the file is set to none first. + // iSCSI LUN delete already drops object_count. + // TODO: revisit this NFS none-policy step when the file is deleted from ONTAP instead of KVM. + boolean qosCleared = false; + if (!isIscsi(details)) { + qosCleared = clearQosPolicyBeforeDelete(storageStrategy, storagePool, details, volumeInfo, qosPolicyDetail); + } + try { + CloudStackVolume cloudStackVolumeRequest = createDeleteCloudStackVolumeRequest(storagePool, details, volumeInfo); + storageStrategy.deleteCloudStackVolume(cloudStackVolumeRequest); + } catch (Exception e) { + if (qosCleared) { + restoreQosPolicyAfterFailedDelete(storageStrategy, storagePool, details, volumeInfo, qosPolicyDetail); + } + throw e; + } + if (qosPolicyDetail != null) { + deleteUnusedQosPolicy(storageStrategy, qosPolicyDetail.getValue()); + } logger.info("deleteAsync: Volume deleted: " + volumeInfo.getId()); + removeTemporarySnapshotCopyRecord(volumeInfo.getId()); + commandResult.setResult(null); + commandResult.setSuccess(true); + } else if (data.getType() == DataObjectType.TEMPLATE) { + deleteTemplateOnPrimary(store, (TemplateInfo) data); commandResult.setResult(null); commandResult.setSuccess(true); } else if (data.getType() == DataObjectType.SNAPSHOT) { - // Delete the ONTAP FlexVolume snapshot that was created by takeSnapshot - deleteOntapSnapshot((SnapshotInfo) data, commandResult); + logger.info("deleteAsync: volume-snapshot delete for CloudStack snapshot [{}] on primary pool [{}] — " + + "delegating ONTAP FlexVol cleanup to StorageStrategy", data.getId(), store.getId()); + deleteCloudStackVolumeSnapshot((SnapshotInfo) data, commandResult); } else { throw new CloudRuntimeException("Unsupported data object type: " + data.getType()); } @@ -247,104 +853,386 @@ public void deleteAsync(DataStore store, DataObject data, AsyncCompletionCallbac commandResult.setSuccess(false); commandResult.setResult(e.getMessage()); } finally { - callback.complete(commandResult); + // StorageSystemDataMotionStrategy deletes its temporary snapshot copy with a null callback. + if (callback != null) { + callback.complete(commandResult); + } + } + } + + /** + * Sets the NFS file QoS policy to {@code none} before the file is deleted. + * Failure must not block volume delete. + * + * @return true when the file was moved off its policy + */ + private boolean clearQosPolicyBeforeDelete(StorageStrategy storageStrategy, StoragePoolVO storagePool, + Map details, VolumeInfo volumeInfo, + VolumeDetailVO qosPolicyDetail) { + if (qosPolicyDetail == null || qosPolicyDetail.getValue() == null) { + return false; + } + VolumeQosPolicy nonePolicy = new VolumeQosPolicy(); + nonePolicy.setName(OntapStorageConstants.QOS_POLICY_NONE); + try { + CloudStackVolume request = createCloudStackVolumeRequestByProtocol(storagePool, details, volumeInfo, nonePolicy); + storageStrategy.updateCloudStackVolume(request); + return true; + } catch (Exception e) { + logger.warn("Could not clear QoS policy [{}] on volume [{}] before delete: {}", + qosPolicyDetail.getValue(), volumeInfo.getId(), e.getMessage()); + return false; + } + } + + /** + * Puts the previous QoS policy back on an NFS file when delete fails after the policy was cleared. + */ + private void restoreQosPolicyAfterFailedDelete(StorageStrategy storageStrategy, StoragePoolVO storagePool, + Map details, VolumeInfo volumeInfo, + VolumeDetailVO qosPolicyDetail) { + if (qosPolicyDetail == null || qosPolicyDetail.getValue() == null) { + return; + } + VolumeQosPolicy previousPolicy = new VolumeQosPolicy(); + previousPolicy.setUuid(qosPolicyDetail.getValue()); + try { + CloudStackVolume request = createCloudStackVolumeRequestByProtocol(storagePool, details, volumeInfo, previousPolicy); + storageStrategy.updateCloudStackVolume(request); + } catch (Exception e) { + logger.warn("Could not restore QoS policy [{}] on volume [{}] after delete failed: {}", + qosPolicyDetail.getValue(), volumeInfo.getId(), e.getMessage()); } } /** - * Deletes an ONTAP FlexVolume snapshot. + * Orchestrates CloudStack volume-snapshot delete on ONTAP. * - *

Retrieves the snapshot details stored during takeSnapshot and calls the ONTAP - * REST API to delete the FlexVolume snapshot.

+ *

This method is intentionally thin: it resolves identifiers persisted during + * {@link #takeSnapshot} into {@code snapshot_details} and delegates to the protocol + * {@link StorageStrategy} selected from pool details (NFS → {@code UnifiedNASStrategy}, + * iSCSI → {@code UnifiedSANStrategy}). Both protocols share the same FlexVol-level + * snapshot delete REST API.

* - * @param snapshotInfo The CloudStack snapshot to delete - * @param commandResult Result object to populate with success/failure + *

Required {@code snapshot_details} keys (see {@link OntapStorageConstants}):

+ *
    + *
  • {@code base_ontap_fv_id} — FlexVol UUID
    • + *
  • {@code ontap_snap_id} — ONTAP snapshot UUID
    • + *
  • {@code ontap_snap_name} — snapshot name (logging)
    • + *
  • {@code primary_pool_id} — pool used to obtain credentials/protocol strategy
    • + *
*/ - private void deleteOntapSnapshot(SnapshotInfo snapshotInfo, CommandResult commandResult) { + private void deleteCloudStackVolumeSnapshot(SnapshotInfo snapshotInfo, CommandResult commandResult) { long snapshotId = snapshotInfo.getId(); - logger.info("deleteOntapSnapshot: Deleting ONTAP FlexVolume snapshot for CloudStack snapshot [{}]", snapshotId); + logger.info("deleteCloudStackVolumeSnapshot: starting ONTAP delete for CloudStack volume snapshot [{}]", snapshotId); try { - // Retrieve snapshot details stored during takeSnapshot String flexVolUuid = getSnapshotDetail(snapshotId, OntapStorageConstants.BASE_ONTAP_FV_ID); String ontapSnapshotUuid = getSnapshotDetail(snapshotId, OntapStorageConstants.ONTAP_SNAP_ID); String snapshotName = getSnapshotDetail(snapshotId, OntapStorageConstants.ONTAP_SNAP_NAME); String poolIdStr = getSnapshotDetail(snapshotId, OntapStorageConstants.PRIMARY_POOL_ID); if (flexVolUuid == null || ontapSnapshotUuid == null) { - logger.warn("deleteOntapSnapshot: Missing ONTAP snapshot details for snapshot [{}]. " + - "flexVolUuid={}, ontapSnapshotUuid={}. Snapshot may have been created by a different method or already deleted.", + logger.warn("deleteCloudStackVolumeSnapshot: missing ONTAP identity for snapshot [{}] " + + "(flexVolUuid={}, ontapSnapshotUuid={}). Cannot call ONTAP delete; " + + "treating as no-op — verify snapshot_details were written during takeSnapshot", snapshotId, flexVolUuid, ontapSnapshotUuid); - // Consider this a success since there's nothing to delete on ONTAP commandResult.setSuccess(true); commandResult.setResult(null); return; } - long poolId = Long.parseLong(poolIdStr); + long poolId = resolveSnapshotPoolId(poolIdStr, snapshotId); Map poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(poolId); - + String protocol = poolDetails.get(OntapStorageConstants.PROTOCOL); StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(poolDetails); - SnapshotFeignClient snapshotClient = storageStrategy.getSnapshotFeignClient(); - String authHeader = storageStrategy.getAuthHeader(); - logger.info("deleteOntapSnapshot: Deleting ONTAP snapshot [{}] (uuid={}) from FlexVol [{}]", - snapshotName, ontapSnapshotUuid, flexVolUuid); + logger.info("deleteCloudStackVolumeSnapshot: snapshot [{}] — protocol [{}], pool [{}], " + + "flexVol [{}], ontapSnapshot [{}] (name [{}])", + snapshotId, protocol, poolId, flexVolUuid, ontapSnapshotUuid, snapshotName); - // Call ONTAP REST API to delete the snapshot - JobResponse jobResponse = snapshotClient.deleteSnapshot(authHeader, flexVolUuid, ontapSnapshotUuid); + storageStrategy.deleteFlexVolSnapshotForCloudStackVolume(flexVolUuid, ontapSnapshotUuid, snapshotName); - if (jobResponse != null && jobResponse.getJob() != null) { - // Poll for job completion - Boolean jobSucceeded = storageStrategy.jobPollForSuccess(jobResponse.getJob().getUuid(), 30, 2000); - if (!jobSucceeded) { - throw new CloudRuntimeException("Delete job failed for snapshot [" + - snapshotName + "] on FlexVol [" + flexVolUuid + "]"); - } + logger.info("deleteCloudStackVolumeSnapshot: completed ONTAP delete for CloudStack volume snapshot [{}]", snapshotId); + commandResult.setSuccess(true); + commandResult.setResult(null); + } catch (Exception e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + logger.warn("deleteCloudStackVolumeSnapshot: ONTAP snapshot for CloudStack snapshot [{}] " + + "already absent (idempotent success): {}", snapshotId, e.getMessage()); + commandResult.setSuccess(true); + commandResult.setResult(null); + return; + } + logger.error("deleteCloudStackVolumeSnapshot: ONTAP delete failed for CloudStack snapshot [{}]: {}", + snapshotId, e.getMessage(), e); + commandResult.setSuccess(false); + commandResult.setResult(e.getMessage()); + } + } + + private long resolveSnapshotPoolId(String poolIdStr, long snapshotId) { + if (poolIdStr != null && !poolIdStr.isEmpty()) { + return Long.parseLong(poolIdStr); + } + SnapshotVO snapshotVO = snapshotDao.findById(snapshotId); + if (snapshotVO == null) { + throw new CloudRuntimeException("Snapshot not found for snapshot [" + snapshotId + "]"); + } + VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(snapshotVO.getVolumeId()); + if (volumeVO == null) { + throw new CloudRuntimeException("CloudStack Volume not found for snapshot [" + snapshotId + "]"); + } + Long poolId = volumeVO.getPoolId() != null ? volumeVO.getPoolId() : volumeVO.getLastPoolId(); + if (poolId == null || poolId <= 0) { + throw new CloudRuntimeException("Cannot resolve storage pool for snapshot [" + snapshotId + "]"); + } + return poolId; + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback callback) { + copyAsync(srcData, destData, null, callback); + } + + /** + * Dispatches a driver-level copy by (source type, destination type). Only the pairs accepted by + * {@link #canCopy} are handled; everything else is left to the data motion strategies. + * + *

Callers may pass a null callback (for example {@code StorageSystemDataMotionStrategy} for the + * temporary snapshot copy), in which case a failed copy is thrown.

+ */ + @Override + public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback callback) { + CopyCommandResult result; + if (isSnapshotToTemporaryVolumeCopy(srcData, destData)) { + result = copySnapshotToTemporaryVolume(srcData, (VolumeInfo) destData); + } else { + throw new UnsupportedOperationException("Copy from " + describeForCopy(srcData) + " to " + + describeForCopy(destData) + " is not supported for ONTAP primary storage."); + } + + if (callback != null) { + callback.complete(result); + } else if (!result.isSuccess()) { + throw new CloudRuntimeException("Failed to copy " + srcData.getType() + " [" + srcData.getId() + "] to " + + destData.getType() + " [" + destData.getId() + "]: " + result.getResult()); + } + } + + /** + * Returns true only for the (source type, destination type) pairs the driver copies itself. + * + *

{@code DataMotionServiceImpl} consults {@code canCopy} before choosing a data motion strategy, + * so any pair accepted here bypasses {@code StorageSystemDataMotionStrategy} (volume migration, + * createVolume(snapshotid), template caching). Add a pair only together with its handler in + * {@link #copyAsync}.

+ */ + @Override + public boolean canCopy(DataObject srcData, DataObject destData) { + if (srcData == null || destData == null || srcData.getDataStore() == null || destData.getDataStore() == null) { + return false; + } + return isSnapshotToTemporaryVolumeCopy(srcData, destData); + } + + private String describeForCopy(DataObject data) { + return data == null ? "null" : data.getType() + " [" + data.getId() + "]"; + } + + /** + * Matches the temporary volume {@code StorageSystemDataMotionStrategy} creates on the snapshot's own + * pool while copying the snapshot to secondary storage for {@code createTemplate(snapshotid)}. + */ + private boolean isSnapshotToTemporaryVolumeCopy(DataObject srcData, DataObject destData) { + if (srcData == null || destData == null || srcData.getDataStore() == null || destData.getDataStore() == null) { + return false; + } + if (srcData.getType() != DataObjectType.SNAPSHOT || destData.getType() != DataObjectType.VOLUME) { + return false; + } + if (srcData.getDataStore().getRole() != DataStoreRole.Primary + || srcData.getDataStore().getId() != destData.getDataStore().getId()) { + return false; + } + return isTemporarySnapshotCopyVolume(volumeDao.findById(destData.getId())); + } + + /** + * Clones a CloudStack volume snapshot into the temporary volume used for + * {@code createTemplate(snapshotid)}. The strategy then maps that volume to a KVM host, copies it + * into the template and deletes it again through {@link #deleteAsync}. + */ + private CopyCommandResult copySnapshotToTemporaryVolume(DataObject srcData, VolumeInfo volInfo) { + try { + StoragePoolVO storagePool = storagePoolDao.findById(volInfo.getDataStore().getId()); + if (storagePool == null) { + throw new CloudRuntimeException("Storage Pool not found for id: " + volInfo.getDataStore().getId()); + } + Map details = storagePoolDetailsDao.listDetailsKeyPairs(storagePool.getId()); + validateProtocol(details, volInfo.getDataStore()); + VolumeVO volumeVO = volumeDao.findById(volInfo.getId()); + + renameTemporarySnapshotCopy(volInfo, volumeVO, srcData.getId()); + logger.info("copySnapshotToTemporaryVolume: Cloning CS snapshot [{}] into temporary volume [{}] on pool [{}] for template creation", + srcData.getId(), volInfo.getId(), storagePool.getId()); + CloudStackVolume cloned = cloneCloudStackVolumeFromSnapshot(storagePool, volInfo, details, srcData.getId(), false); + if (ProtocolType.NFS3.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { + volumeVO.setPath(volInfo.getUuid()); } + recordCreatedVolume(storagePool, volInfo, volumeVO, details, cloned); + return new CopyCommandResult(null, new CopyCmdAnswer(volInfo.getTO())); + } catch (Exception e) { + logger.error("copySnapshotToTemporaryVolume: Failed to clone snapshot [{}] into volume [{}]: {}", + srcData.getId(), volInfo.getId(), e.getMessage()); + CopyCommandResult result = new CopyCommandResult(null, new CopyCmdAnswer(e.getMessage())); + result.setResult(e.getMessage()); + return result; + } + } + + /** + * The temporary volume created by {@code StorageSystemDataMotionStrategy} for a snapshot copy is + * persisted in Allocated state without a disk offering; user volumes always carry a disk offering. + */ + private boolean isTemporarySnapshotCopyVolume(VolumeVO volumeVO) { + if (volumeVO == null || volumeVO.getState() != Volume.State.Allocated) { + return false; + } + Long diskOfferingId = volumeVO.getDiskOfferingId(); + return diskOfferingId == null || diskOfferingId == 0L; + } + + /** + * The framework names the temporary volume {@code _.TMP}, which is not a valid + * ONTAP LUN name, so it is renamed before cloning. The in-memory object is updated as well because the + * SAN clone derives the LUN name from {@link VolumeInfo#getName()}. + */ + private void renameTemporarySnapshotCopy(VolumeInfo volInfo, VolumeVO volumeVO, long csSnapshotId) { + String name = OntapStorageConstants.TEMP_SNAPSHOT_COPY_NAME_PREFIX + csSnapshotId + OntapStorageConstants.UNDERSCORE + volInfo.getId(); + volumeVO.setName(name); + if (volInfo.getVolume() instanceof VolumeVO) { + ((VolumeVO) volInfo.getVolume()).setName(name); + } + } + + /** + * {@code StorageSystemDataMotionStrategy} deletes the temporary snapshot copy on the array but leaves its + * volume row behind on success, so it is removed here once the backend object is gone. + */ + private void removeTemporarySnapshotCopyRecord(long volumeId) { + VolumeVO volumeVO = volumeDao.findById(volumeId); + if (!isTemporarySnapshotCopyVolume(volumeVO) + || volumeVO.getName() == null || !volumeVO.getName().startsWith(OntapStorageConstants.TEMP_SNAPSHOT_COPY_NAME_PREFIX)) { + return; + } + volumeDetailsDao.removeDetails(volumeId); + volumeDao.remove(volumeId); + logger.info("deleteAsync: Removed temporary snapshot copy volume record [{}]", volumeId); + } + + /** + * Resizes a volume. Size and QoS from the resize window are applied together by + * {@link StorageStrategy#resizeCloudStackVolume}. + */ + @Override + public void resize(DataObject data, AsyncCompletionCallback callback) { + CreateCmdResult result = null; + try { + if (data != null && data.getType() == DataObjectType.VOLUME) { + if (data instanceof VolumeInfo) { + VolumeQosPolicy policyToApply = null; + VolumeInfo volumeInfo = (VolumeInfo) data; + Object rawPayload = volumeInfo.getpayload(); + ResizeVolumePayload payload = (rawPayload instanceof ResizeVolumePayload) + ? (ResizeVolumePayload) rawPayload : null; + if (payload == null || payload.newSize == null) { + throw new CloudRuntimeException("Invalid resize payload for volume " + volumeInfo.getId()); + } + if (volumeInfo.getDataStore() == null) { + throw new CloudRuntimeException("Data store not found for volume " + volumeInfo.getId()); + } + + StoragePoolVO storagePool = storagePoolDao.findById(volumeInfo.getDataStore().getId()); + if (storagePool == null) { + throw new CloudRuntimeException("Storage pool not found for volume " + volumeInfo.getId()); + } + Map details = storagePoolDetailsDao.listDetailsKeyPairs(storagePool.getId()); + + VolumeVO volumeVO = volumeDao.findById(volumeInfo.getId()); + if (volumeVO == null) { + throw new CloudRuntimeException("Volume not found for id " + volumeInfo.getId()); + } + if (payload.newSize < volumeVO.getSize()) { + throw new CloudRuntimeException("Unable to shrink volume."); + } - logger.info("deleteOntapSnapshot: Successfully deleted ONTAP snapshot [{}] (uuid={}) for CloudStack snapshot [{}]", - snapshotName, ontapSnapshotUuid, snapshotId); + verifySufficientIopsForStoragePool(storagePool, payload.newMinIops, volumeVO.getId()); + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + VolumeDetailVO qosDetail = volumeDetailsDao.findDetail(volumeVO.getId(), OntapStorageConstants.QOS_POLICY_UUID); + String previousUuid = qosDetail != null ? qosDetail.getValue() : null; + VolumeQosPolicy qosPolicy = createQosPolicyIfNeeded(storageStrategy, details, + payload.newMinIops, payload.newMaxIops, storagePool.getId()); + + if (qosPolicy != null && !Objects.equals(previousUuid, qosPolicy.getUuid())) { + policyToApply = qosPolicy; + } else if (qosPolicy == null && previousUuid != null) { + policyToApply = new VolumeQosPolicy(); + policyToApply.setName(OntapStorageConstants.QOS_POLICY_NONE); + } - commandResult.setSuccess(true); - commandResult.setResult(null); + try { + CloudStackVolume cloudStackVolume = resizeCloudStackVolumeRequest(details, volumeInfo, policyToApply); + storageStrategy.resizeCloudStackVolume(cloudStackVolume, payload.newSize); + } catch (RuntimeException e) { + if (qosPolicy != null && !Objects.equals(previousUuid, qosPolicy.getUuid())) { + deleteUnusedQosPolicy(storageStrategy, qosPolicy.getUuid()); + } + throw e; + } + if (policyToApply != null) { + persistQosPolicyDetails(volumeVO.getId(), qosPolicy); + deleteUnusedQosPolicy(storageStrategy, previousUuid); + } - } catch (Exception e) { - // Check if the error indicates snapshot doesn't exist (already deleted) - String errorMsg = e.getMessage(); - if (errorMsg != null && (errorMsg.contains("404") || errorMsg.contains("not found") || - errorMsg.contains("does not exist"))) { - logger.warn("deleteOntapSnapshot: ONTAP snapshot for CloudStack snapshot [{}] not found, " + - "may have been already deleted. Treating as success.", snapshotId); - commandResult.setSuccess(true); - commandResult.setResult(null); + volumeVO.setSize(payload.newSize); + if (!volumeDao.update(volumeVO.getId(), volumeVO)) { + throw new CloudRuntimeException("Failed to update volume " + volumeVO.getId() + + " after resizing the ONTAP backing object"); + } + result = new CreateCmdResult(volumeVO.getPath(), new Answer(null, true, null)); + logger.info("resize: Successfully resized volume [{}] to [{}] bytes", volumeInfo.getId(), payload.newSize); + }else{ + String errorMessage = "Invalid DataObjectType (" + data.getType() + ") passed to resize"; + result = new CreateCmdResult(null, new Answer(null, false, errorMessage)); + result.setResult(errorMessage); + return; + } } else { - logger.error("deleteOntapSnapshot: Failed to delete ONTAP snapshot for CloudStack snapshot [{}]: {}", - snapshotId, e.getMessage(), e); - commandResult.setSuccess(false); - commandResult.setResult(e.getMessage()); + throw new CloudRuntimeException("Expected a VOLUME DataObject but received " + + (data != null ? data.getType() : "null")); } + } catch (Exception e) { + String errMsg = e.getMessage(); + logger.error("resize: Failed for volume [{}]: {}", data != null ? data.getId() : null, errMsg, e); + result = new CreateCmdResult(null, new Answer(null, false, errMsg)); + result.setResult(errMsg); + } finally { + callback.complete(result); } } - @Override - public void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback callback) { - throw new UnsupportedOperationException("Copy operation is not supported for ONTAP primary storage."); - } - - @Override - public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback callback) { - throw new UnsupportedOperationException("Copy operation is not supported for ONTAP primary storage."); - } - - @Override - public boolean canCopy(DataObject srcData, DataObject destData) { - return false; + private void deleteUnusedQosPolicy(StorageStrategy storageStrategy, String policyUuid) { + if (policyUuid == null || policyUuid.isEmpty()) { + return; + } + try { + storageStrategy.deleteVolumeQosPolicy(policyUuid); + } catch (Exception e) { + logger.error("Unused QoS policy [{}] was not deleted: {}", policyUuid, e.getMessage()); + } } - @Override - public void resize(DataObject data, AsyncCompletionCallback callback) {} - @Override public ChapInfo getChapInfo(DataObject dataObject) { return null; @@ -399,6 +1287,9 @@ public boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore volumeVO.setPoolType(storagePool.getPoolType()); volumeVO.setPoolId(storagePool.getId()); volumeDao.update(volumeVO.getId(), volumeVO); + syncTemporarySnapshotCopyPath(dataObject, volumeVO); + } else if (dataObject.getType() == DataObjectType.TEMPLATE) { + grantAccessTemplate((TemplateInfo) dataObject, host, dataStore, storagePool); } else { logger.error("Invalid DataObjectType (" + dataObject.getType() + ") passed to grantAccess"); throw new CloudRuntimeException("Invalid DataObjectType (" + dataObject.getType() + ") passed to grantAccess"); @@ -410,28 +1301,122 @@ public boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore } } + /** + * For the temporary snapshot copy, {@code StorageSystemDataMotionStrategy} builds the CopyCommand from + * the same in-memory volume object it passed to grantAccess, so the iSCSI path resolved by the LUN + * mapping must be reflected on that object too. + */ + private void syncTemporarySnapshotCopyPath(DataObject dataObject, VolumeVO volumeVO) { + if (!isTemporarySnapshotCopyVolume(volumeVO) || !(dataObject instanceof VolumeInfo)) { + return; + } + Volume inMemoryVolume = ((VolumeInfo) dataObject).getVolume(); + if (inMemoryVolume instanceof VolumeVO) { + ((VolumeVO) inMemoryVolume).setPath(volumeVO.getPath()); + ((VolumeVO) inMemoryVolume).set_iScsiName(volumeVO.get_iScsiName()); + } + } + private void grantAccessIscsi(Host host, VolumeVO volumeVO, Map details, String svmName, StoragePoolVO storagePool) { String cloudStackVolumeName = volumeDetailsDao.findDetail(volumeVO.getId(), OntapStorageConstants.LUN_DOT_NAME).getValue(); UnifiedSANStrategy sanStrategy = (UnifiedSANStrategy) OntapStorageUtils.getStrategyByStoragePoolDetails(details); String accessGroupName = OntapStorageUtils.getIgroupName(svmName, host.getUuid()); - // Validate if Igroup exist ONTAP for this host as we may be using delete_on_unmap= true and igroup may be deleted by ONTAP automatically + ensureAccessGroupForHost(sanStrategy, host, storagePool, svmName, accessGroupName); + + // Create or retrieve existing LUN mapping. + String lunNumber = sanStrategy.ensureLunMapped(svmName, cloudStackVolumeName, accessGroupName); + String lunWwid = resolveVolumeLunWwid(sanStrategy, svmName, cloudStackVolumeName, volumeVO.getId()); + logger.debug("grantAccessIscsi: LUN [{}] mapped to igroup [{}] as logical unit number [{}], WWID [{}]", + cloudStackVolumeName, accessGroupName, lunNumber, lunWwid); + + // Update volume path if changed (e.g. the WWID was not yet known for a pre-existing volume) + String iscsiPath = buildIscsiPath(storagePool, lunWwid); + if (volumeVO.getPath() == null || !volumeVO.getPath().equals(iscsiPath)) { + volumeVO.set_iScsiName(iscsiPath); + volumeVO.setPath(iscsiPath); + } + } + + /** + * Maps the cached template LUN to the host so the KVM agent can write the image into it. + * + *

Called by the framework from {@code copyTemplateToManagedTemplateVolume} just before it + * issues the {@code CopyCommand}. That method reads {@code managedStoreTarget} from the pool + * details before this call, when the LUN number does not exist yet, so the stale + * value is corrected here. The datastore details are re-read when the command is built, so the + * update lands in time.

+ */ + private void grantAccessTemplate(TemplateInfo templateInfo, Host host, DataStore dataStore, StoragePoolVO storagePool) { + Map details = storagePoolDetailsDao.listDetailsKeyPairs(storagePool.getId()); + if (!ProtocolType.ISCSI.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { + logger.debug("grantAccessTemplate: NFS template [{}], no igroup mapping required", templateInfo.getUuid()); + return; + } + + String svmName = details.get(OntapStorageConstants.SVM_NAME); + VMTemplateStoragePoolVO templatePoolRef = findTemplatePoolRef(storagePool.getId(), templateInfo.getId()); + String lunName = getTemplateLunName(storagePool, templateInfo.getId()); + + UnifiedSANStrategy sanStrategy = (UnifiedSANStrategy) OntapStorageUtils.getStrategyByStoragePoolDetails(details); + String accessGroupName = OntapStorageUtils.getIgroupName(svmName, host.getUuid()); + + ensureAccessGroupForHost(sanStrategy, host, storagePool, svmName, accessGroupName); + + boolean lunMapped = false; + try { + String lunNumber = sanStrategy.ensureLunMapped(svmName, lunName, accessGroupName); + lunMapped = true; + // The template cache LUN has no volume_details row to cache the WWID in, and this runs + // once per template per pool, so it is fetched from the array each time. + String lunWwid = fetchLunWwid(sanStrategy, svmName, lunName); + logger.debug("grantAccessTemplate: template cache LUN [{}] mapped as logical unit number [{}], WWID [{}]", + lunName, lunNumber, lunWwid); + String iscsiPath = buildIscsiPath(storagePool, lunWwid); + + templatePoolRef.setInstallPath(iscsiPath); + vmTemplatePoolDao.update(templatePoolRef.getId(), templatePoolRef); + refreshManagedStoreTarget(dataStore, iscsiPath); + + logger.info("grantAccessTemplate: Mapped template cache LUN [{}] to igroup [{}] as [{}] for template [{}]", + lunName, accessGroupName, iscsiPath, templateInfo.getId()); + } catch (RuntimeException e) { + // Undo only this call's side effect (map). Do not delete the cache LUN — create already succeeded. + if (lunMapped) { + try { + unmapLunFromHost(sanStrategy, svmName, lunName, accessGroupName, host); + } catch (Exception rollbackEx) { + logger.warn("grantAccessTemplate: Failed to unmap template cache LUN [{}] from igroup [{}] after grant failure: {}", + lunName, accessGroupName, rollbackEx.getMessage()); + } + } + throw e; + } + } + + /** + * Ensures an igroup containing this host's initiator exists on the SVM. + * + *

The igroup may be absent even for a host that used the pool before, because LUN maps are + * created with {@code delete_on_unmap}, which lets ONTAP remove the igroup on its own.

+ */ + private void ensureAccessGroupForHost(UnifiedSANStrategy sanStrategy, Host host, StoragePoolVO storagePool, + String svmName, String accessGroupName) { Map getAccessGroupMap = Map.of( OntapStorageConstants.NAME, accessGroupName, OntapStorageConstants.SVM_DOT_NAME, svmName ); AccessGroup accessGroup = sanStrategy.getAccessGroup(getAccessGroupMap); - if(accessGroup == null || accessGroup.getIgroup() == null) { - logger.info("grantAccess: Igroup {} does not exist for the host {} : Need to create Igroup for the host ", accessGroupName, host.getName()); - // create the igroup for the host and perform lun-mapping + if (accessGroup == null || accessGroup.getIgroup() == null) { + logger.info("ensureAccessGroupForHost: Igroup {} does not exist for the host {} : Need to create Igroup for the host ", accessGroupName, host.getName()); accessGroup = new AccessGroup(); List hosts = new ArrayList<>(); hosts.add((HostVO) host); accessGroup.setHostsToConnect(hosts); accessGroup.setStoragePoolId(storagePool.getId()); accessGroup = sanStrategy.createAccessGroup(accessGroup); - }else{ - logger.info("grantAccess: Igroup {} already exist for the host {}: ", accessGroup.getIgroup().getName() , host.getName()); + } else { + logger.info("ensureAccessGroupForHost: Igroup {} already exist for the host {}: ", accessGroup.getIgroup().getName(), host.getName()); /* TODO Below cases will be covered later, for now they will be a pre-requisite on customer side 1. Igroup exist with the same name but host initiator has been removed 2. Igroup exist with the same name but host initiator has been changed may be due to new NIC or new adapter @@ -439,16 +1424,79 @@ private void grantAccessIscsi(Host host, VolumeVO volumeVO, Map Incase it is not , add it and proceed for lun-mapping */ } - logger.info("grantAccess: Igroup {} is present now with initiators {} ", accessGroup.getIgroup().getName(), accessGroup.getIgroup().getInitiators()); - // Create or retrieve existing LUN mapping - String lunNumber = sanStrategy.ensureLunMapped(svmName, cloudStackVolumeName, accessGroupName); + logger.info("ensureAccessGroupForHost: Igroup {} is present now with initiators {} ", accessGroup.getIgroup().getName(), accessGroup.getIgroup().getInitiators()); + } - // Update volume path if changed (e.g., after migration or re-mapping) - String iscsiPath = OntapStorageConstants.SLASH + storagePool.getPath() + OntapStorageConstants.SLASH + lunNumber; - if (volumeVO.getPath() == null || !volumeVO.getPath().equals(iscsiPath)) { - volumeVO.set_iScsiName(iscsiPath); - volumeVO.setPath(iscsiPath); + /** + * Builds the volume path the KVM agent expects for managed iSCSI: {@code //}. + */ + private String buildIscsiPath(StoragePoolVO storagePool, String lunWwid) { + return OntapStorageConstants.SLASH + storagePool.getPath() + OntapStorageConstants.SLASH + lunWwid; + } + + /** + * Resolves a volume's LUN WWID, caching it as a volume detail after the first lookup. + * + *

Resolved at grant time rather than at create time so that volumes created before this + * became the path format pick the WWID up on their next start.

+ */ + private String resolveVolumeLunWwid(UnifiedSANStrategy sanStrategy, String svmName, String lunName, long volumeId) { + VolumeDetailVO cached = volumeDetailsDao.findDetail(volumeId, DiskTO.SCSI_NAA_DEVICE_ID); + if (cached != null && cached.getValue() != null && !cached.getValue().isEmpty()) { + return cached.getValue(); + } + + String serialNumber = fetchLunSerialNumber(sanStrategy, svmName, lunName); + String wwid = toNaaWwid(serialNumber); + + volumeDetailsDao.addDetail(volumeId, OntapStorageConstants.LUN_DOT_SERIAL_NUMBER, serialNumber, false); + volumeDetailsDao.addDetail(volumeId, DiskTO.SCSI_NAA_DEVICE_ID, wwid, false); + return wwid; + } + + private String fetchLunWwid(UnifiedSANStrategy sanStrategy, String svmName, String lunName) { + return toNaaWwid(fetchLunSerialNumber(sanStrategy, svmName, lunName)); + } + + private String fetchLunSerialNumber(UnifiedSANStrategy sanStrategy, String svmName, String lunName) { + String serialNumber = sanStrategy.getLunSerialNumber(svmName, lunName); + if (serialNumber == null || serialNumber.isEmpty()) { + throw new CloudRuntimeException("ONTAP did not report a serial number for LUN " + lunName + + " on SVM " + svmName + "; cannot build a host-independent device path"); + } + return serialNumber; + } + + /** + * Derives a LUN's SCSI WWID from its ONTAP serial number. + * + *

NetApp builds the NAA IEEE Registered Extended identifier as the format designator ({@code 6}), + * its OUI ({@code 00a098}), a 4-bit pad nibble ({@code 0}), and then the ASCII bytes of the 12-char + * serial number in hex. For serial {@code x0M-8?/kEsJT} this yields + * {@code 600a098078304d2d383f2f6b45734a54}, which the host exposes as + * {@code /dev/disk/by-id/wwn-0x600a098078304d2d383f2f6b45734a54} and vSphere shows as + * {@code naa.600a098078304d2d383f2f6b45734a54}.

+ */ + private String toNaaWwid(String serialNumber) { + StringBuilder wwid = new StringBuilder(OntapStorageConstants.NETAPP_NAA_PREFIX); + for (byte b : serialNumber.getBytes(StandardCharsets.US_ASCII)) { + wwid.append(String.format("%02x", b)); + } + return wwid.toString(); + } + + private void refreshManagedStoreTarget(DataStore dataStore, String iscsiPath) { + if (!(dataStore instanceof PrimaryDataStore)) { + return; } + PrimaryDataStore primaryDataStore = (PrimaryDataStore) dataStore; + Map storeDetails = primaryDataStore.getDetails(); + if (storeDetails == null) { + return; + } + Map updated = new HashMap<>(storeDetails); + updated.put(PrimaryDataStore.MANAGED_STORE_TARGET, iscsiPath); + primaryDataStore.setDetails(updated); } /** @@ -485,6 +1533,8 @@ public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore) throw new CloudRuntimeException("CloudStack Volume not found for id: " + dataObject.getId()); } revokeAccessForVolume(storagePool, volumeVO, host); + } else if (dataObject.getType() == DataObjectType.TEMPLATE) { + revokeAccessForTemplate(storagePool, (TemplateInfo) dataObject, host); } else { logger.error("revokeAccess: Invalid DataObjectType (" + dataObject.getType() + ") passed to revokeAccess"); throw new CloudRuntimeException("Invalid DataObjectType (" + dataObject.getType() + ") passed to revokeAccess"); @@ -510,46 +1560,73 @@ private void revokeAccessForVolume(StoragePoolVO storagePool, VolumeVO volumeVO, // Retrieve LUN name from volume details; if missing, volume may not have been fully created VolumeDetailVO lunDetail = volumeDetailsDao.findDetail(volumeVO.getId(), OntapStorageConstants.LUN_DOT_NAME); - ValidateRevoke result = getValidateRevoke(volumeVO, host, lunDetail, storageStrategy, svmName, accessGroupName); - if (result == null) return; - - // Remove the LUN mapping from the igroup - Map disableLogicalAccessMap = new HashMap<>(); - disableLogicalAccessMap.put(OntapStorageConstants.LUN_DOT_UUID, result.cloudStackVolume.getLun().getUuid()); - disableLogicalAccessMap.put(OntapStorageConstants.IGROUP_DOT_UUID, result.accessGroup.getIgroup().getUuid()); - storageStrategy.disableLogicalAccess(disableLogicalAccessMap); + String lunName = lunDetail != null ? lunDetail.getValue() : null; + if (lunName == null) { + logger.warn("revokeAccessForVolume: No LUN name found for volume [{}]; skipping revoke", volumeVO.getId()); + return; + } + unmapLunFromHost(storageStrategy, svmName, lunName, accessGroupName, host); + } + } - logger.info("revokeAccessForVolume: Successfully revoked access to LUN [{}] for host [{}]", - result.lunName, host.getName()); + /** + * Unmaps the cached template LUN once the framework has finished writing the image into it. + */ + private void revokeAccessForTemplate(StoragePoolVO storagePool, TemplateInfo templateInfo, Host host) { + Map details = storagePoolDetailsDao.listDetailsKeyPairs(storagePool.getId()); + if (!ProtocolType.ISCSI.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL))) { + logger.debug("revokeAccessForTemplate: NFS template [{}], no igroup mapping to remove", templateInfo.getUuid()); + return; } + + String svmName = details.get(OntapStorageConstants.SVM_NAME); + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + String accessGroupName = OntapStorageUtils.getIgroupName(svmName, host.getUuid()); + String lunName = getTemplateLunName(storagePool, templateInfo.getId()); + + logger.info("revokeAccessForTemplate: Revoking access to template cache LUN [{}] for host [{}]", lunName, host.getName()); + unmapLunFromHost(storageStrategy, svmName, lunName, accessGroupName, host); } - @Nullable - private ValidateRevoke getValidateRevoke(VolumeVO volumeVO, Host host, VolumeDetailVO lunDetail, StorageStrategy storageStrategy, String svmName, String accessGroupName) { - String lunName = lunDetail != null ? lunDetail.getValue() : null; - if (lunName == null) { - logger.warn("revokeAccessForVolume: No LUN name found for volume [{}]; skipping revoke", volumeVO.getId()); - return null; + /** + * Removes a LUN-to-igroup mapping, skipping quietly when the LUN, the igroup or the host + * initiator is already gone. + */ + private void unmapLunFromHost(StorageStrategy storageStrategy, String svmName, String lunName, + String accessGroupName, Host host) { + ValidateRevoke result = getValidateRevoke(lunName, host, storageStrategy, svmName, accessGroupName); + if (result == null) { + return; } + Map disableLogicalAccessMap = new HashMap<>(); + disableLogicalAccessMap.put(OntapStorageConstants.LUN_DOT_UUID, result.cloudStackVolume.getLun().getUuid()); + disableLogicalAccessMap.put(OntapStorageConstants.IGROUP_DOT_UUID, result.accessGroup.getIgroup().getUuid()); + storageStrategy.disableLogicalAccess(disableLogicalAccessMap); + + logger.info("unmapLunFromHost: Successfully revoked access to LUN [{}] for host [{}]", result.lunName, host.getName()); + } + + @Nullable + private ValidateRevoke getValidateRevoke(String lunName, Host host, StorageStrategy storageStrategy, String svmName, String accessGroupName) { // Verify LUN still exists on ONTAP (may have been manually deleted) CloudStackVolume cloudStackVolume = getCloudStackVolumeByName(storageStrategy, svmName, lunName); if (cloudStackVolume == null || cloudStackVolume.getLun() == null || cloudStackVolume.getLun().getUuid() == null) { - logger.warn("revokeAccessForVolume: LUN for volume [{}] not found on ONTAP, skipping revoke", volumeVO.getId()); + logger.warn("getValidateRevoke: LUN [{}] not found on ONTAP, skipping revoke", lunName); return null; } // Verify igroup still exists on ONTAP AccessGroup accessGroup = getAccessGroupByName(storageStrategy, svmName, accessGroupName); if (accessGroup == null || accessGroup.getIgroup() == null || accessGroup.getIgroup().getUuid() == null) { - logger.warn("revokeAccessForVolume: iGroup [{}] not found on ONTAP, skipping revoke", accessGroupName); + logger.warn("getValidateRevoke: iGroup [{}] not found on ONTAP, skipping revoke", accessGroupName); return null; } // Verify host initiator is in the igroup before attempting to remove mapping SANStrategy sanStrategy = (UnifiedSANStrategy) storageStrategy; if (!sanStrategy.validateInitiatorInAccessGroup(host.getStorageUrl(), svmName, accessGroup.getIgroup())) { - logger.warn("revokeAccessForVolume: Initiator [{}] is not in iGroup [{}], skipping revoke", + logger.warn("getValidateRevoke: Initiator [{}] is not in iGroup [{}], skipping revoke", host.getStorageUrl(), accessGroupName); return null; } @@ -600,24 +1677,138 @@ private AccessGroup getAccessGroupByName(StorageStrategy storageStrategy, String return accessGroup; } + /** + * ONTAP is only supported with KVM, which does not take hypervisor-side snapshots into the + * volume itself, so no reserve is added on top of the requested size. + * + *

For a template this returns the virtual size ({@code VMTemplateVO.size}), not the + * compressed size on secondary storage. The cached template LUN is written by the KVM agent + * with {@code qemu-img convert} from QCOW2 to RAW, so it must be able to hold the fully + * expanded image.

+ */ @Override public long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataObject, StoragePool storagePool) { - return 0; + if (dataObject == null) { + return 0; + } + Long size = dataObject.getSize(); + return size != null && size > 0 ? size : 0; } @Override public long getBytesRequiredForTemplate(TemplateInfo templateInfo, StoragePool storagePool) { - return 0; + if (templateInfo == null || storagePool == null) { + return 0; + } + // template_spool_ref is inserted in Allocated before the cache exists; + // only skip reservation when the template is Ready and has a backend identity. + VMTemplateStoragePoolVO templatePoolRef = + vmTemplatePoolDao.findByPoolTemplate(storagePool.getId(), templateInfo.getId(), null); + Map details = storagePoolDetailsDao.listDetailsKeyPairs(storagePool.getId()); + if (isTemplateCachedOnPool(templatePoolRef, details)) { + return 0; + } + return getDataObjectSizeIncludingHypervisorSnapshotReserve(templateInfo, storagePool); + } + + /** + * Returns true when the primary template cache is present and usable for clone/deploy. + * A spool_ref row alone is not enough: CloudStack creates it in Allocated before the LUN/file exists. + * Ready is sufficient; downloadState is set alongside Ready on the managed-cache success path. + */ + private boolean isTemplateCachedOnPool(VMTemplateStoragePoolVO templatePoolRef, Map details) { + if (templatePoolRef == null) { + return false; + } + if (templatePoolRef.getState() != ObjectInDataStoreStateMachine.State.Ready) { + return false; + } + if (details != null && isIscsi(details)) { + return StringUtils.isNotBlank(templatePoolRef.getLocalDownloadPath()); + } + return StringUtils.isNotBlank(templatePoolRef.getInstallPath()); } + /** + * Returns the bytes used on the FlexVolume backing this pool, read directly from ONTAP + * ({@code space.used}). + * + *

Fails closed when ONTAP cannot provide trustworthy usage data. Returning zero for an + * unreachable or incomplete backend would make capacity checks treat an unknown pool as empty + * and could incorrectly authorize a volume grow.

+ * + * @throws InvalidParameterValueException if {@code storagePool} is null + * @throws CloudRuntimeException if the pool has no FlexVolume UUID in its details, ONTAP + * cannot be queried, or used-space data is missing + */ @Override public long getUsedBytes(StoragePool storagePool) { - return 0; + if (storagePool == null) { + throw new InvalidParameterValueException("storagePool is null, ensure the pool exists and is fully initialised before querying used bytes"); + } + + Map poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(storagePool.getId()); + String flexVolUuid = poolDetails != null ? poolDetails.get(OntapStorageConstants.VOLUME_UUID) : null; + + if (StringUtils.isBlank(flexVolUuid)) { + throw new CloudRuntimeException("FlexVolume UUID not found in pool details for pool " + storagePool.getId()); + } + + try { + StorageStrategy strategy = OntapStorageUtils.getStrategyByStoragePoolDetails(poolDetails); + Map queryParams = new HashMap<>(); + queryParams.put(OntapStorageConstants.FIELDS, OntapStorageConstants.SPACE_USED); + var flexVol = strategy.getStorageVolume(flexVolUuid, queryParams); + + if (flexVol == null) { + throw new CloudRuntimeException(String.format( + "FlexVolume [%s] backing pool [%s] was not found on ONTAP", + flexVolUuid, storagePool.getId())); + } + if (flexVol.getSpace() == null) { + throw new CloudRuntimeException(String.format( + "ONTAP returned no space information for FlexVolume [%s] backing pool [%s]", + flexVolUuid, storagePool.getId())); + } + + logger.debug("getUsedBytes: FlexVolume [{}] backing pool [{}] reports {} bytes used", + flexVolUuid, storagePool.getId(), flexVol.getSpace().getUsed()); + return flexVol.getSpace().getUsed(); + } catch (CloudRuntimeException e) { + logger.error("getUsedBytes: Failed to get used bytes for pool [{}]", storagePool.getId(), e); + throw e; + } catch (Exception e) { + logger.error("getUsedBytes: Failed to get used bytes for pool [{}]", storagePool.getId(), e); + throw new CloudRuntimeException( + String.format("Could not read used space for pool [%s]: %s", storagePool.getId(), e.getMessage()), e); + } } + /** + * Returns min IOPS reserved on the pool, including volumes still being created. + * Destroyed and expunged volumes are omitted by the DAO query. + */ @Override public long getUsedIops(StoragePool storagePool) { - return 0; + return getAllocatedMinIops(storagePool, null); + } + + private long getAllocatedMinIops(StoragePool storagePool, Long excludeVolumeId) { + long usedIops = 0; + + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); + if (volumes != null) { + for (VolumeVO volume : volumes) { + if (excludeVolumeId != null && excludeVolumeId.equals(volume.getId())) { + continue; + } + if (volume.getMinIops() != null) { + usedIops += volume.getMinIops(); + } + } + } + + return usedIops; } /** @@ -647,7 +1838,7 @@ public void takeSnapshot(SnapshotInfo snapshot, AsyncCompletionCallback details) { + return ProtocolType.ISCSI.name().equalsIgnoreCase(details.get(OntapStorageConstants.PROTOCOL)); + } + + /** + * Resize request. Size is passed separately to {@code resizeCloudStackVolume}. + * When QoS changes, iSCSI carries the LUN uuid and policy, and NFS carries the FlexVol uuid and file policy. + */ + private CloudStackVolume resizeCloudStackVolumeRequest(Map details, VolumeInfo volumeInfo, + VolumeQosPolicy qosPolicy) { + CloudStackVolume resizeRequest = new CloudStackVolume(); + resizeRequest.setVolumeInfo(volumeInfo); + if (qosPolicy == null) { + return resizeRequest; + } + VolumeQosPolicy qosPolicyReference = new VolumeQosPolicy(); + qosPolicyReference.setName(qosPolicy.getName()); + qosPolicyReference.setUuid(qosPolicy.getUuid()); + + String protocol = details.get(OntapStorageConstants.PROTOCOL); + ProtocolType protocolType = ProtocolType.valueOf(protocol); + switch (protocolType) { + case NFS3: + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicyReference); + resizeRequest.setFile(fileInfo); + resizeRequest.setFlexVolumeUuid(details.get(OntapStorageConstants.VOLUME_UUID)); + return resizeRequest; + case ISCSI: + VolumeDetailVO lunUuid = volumeDetailsDao.findDetail(volumeInfo.getId(), OntapStorageConstants.LUN_DOT_UUID); + if (lunUuid == null || lunUuid.getValue() == null) { + throw new CloudRuntimeException("LUN UUID is missing for volume " + volumeInfo.getId()); + } + Lun lun = new Lun(); + lun.setUuid(lunUuid.getValue()); + lun.setQosPolicy(qosPolicyReference); + resizeRequest.setLun(lun); + return resizeRequest; + default: + throw new CloudRuntimeException("Unsupported protocol " + protocol); + } + } + + /** + * Builds the request that creates or updates a volume (LUN for iSCSI, qcow2 file for NFS), + * attaching a QoS policy reference when one is provided. + */ + private CloudStackVolume createCloudStackVolumeRequestByProtocol(StoragePoolVO storagePool, Map details, + DataObject volumeObject, VolumeQosPolicy qosPolicy) { + VolumeQosPolicy qosPolicyReference = null; + if (qosPolicy != null) { + qosPolicyReference = new VolumeQosPolicy(); + qosPolicyReference.setName(qosPolicy.getName()); + qosPolicyReference.setUuid(qosPolicy.getUuid()); + } + + String protocol = details.get(OntapStorageConstants.PROTOCOL); + ProtocolType protocolType = ProtocolType.valueOf(protocol); + switch (protocolType) { + case NFS3: + CloudStackVolume nfsRequest = new CloudStackVolume(); + nfsRequest.setDatastoreId(String.valueOf(storagePool.getId())); + nfsRequest.setFlexVolumeUuid(details.get(OntapStorageConstants.VOLUME_UUID)); + nfsRequest.setVolumeInfo(volumeObject); + if (qosPolicyReference != null) { + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicyReference); + nfsRequest.setFile(fileInfo); + } + return nfsRequest; + case ISCSI: + Svm svm = new Svm(); + svm.setName(details.get(OntapStorageConstants.SVM_NAME)); + CloudStackVolume iscsiRequest = new CloudStackVolume(); + Lun lunRequest = new Lun(); + lunRequest.setSvm(svm); + + LunSpace lunSpace = new LunSpace(); + lunSpace.setSize(volumeObject.getSize()); + lunRequest.setSpace(lunSpace); + String lunName = volumeObject.getName().replace(OntapStorageConstants.HYPHEN, OntapStorageConstants.UNDERSCORE); + if (!OntapStorageUtils.isValidName(lunName)) { + throw new InvalidParameterValueException("Invalid dataObject name [" + lunName + + "]. It must start with a letter and can only contain letters, digits, and underscores, and be up to 200 characters long."); + } + lunRequest.setName(OntapStorageUtils.getLunName(storagePool.getName(), lunName)); + lunRequest.setOsType(Lun.OsTypeEnum.valueOf( + OntapStorageUtils.getOSTypeFromHypervisor(storagePool.getHypervisor().name()))); + lunRequest.setQosPolicy(qosPolicyReference); + iscsiRequest.setLun(lunRequest); + return iscsiRequest; + default: + throw new CloudRuntimeException("Unsupported protocol " + protocol); + } + } + + /** + * LUN path used to cache a template on this pool: {@code /vol//cs_tmpl_}. + */ + private String getTemplateLunName(StoragePoolVO storagePool, long templateId) { + return OntapStorageUtils.getLunName(storagePool.getName(), OntapStorageConstants.TEMPLATE_LUN_PREFIX + templateId); + } + + /** + * Builds the request that clones the cached template LUN into a new volume LUN. + * + *

Source identity mirrors the NFS file-clone path workflow: {@code clone.source.name} is + * the same deterministic ONTAP path used at template create + * ({@code /vol//cs_tmpl_}). {@code local_download_path} (LUN uuid) is + * still sent as a secondary identity.

+ * + *

Size is omitted: ONTAP rejects a size on a clone create, and the clone inherits the + * source size. Growing to the requested volume size is a separate PATCH.

+ */ + private CloudStackVolume createCloneLunRequest(StoragePoolVO storagePool, Map details, + VolumeInfo volumeObject, VMTemplateStoragePoolVO templatePoolRef, + long templateId, VolumeQosPolicy qosPolicy) { + String sourceLunUuid = templatePoolRef.getLocalDownloadPath(); + if (sourceLunUuid == null || sourceLunUuid.isEmpty()) { + throw new CloudRuntimeException("Template [" + templateId + "] has no cached LUN on pool [" + + storagePool.getId() + "]; cannot clone volume [" + volumeObject.getId() + "]"); + } + + Svm svm = new Svm(); + svm.setName(details.get(OntapStorageConstants.SVM_NAME)); + + String lunName = volumeObject.getName().replace(OntapStorageConstants.HYPHEN, OntapStorageConstants.UNDERSCORE); + if (!OntapStorageUtils.isValidName(lunName)) { + throw new InvalidParameterValueException("Invalid dataObject name [" + lunName + + "]. It must start with a letter and can only contain letters, digits, and underscores, and be up to 200 characters long."); + } + + Lun.Source source = new Lun.Source(); + source.setName(getTemplateLunName(storagePool, templateId)); + source.setUuid(sourceLunUuid); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); + + Lun lunRequest = new Lun(); + lunRequest.setSvm(svm); + lunRequest.setName(OntapStorageUtils.getLunName(storagePool.getName(), lunName)); + lunRequest.setClone(clone); + if (qosPolicy != null) { + VolumeQosPolicy qosPolicyReference = new VolumeQosPolicy(); + qosPolicyReference.setName(qosPolicy.getName()); + qosPolicyReference.setUuid(qosPolicy.getUuid()); + lunRequest.setQosPolicy(qosPolicyReference); + } + + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lunRequest); + request.setVolumeInfo(volumeObject); + return request; + } + + /** + * Builds the request that clones the cached qcow2 ({@code install_path}) into a new file + * named after the volume uuid, inside the same FlexVolume. + */ + private CloudStackVolume createCloneFileRequest(StoragePoolVO storagePool, VolumeInfo volumeInfo, + VMTemplateStoragePoolVO templatePoolRef, long templateId) { + String sourcePath = templatePoolRef.getInstallPath(); + if (sourcePath == null || sourcePath.isEmpty()) { + throw new CloudRuntimeException("Template [" + templateId + "] has no cached file on pool [" + + storagePool.getId() + "]; cannot clone volume [" + volumeInfo.getId() + "]"); + } + + FileInfo file = new FileInfo(); + file.setPath(sourcePath); + + CloudStackVolume request = new CloudStackVolume(); + request.setDatastoreId(String.valueOf(storagePool.getId())); + request.setVolumeInfo(volumeInfo); + request.setFile(file); + request.setDestinationPath(volumeInfo.getUuid()); + return request; + } + + /** + * Builds the request that clones a LUN out of a FlexVolume snapshot into a new volume LUN. + * + *

{@code clone.source.name} is {@code /vol//.snapshot//}; ONTAP cannot + * resolve a snapshot-resident LUN by uuid, so no uuid is sent.

+ */ + private CloudStackVolume createCloneLunFromSnapshotRequest(StoragePoolVO storagePool, Map details, + VolumeInfo volumeInfo, String sourceVolumePath, + String snapshotName, VolumeQosPolicy qosPolicy) { + String lunName = volumeInfo.getName().replace(OntapStorageConstants.HYPHEN, OntapStorageConstants.UNDERSCORE); + if (!OntapStorageUtils.isValidName(lunName)) { + throw new CloudRuntimeException("Invalid dataObject name [" + lunName + + "]. It must start with a letter and can only contain letters, digits, and underscores, and be up to 200 characters long."); + } + + String flexVolName = details.get(OntapStorageConstants.VOLUME_NAME); + if (flexVolName == null || flexVolName.isEmpty()) { + flexVolName = storagePool.getName(); + } + + Svm svm = new Svm(); + svm.setName(details.get(OntapStorageConstants.SVM_NAME)); + + Lun.Source source = new Lun.Source(); + source.setName(OntapStorageUtils.toLunCloneSourcePathInSnapshot(sourceVolumePath, flexVolName, snapshotName)); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); + + Lun lunRequest = new Lun(); + lunRequest.setSvm(svm); + lunRequest.setName(OntapStorageUtils.getLunName(storagePool.getName(), lunName)); + lunRequest.setClone(clone); + if (qosPolicy != null) { + VolumeQosPolicy qosPolicyReference = new VolumeQosPolicy(); + qosPolicyReference.setName(qosPolicy.getName()); + qosPolicyReference.setUuid(qosPolicy.getUuid()); + lunRequest.setQosPolicy(qosPolicyReference); + } + + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lunRequest); + request.setVolumeInfo(volumeInfo); + return request; + } + + /** + * Builds the request that clones a file out of a FlexVolume snapshot into a new file named after + * the volume uuid, inside the same FlexVolume. + */ + private CloudStackVolume createCloneFileFromSnapshotRequest(StoragePoolVO storagePool, Map details, + VolumeInfo volumeInfo, String sourceVolumePath, + String snapshotName) { + String flexVolName = details.get(OntapStorageConstants.VOLUME_NAME); + + FileInfo file = new FileInfo(); + // FlexVol-relative source path, for example "". + String relativeSourcePath = OntapStorageUtils.toFlexVolRelativePath(sourceVolumePath, flexVolName); + // Path assigned to file.setPath(), for example "./.snapshot//". + file.setPath(OntapStorageConstants.DOT + OntapStorageConstants.SNAPSHOT_PATH_SEGMENT + snapshotName + + OntapStorageConstants.SLASH + relativeSourcePath); + + CloudStackVolume request = new CloudStackVolume(); + request.setDatastoreId(String.valueOf(storagePool.getId())); + request.setVolumeInfo(volumeInfo); + request.setFile(file); + request.setDestinationPath(OntapStorageUtils.toFlexVolRelativePath(volumeInfo.getUuid(), flexVolName)); + return request; + } + // ────────────────────────────────────────────────────────────────────────── // Snapshot Helper Methods // ────────────────────────────────────────────────────────────────────────── /** - * Builds a snapshot name with proper length constraints. - * Format: {@code -} + * Builds an ONTAP-safe snapshot name from the CloudStack UI name with uniqueness suffix. + */ + private String buildSnapshotName(String cloudStackSnapshotName, long snapshotId) { + return OntapStorageUtils.buildOntapSnapshotName(cloudStackSnapshotName, OntapStorageConstants.CS + snapshotId); + } + + /** + * Only ISCSI and NFS3 pools can be provisioned; any other protocol would fall through + * createAsync without producing a result, leaving the caller with a null callback value. */ - private String buildSnapshotName(String volumeName, String snapshotUuid) { - String name = volumeName + "-" + snapshotUuid; - int maxLength = OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH; - int trimRequired = name.length() - maxLength; + private void validateProtocol(Map details, DataStore dataStore) { + String protocol = details == null ? null : details.get(OntapStorageConstants.PROTOCOL); + boolean supported = protocol != null && Arrays.stream(ProtocolType.values()) + .anyMatch(type -> type.name().equalsIgnoreCase(protocol)); + if (!supported) { + throw new CloudRuntimeException("Unsupported protocol [" + protocol + "] on storage pool [" + + dataStore.getName() + "]; supported protocols are " + Arrays.toString(ProtocolType.values())); + } + } - if (trimRequired > 0) { - name = StringUtils.left(volumeName, volumeName.length() - trimRequired) + "-" + snapshotUuid; + private Storage.ImageFormat getImageFormat(StoragePoolVO storagePool) { + HypervisorType hypervisorType = storagePool.getHypervisor(); + if (!HypervisorType.KVM.equals(hypervisorType)) { + throw new CloudRuntimeException("Unsupported hypervisor [" + hypervisorType + "] for ONTAP image format resolution"); + } + Storage.StoragePoolType spType = storagePool.getPoolType(); + switch (spType) { + case OntapiSCSI: + return Storage.ImageFormat.RAW; + case NetworkFilesystem: + return Storage.ImageFormat.QCOW2; + default: + throw new CloudRuntimeException("Unsupported pool type [" + spType + "] for ONTAP image format resolution"); } - return name; } /** - * Persists snapshot metadata in snapshot_details table. + * Persists ONTAP snapshot metadata in {@code snapshot_details} for revert, delete, and + * create-volume-from-snapshot. * - * @param csSnapshotId CloudStack snapshot ID - * @param csVolumeId Source CloudStack volume ID - * @param flexVolUuid ONTAP FlexVolume UUID - * @param ontapSnapshotUuid ONTAP FlexVolume snapshot UUID - * @param snapshotName ONTAP snapshot name - * @param volumePath Path of the volume file/LUN within the FlexVolume (for restore) - * @param storagePoolId Primary storage pool ID - * @param protocol Storage protocol (NFS3 or ISCSI) - * @param lunUuid LUN UUID (only for iSCSI, null for NFS) + *

Volume-snapshot delete reads {@code base_ontap_fv_id} and {@code ontap_snap_id} here + * during {@link #deleteCloudStackVolumeSnapshot}; missing rows prevent ONTAP cleanup.

+ * + *

All rows go through {@link #persistSnapshotDetail} so DAO writes stay consistent; + * optional fields (LUN uuid, IOPS) are skipped when unset.

+ * + * @param csSnapshotId CloudStack snapshot ID + * @param details {@code snapshot_details} name to value, keyed by {@link OntapStorageConstants}; + * entries with null or blank values are skipped + */ + private void updateSnapshotDetails(long csSnapshotId, Map details) { + for (Map.Entry detail : details.entrySet()) { + persistSnapshotDetail(csSnapshotId, detail.getKey(), detail.getValue()); + } + } + + /** + * Adds an IOPS entry to {@code details} only when {@code iops} is non-null and positive. + */ + private static void putPositiveIopsDetail(Map details, String detailName, Long iops) { + if (iops != null && iops > 0) { + details.put(detailName, String.valueOf(iops)); + } + } + + /** + * Persists one {@code snapshot_details} row. No-op when {@code value} is null or blank so + * optional keys (e.g. LUN uuid on NFS) share the same DAO path as required keys. */ - private void updateSnapshotDetails(long csSnapshotId, long csVolumeId, String flexVolUuid, - String ontapSnapshotUuid, String snapshotName, - String volumePath, long storagePoolId, String protocol, - String lunUuid) { - SnapshotDetailsVO snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.SRC_CS_VOLUME_ID, String.valueOf(csVolumeId), false); - snapshotDetailsDao.persist(snapshotDetail); - - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.BASE_ONTAP_FV_ID, flexVolUuid, false); - snapshotDetailsDao.persist(snapshotDetail); - - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.ONTAP_SNAP_ID, ontapSnapshotUuid, false); - snapshotDetailsDao.persist(snapshotDetail); - - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.ONTAP_SNAP_NAME, snapshotName, false); - snapshotDetailsDao.persist(snapshotDetail); - - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.VOLUME_PATH, volumePath, false); - snapshotDetailsDao.persist(snapshotDetail); - - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.PRIMARY_POOL_ID, String.valueOf(storagePoolId), false); - snapshotDetailsDao.persist(snapshotDetail); - - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.PROTOCOL, protocol, false); - snapshotDetailsDao.persist(snapshotDetail); - - // Store LUN UUID for iSCSI volumes (required for LUN restore API) - if (lunUuid != null && !lunUuid.isEmpty()) { - snapshotDetail = new SnapshotDetailsVO(csSnapshotId, - OntapStorageConstants.LUN_DOT_UUID, lunUuid, false); - snapshotDetailsDao.persist(snapshotDetail); + private void persistSnapshotDetail(long csSnapshotId, String detailName, String value) { + if (value == null || value.isEmpty()) { + return; } + snapshotDetailsDao.persist(new SnapshotDetailsVO(csSnapshotId, detailName, value, false)); } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/AggregateFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/AggregateFeignClient.java index f756c3d32f18..7e026b0a6b19 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/AggregateFeignClient.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/AggregateFeignClient.java @@ -19,10 +19,14 @@ package org.apache.cloudstack.storage.feign.client; +import java.util.Map; + import org.apache.cloudstack.storage.feign.model.Aggregate; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; + import feign.Headers; import feign.Param; +import feign.QueryMap; import feign.RequestLine; public interface AggregateFeignClient { @@ -33,5 +37,6 @@ public interface AggregateFeignClient { @RequestLine("GET /api/storage/aggregates/{uuid}") @Headers({"Authorization: {authHeader}"}) - Aggregate getAggregateByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid); + Aggregate getAggregateByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid, + @QueryMap Map queryParams); } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/ClusterFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/ClusterFeignClient.java index 582fb58e6f3b..476ef2e392fc 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/ClusterFeignClient.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/ClusterFeignClient.java @@ -19,14 +19,24 @@ package org.apache.cloudstack.storage.feign.client; +import feign.QueryMap; import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.ClusterNode; +import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import feign.Headers; import feign.Param; import feign.RequestLine; +import java.util.Map; + public interface ClusterFeignClient { @RequestLine("GET /api/cluster") @Headers({"Authorization: {authHeader}", "return_records: {returnRecords}"}) Cluster getCluster(@Param("authHeader") String authHeader, @Param("returnRecords") boolean returnRecords); + + @RequestLine("GET /api/cluster/nodes") + @Headers({"Authorization: {authHeader}"}) + OntapResponse getClusterNodes(@Param("authHeader") String authHeader, + @QueryMap Map queryMap); } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/EmsFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/EmsFeignClient.java new file mode 100644 index 000000000000..c3e4c73d6165 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/EmsFeignClient.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.feign.client; + +import feign.Headers; +import feign.Param; +import feign.RequestLine; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; + +public interface EmsFeignClient { + + @RequestLine("POST /api/support/ems/application-logs") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + void sendEmsApplicationLog(@Param("authHeader") String authHeader, EmsApplicationLog emsApplicationLog); +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/NASFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/NASFeignClient.java index 8cf21b94b2f1..ba38f57ef1c7 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/NASFeignClient.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/NASFeignClient.java @@ -21,7 +21,9 @@ import feign.QueryMap; import org.apache.cloudstack.storage.feign.model.ExportPolicy; +import org.apache.cloudstack.storage.feign.model.FileCloneRequest; import org.apache.cloudstack.storage.feign.model.FileInfo; +import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import feign.Headers; import feign.Param; @@ -58,6 +60,15 @@ void createFile(@Param("authHeader") String authHeader, @Param("path") String filePath, FileInfo file); + /** + * Creates a space-efficient clone of a file within a FlexVolume. + * + *

ONTAP REST: {@code POST /api/storage/file/clone}

+ */ + @RequestLine("POST /api/storage/file/clone") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + JobResponse cloneFile(@Param("authHeader") String authHeader, FileCloneRequest request); + // Export Policy Operations @RequestLine("POST /api/protocols/nfs/export-policies") @Headers({"Authorization: {authHeader}"}) diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/QosFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/QosFeignClient.java new file mode 100644 index 000000000000..669ea935124d --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/QosFeignClient.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.feign.client; + +import java.util.Map; + +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; +import org.apache.cloudstack.storage.feign.model.response.JobResponse; +import org.apache.cloudstack.storage.feign.model.response.OntapResponse; + +import feign.Headers; +import feign.Param; +import feign.QueryMap; +import feign.RequestLine; + +public interface QosFeignClient { + + @RequestLine("POST /api/storage/qos/policies") + @Headers({"Authorization: {authHeader}"}) + JobResponse createPolicy(@Param("authHeader") String authHeader, VolumeQosPolicy policy); + + @RequestLine("GET /api/storage/qos/policies") + @Headers({"Authorization: {authHeader}"}) + OntapResponse getPolicies(@Param("authHeader") String authHeader, + @QueryMap Map queryParams); + + @RequestLine("GET /api/storage/qos/policies/{uuid}") + @Headers({"Authorization: {authHeader}"}) + VolumeQosPolicy getPolicy(@Param("authHeader") String authHeader, @Param("uuid") String uuid, + @QueryMap Map queryParams); + + @RequestLine("DELETE /api/storage/qos/policies/{uuid}") + @Headers({"Authorization: {authHeader}"}) + JobResponse deletePolicy(@Param("authHeader") String authHeader, @Param("uuid") String uuid); +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SANFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SANFeignClient.java index 7281dc2ecbeb..8a60a9a3e2bb 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SANFeignClient.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SANFeignClient.java @@ -50,13 +50,13 @@ public interface SANFeignClient { @Headers({"Authorization: {authHeader}"}) Lun getLunByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid); - @RequestLine("PATCH /{uuid}") - @Headers({"Authorization: {authHeader}"}) - void updateLun(@Param("authHeader") String authHeader, @Param("uuid") String uuid, Lun lun); + @RequestLine("PATCH /api/storage/luns/{uuid}") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + JobResponse updateLun(@Param("authHeader") String authHeader, @Param("uuid") String uuid, Lun lun); @RequestLine("DELETE /api/storage/luns/{uuid}") @Headers({"Authorization: {authHeader}"}) - void deleteLun(@Param("authHeader") String authHeader, @Param("uuid") String uuid, @QueryMap Map queryMap); + JobResponse deleteLun(@Param("authHeader") String authHeader, @Param("uuid") String uuid, @QueryMap Map queryMap); // iGroup Operation APIs @RequestLine("POST /api/protocols/san/igroups?return_records={returnRecords}") diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SnapshotFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SnapshotFeignClient.java index 2f0e050d6f55..cb7375aead88 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SnapshotFeignClient.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SnapshotFeignClient.java @@ -23,6 +23,8 @@ import feign.QueryMap; import feign.RequestLine; import org.apache.cloudstack.storage.feign.model.CliSnapshotRestoreRequest; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroup; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroupSnapshot; import org.apache.cloudstack.storage.feign.model.FlexVolSnapshot; import org.apache.cloudstack.storage.feign.model.SnapshotFileRestoreRequest; import org.apache.cloudstack.storage.feign.model.response.JobResponse; @@ -181,4 +183,96 @@ JobResponse restoreFileFromSnapshot(@Param("authHeader") String authHeader, @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) JobResponse restoreFileFromSnapshotCli(@Param("authHeader") String authHeader, CliSnapshotRestoreRequest request); + + /** + * Creates a consistency group. + * + *

ONTAP REST: {@code POST /api/application/consistency-groups}

+ * + * @param authHeader Basic auth header + * @param request consistency group create request body + * @return JobResponse containing the async job reference + */ + @RequestLine("POST /api/application/consistency-groups") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + JobResponse createConsistencyGroup(@Param("authHeader") String authHeader, + ConsistencyGroup request); + + /** + * Lists consistency groups. + * + *

ONTAP REST: {@code GET /api/application/consistency-groups}

+ * + * @param authHeader Basic auth header + * @param queryParams Optional query parameters + * @return Paginated consistency group records + */ + @RequestLine("GET /api/application/consistency-groups") + @Headers({"Authorization: {authHeader}"}) + OntapResponse getConsistencyGroups(@Param("authHeader") String authHeader, + @QueryMap Map queryParams); + + /** + * Creates (starts) a consistency group snapshot. + * + *

ONTAP REST: {@code POST /api/application/consistency-groups/{cgUuid}/snapshots}

+ * + * @param authHeader Basic auth header + * @param cgUuid consistency group UUID + * @param request snapshot start request body + * @return JobResponse containing the async job reference + */ + @RequestLine("POST /api/application/consistency-groups/{cgUuid}/snapshots") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + JobResponse createConsistencyGroupSnapshot(@Param("authHeader") String authHeader, + @Param("cgUuid") String cgUuid, + ConsistencyGroupSnapshot request); + + /** + * Lists snapshots for a consistency group. + * + *

ONTAP REST: {@code GET /api/application/consistency-groups/{cgUuid}/snapshots}

+ * + * @param authHeader Basic auth header + * @param cgUuid consistency group UUID + * @param queryParams Optional query parameters + * @return Paginated consistency group snapshot records + */ + @RequestLine("GET /api/application/consistency-groups/{cgUuid}/snapshots") + @Headers({"Authorization: {authHeader}"}) + OntapResponse getConsistencyGroupSnapshots(@Param("authHeader") String authHeader, + @Param("cgUuid") String cgUuid, + @QueryMap Map queryParams); + + /** + * Commits a started consistency group snapshot. + * + *

ONTAP REST: {@code PATCH /api/application/consistency-groups/{cgUuid}/snapshots/{snapshotUuid}}

+ * + * @param authHeader Basic auth header + * @param cgUuid consistency group UUID + * @param snapshotUuid consistency group snapshot UUID + * @param request commit request body + * @return JobResponse containing the async job reference + */ + @RequestLine("PATCH /api/application/consistency-groups/{cgUuid}/snapshots/{snapshotUuid}") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + JobResponse commitConsistencyGroupSnapshot(@Param("authHeader") String authHeader, + @Param("cgUuid") String cgUuid, + @Param("snapshotUuid") String snapshotUuid, + ConsistencyGroupSnapshot request); + + /** + * Deletes a consistency group. + * + *

ONTAP REST: {@code DELETE /api/application/consistency-groups/{cgUuid}}

+ * + * @param authHeader Basic auth header + * @param cgUuid consistency group UUID + * @return JobResponse containing the async job reference + */ + @RequestLine("DELETE /api/application/consistency-groups/{cgUuid}") + @Headers({"Authorization: {authHeader}"}) + JobResponse deleteConsistencyGroup(@Param("authHeader") String authHeader, + @Param("cgUuid") String cgUuid); } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/VolumeFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/VolumeFeignClient.java index 6384566487d4..d272ee6df13c 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/VolumeFeignClient.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/VolumeFeignClient.java @@ -44,7 +44,7 @@ public interface VolumeFeignClient { @RequestLine("GET /api/storage/volumes/{uuid}") @Headers({"Authorization: {authHeader}"}) - Volume getVolumeByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid); + Volume getVolumeByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid, @QueryMap Map queryParams); @RequestLine("GET /api/storage/volumes") @Headers({"Authorization: {authHeader}"}) @@ -52,5 +52,5 @@ public interface VolumeFeignClient { @RequestLine("PATCH /api/storage/volumes/{uuid}") @Headers({ "Authorization: {authHeader}"}) - JobResponse updateVolumeRebalancing(@Param("authHeader") String authHeader, @Param("uuid") String uuid, Volume volumeRequest); + JobResponse updateVolume(@Param("authHeader") String authHeader, @Param("uuid") String uuid, Volume volumeRequest); } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Aggregate.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Aggregate.java index 8ac1717604a5..7b57be59ec25 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Aggregate.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Aggregate.java @@ -19,14 +19,14 @@ package org.apache.cloudstack.storage.feign.model; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; -import java.util.Objects; - @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class Aggregate { @@ -77,6 +77,17 @@ public int hashCode() { @JsonProperty("space") private AggregateSpace space = null; + @JsonProperty("node") + private Node node = null; + + + public Node getNode() { + return node; + } + + public void setNode(Node node) { + this.node = node; + } public Aggregate name(String name) { this.name = name; @@ -107,10 +118,18 @@ public StateEnum getState() { return state; } + public void setState(StateEnum state) { + this.state = state; + } + public AggregateSpace getSpace() { return space; } + public void setSpace(AggregateSpace space) { + this.space = space; + } + public Double getAvailableBlockStorageSpace() { if (space != null && space.blockStorage != null) { return space.blockStorage.available; @@ -148,9 +167,32 @@ public String toString() { return "DiskAggregates [name=" + name + ", uuid=" + uuid + "]"; } + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class Node { + @JsonProperty("name") + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + public static class AggregateSpace { @JsonProperty("block_storage") private AggregateSpaceBlockStorage blockStorage = null; + + public AggregateSpaceBlockStorage getBlockStorage() { + return blockStorage; + } + + public void setBlockStorage(AggregateSpaceBlockStorage blockStorage) { + this.blockStorage = blockStorage; + } } public static class AggregateSpaceBlockStorage { @@ -160,6 +202,14 @@ public static class AggregateSpaceBlockStorage { private Double size = null; @JsonProperty("used") private Double used = null; + + public Double getAvailable() { + return available; + } + + public void setAvailable(Double available) { + this.available = available; + } } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Cluster.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Cluster.java index 061372756175..f0b870f25423 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Cluster.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Cluster.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.feign.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -48,6 +49,12 @@ public class Cluster { @JsonProperty("disaggregated") private Boolean disaggregated = null; + @JsonIgnore + private String model; + + @JsonIgnore + private String platformType; + public String getHealth() { return health; @@ -104,6 +111,22 @@ public void setDisaggregated(Boolean disaggregated) { this.disaggregated = disaggregated; } + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public String getPlatformType() { + return platformType; + } + + public void setPlatformType(String platformType) { + this.platformType = platformType; + } + @Override public int hashCode() { return Objects.hash(getName(), getUuid()); @@ -129,6 +152,8 @@ public String toString() { ", version=" + version + ", sanOptimized=" + sanOptimized + ", disaggregated=" + disaggregated + + ", model='" + model + '\'' + + ", platformType='" + platformType + '\'' + '}'; } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ClusterNode.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ClusterNode.java new file mode 100644 index 000000000000..8d067c05790c --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ClusterNode.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Subset of {@code GET /api/cluster/nodes} used for ASUP + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ClusterNode { + + @JsonProperty("model") + private String model; + + @JsonProperty("is_all_flash_optimized") + private Boolean allFlashOptimized; + + @JsonProperty("is_perf_optimized") + private Boolean performanceOptimized; + + @JsonProperty("is_capacity_optimized") + private Boolean capacityOptimized; + + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public Boolean getAllFlashOptimized() { + return allFlashOptimized; + } + + public void setAllFlashOptimized(Boolean allFlashOptimized) { + this.allFlashOptimized = allFlashOptimized; + } + + public Boolean getPerformanceOptimized() { + return performanceOptimized; + } + + public void setPerformanceOptimized(Boolean performanceOptimized) { + this.performanceOptimized = performanceOptimized; + } + + public Boolean getCapacityOptimized() { + return capacityOptimized; + } + + public void setCapacityOptimized(Boolean capacityOptimized) { + this.capacityOptimized = capacityOptimized; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroup.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroup.java new file mode 100644 index 000000000000..2c32a04d6b65 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroup.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * Model representing an ONTAP application consistency group. + * + *

Maps to the ONTAP REST API resource at + * {@code /api/application/consistency-groups}.

+ * + * @see + * ONTAP REST API - Create Consistency Group + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ConsistencyGroup { + + @JsonProperty("uuid") + private String uuid; + + @JsonProperty("name") + private String name; + + @JsonProperty("svm") + private Svm svm; + + @JsonProperty("volumes") + private List volumes; + + public ConsistencyGroup() { + } + + public ConsistencyGroup(String name) { + this.name = name; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Svm getSvm() { + return svm; + } + + public void setSvm(Svm svm) { + this.svm = svm; + } + + public List getVolumes() { + return volumes; + } + + public void setVolumes(List volumes) { + this.volumes = volumes; + } + + @Override + public String toString() { + return "ConsistencyGroup{" + + "uuid='" + uuid + '\'' + + ", name='" + name + '\'' + + ", volumes=" + (volumes != null ? volumes.size() : 0) + + '}'; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupSnapshot.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupSnapshot.java new file mode 100644 index 000000000000..974745f02d4d --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupSnapshot.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Model representing an ONTAP consistency group snapshot. + * + *

Maps to the ONTAP REST API resource at + * {@code /api/application/consistency-groups/{consistency_group.uuid}/snapshots}.

+ * + * @see + * ONTAP REST API - Consistency Group Snapshots + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ConsistencyGroupSnapshot { + + @JsonProperty("uuid") + private String uuid; + + @JsonProperty("name") + private String name; + + @JsonProperty("create_time") + private String createTime; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("consistency_type") + private String consistencyType; + + @JsonProperty("snapmirror_label") + private String snapmirrorLabel; + + @JsonProperty("action") + private String action; + + @JsonProperty("consistency_group") + private VolumeConcise consistencyGroup; + + public ConsistencyGroupSnapshot() { + // default constructor for Jackson + } + + public ConsistencyGroupSnapshot(String name) { + this.name = name; + } + + public ConsistencyGroupSnapshot(String name, String action) { + this.name = name; + this.action = action; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getConsistencyType() { + return consistencyType; + } + + public void setConsistencyType(String consistencyType) { + this.consistencyType = consistencyType; + } + + public String getSnapmirrorLabel() { + return snapmirrorLabel; + } + + public void setSnapmirrorLabel(String snapmirrorLabel) { + this.snapmirrorLabel = snapmirrorLabel; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public VolumeConcise getConsistencyGroup() { + return consistencyGroup; + } + + public void setConsistencyGroup(VolumeConcise consistencyGroup) { + this.consistencyGroup = consistencyGroup; + } + + @Override + public String toString() { + return "ConsistencyGroupSnapshot{" + + "uuid='" + uuid + '\'' + + ", name='" + name + '\'' + + ", createTime='" + createTime + '\'' + + ", comment='" + comment + '\'' + + ", consistencyType='" + consistencyType + '\'' + + '}'; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupVolume.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupVolume.java new file mode 100644 index 000000000000..2a9cac5a6d99 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupVolume.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Volume member reference for consistency group create/update requests. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ConsistencyGroupVolume { + + @JsonProperty("uuid") + private String uuid; + + @JsonProperty("name") + private String name; + + @JsonProperty("provisioning_options") + private ConsistencyGroupVolumeProvisioningOptions provisioningOptions; + + public ConsistencyGroupVolume() { + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public ConsistencyGroupVolumeProvisioningOptions getProvisioningOptions() { + return provisioningOptions; + } + + public void setProvisioningOptions(ConsistencyGroupVolumeProvisioningOptions provisioningOptions) { + this.provisioningOptions = provisioningOptions; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupVolumeProvisioningOptions.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupVolumeProvisioningOptions.java new file mode 100644 index 000000000000..0e1955a62ee7 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ConsistencyGroupVolumeProvisioningOptions.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Provisioning options for a volume member of a consistency group. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ConsistencyGroupVolumeProvisioningOptions { + + @JsonProperty("action") + private String action; + + public ConsistencyGroupVolumeProvisioningOptions() { + } + + public ConsistencyGroupVolumeProvisioningOptions(String action) { + this.action = action; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/EmsApplicationLog.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/EmsApplicationLog.java new file mode 100644 index 000000000000..24289c8d33b2 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/EmsApplicationLog.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EmsApplicationLog { + + @JsonProperty("computer_name") + private String computerName; + + @JsonProperty("event_source") + private String eventSource; + + @JsonProperty("app_version") + private String appVersion; + + @JsonProperty("category") + private String category; + + @JsonProperty("severity") + private String severity; + + @JsonProperty("autosupport_required") + private Boolean autosupportRequired; + + @JsonProperty("event_id") + private String eventId; + + @JsonProperty("event_description") + private String eventDescription; + + public EmsApplicationLog() { + } + + public String getComputerName() { + return computerName; + } + + public void setComputerName(String computerName) { + this.computerName = computerName; + } + + public String getEventSource() { + return eventSource; + } + + public void setEventSource(String eventSource) { + this.eventSource = eventSource; + } + + public String getAppVersion() { + return appVersion; + } + + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getSeverity() { + return severity; + } + + public void setSeverity(String severity) { + this.severity = severity; + } + + public Boolean getAutosupportRequired() { + return autosupportRequired; + } + + public void setAutosupportRequired(Boolean autosupportRequired) { + this.autosupportRequired = autosupportRequired; + } + + public String getEventId() { + return eventId; + } + + public void setEventId(String eventId) { + this.eventId = eventId; + } + + public String getEventDescription() { + return eventDescription; + } + + public void setEventDescription(String eventDescription) { + this.eventDescription = eventDescription; + } + + @Override + public String toString() { + return "EmsApplicationLog{" + + "computerName='" + computerName + '\'' + + ", eventSource='" + eventSource + '\'' + + ", appVersion='" + appVersion + '\'' + + ", category='" + category + '\'' + + ", severity='" + severity + '\'' + + ", autosupportRequired=" + autosupportRequired + + ", eventId='" + eventId + '\'' + + ", eventDescription='" + eventDescription + '\'' + + '}'; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ExportRule.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ExportRule.java index 087e9aa681b4..15374811bf74 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ExportRule.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/ExportRule.java @@ -19,10 +19,13 @@ package org.apache.cloudstack.storage.feign.model; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; +import com.fasterxml.jackson.annotation.JsonValue; /** * ExportRule @@ -54,6 +57,7 @@ public enum ProtocolsEnum { this.value = value; } + @JsonValue public String getValue() { return value; } @@ -63,9 +67,13 @@ public String toString() { return String.valueOf(value); } + @JsonCreator public static ProtocolsEnum fromValue(String text) { + if (text == null) { + return null; + } for (ProtocolsEnum b : ProtocolsEnum.values()) { - if (String.valueOf(b.value).equals(text)) { + if (String.valueOf(b.value).equalsIgnoreCase(text)) { return b; } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileCloneRequest.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileCloneRequest.java new file mode 100644 index 000000000000..129643470895 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileCloneRequest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Request body for the ONTAP file clone API. + * + *

ONTAP REST endpoint: {@code POST /api/storage/file/clone}

+ * + *

Creates a space-efficient copy of a file. Source and destination paths are relative to the + * root of {@code volume}, and both must live in that same FlexVolume.

+ */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class FileCloneRequest { + + @JsonProperty("volume") + private VolumeRef volume; + + @JsonProperty("source_path") + private String sourcePath; + + @JsonProperty("destination_path") + private String destinationPath; + + @JsonProperty("overwrite_destination") + private Boolean overwriteDestination; + + /** + * Optional FlexVolume snapshot to clone from. When set, ONTAP clones {@code source_path} + * as it existed in that snapshot rather than from the live file/LUN. + * + *

Used by create-volume-from-snapshot (same FlexVol). Omitted for live template-cache clones.

+ */ + @JsonProperty("snapshot") + private SnapshotRef snapshot; + + public FileCloneRequest() { + } + + public FileCloneRequest(String flexVolUuid, String flexVolName, String sourcePath, String destinationPath) { + this.volume = new VolumeRef(flexVolUuid, flexVolName); + this.sourcePath = sourcePath; + this.destinationPath = destinationPath; + } + + public VolumeRef getVolume() { + return volume; + } + + public void setVolume(VolumeRef volume) { + this.volume = volume; + } + + public String getSourcePath() { + return sourcePath; + } + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + + public String getDestinationPath() { + return destinationPath; + } + + public void setDestinationPath(String destinationPath) { + this.destinationPath = destinationPath; + } + + public Boolean getOverwriteDestination() { + return overwriteDestination; + } + + public void setOverwriteDestination(Boolean overwriteDestination) { + this.overwriteDestination = overwriteDestination; + } + + public SnapshotRef getSnapshot() { + return snapshot; + } + + public void setSnapshot(SnapshotRef snapshot) { + this.snapshot = snapshot; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class VolumeRef { + + @JsonProperty("uuid") + private String uuid; + + @JsonProperty("name") + private String name; + + public VolumeRef() { + } + + public VolumeRef(String uuid, String name) { + this.uuid = uuid; + this.name = name; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + /** + * Snapshot identity for {@code POST /api/storage/file/clone} when cloning from a FlexVol snapshot. + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class SnapshotRef { + + @JsonProperty("name") + private String name; + + public SnapshotRef() { + } + + public SnapshotRef(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + @Override + public String toString() { + return "FileCloneRequest{volume=" + (volume != null ? volume.getUuid() : null) + + ", sourcePath=" + sourcePath + + ", destinationPath=" + destinationPath + "}"; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileInfo.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileInfo.java index a5dd24a3a286..71bf4c980bd4 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileInfo.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileInfo.java @@ -49,6 +49,8 @@ public class FileInfo { private Boolean overwriteEnabled = null; @JsonProperty("path") private String path = null; + @JsonProperty("qos_policy") + private VolumeQosPolicy qosPolicy = null; @JsonProperty("size") private Long size = null; @JsonProperty("target") @@ -178,6 +180,15 @@ public String getPath() { public void setPath(String path) { this.path = path; } + + public VolumeQosPolicy getQosPolicy() { + return qosPolicy; + } + + public void setQosPolicy(VolumeQosPolicy qosPolicy) { + this.qosPolicy = qosPolicy; + } + public Long getSize() { return size; } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/IpInterface.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/IpInterface.java index c15798a42b70..8070763285c8 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/IpInterface.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/IpInterface.java @@ -19,13 +19,13 @@ package org.apache.cloudstack.storage.feign.model; +import java.util.List; +import java.util.Objects; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.Objects; - @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class IpInterface { @@ -44,6 +44,15 @@ public class IpInterface { @JsonProperty("services") private List services; + @JsonProperty("state") + private String state; + + @JsonProperty("enabled") + private Boolean enabled; + + @JsonProperty("location") + private Location location; + // Getters and setters public String getUuid() { return uuid; @@ -85,6 +94,30 @@ public void setServices(List services) { this.services = services; } + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public Boolean getEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public Location getLocation() { + return location; + } + + public void setLocation(Location location) { + this.location = location; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -98,12 +131,14 @@ public boolean equals(Object o) { Objects.equals(name, that.name) && Objects.equals(ip, that.ip) && Objects.equals(svm, that.svm) && - Objects.equals(services, that.services); + Objects.equals(services, that.services) && + Objects.equals(state, that.state) && + Objects.equals(enabled, that.enabled); } @Override public int hashCode() { - return Objects.hash(uuid, name, ip, svm, services); + return Objects.hash(uuid, name, ip, svm, services, state, enabled); } @Override @@ -114,9 +149,52 @@ public String toString() { ", ip=" + ip + ", svm=" + svm + ", services=" + services + + ", state='" + state + '\'' + + ", enabled=" + enabled + '}'; } + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class Node { + @JsonProperty("name") + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class Location { + @JsonProperty("home_node") + private Node homeNode; + + @JsonProperty("node") + private Node node; + + public Node getHomeNode() { + return homeNode; + } + + public void setHomeNode(Node homeNode) { + this.homeNode = homeNode; + } + + public Node getNode() { + return node; + } + + public void setNode(Node node) { + this.node = node; + } + } + // Nested class for IP information @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Lun.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Lun.java index 364790958c8a..0b2e09dbec45 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Lun.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/Lun.java @@ -83,6 +83,9 @@ public static PropertyClassEnum fromValue(String value) { @JsonProperty("name") private String name = null; + @JsonProperty("qos_policy") + private VolumeQosPolicy qosPolicy = null; + @JsonProperty("clone") private Clone clone = null; @@ -202,6 +205,14 @@ public void setName(String name) { this.name = name; } + public VolumeQosPolicy getQosPolicy() { + return qosPolicy; + } + + public void setQosPolicy(VolumeQosPolicy qosPolicy) { + this.qosPolicy = qosPolicy; + } + public Lun osType(OsTypeEnum osType) { this.osType = osType; return this; @@ -308,6 +319,7 @@ private String toIndentedString(Object o) { } + @JsonInclude(JsonInclude.Include.NON_NULL) public static class Clone { @JsonProperty("source") private Source source = null; @@ -319,6 +331,7 @@ public void setSource(Source source) { } } + @JsonInclude(JsonInclude.Include.NON_NULL) public static class Source { @JsonProperty("name") private String name = null; diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/VolumeQosPolicy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/VolumeQosPolicy.java index 7a9a4307ab1a..2e6ce1b6fa3c 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/VolumeQosPolicy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/VolumeQosPolicy.java @@ -26,43 +26,23 @@ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class VolumeQosPolicy { - @JsonProperty("max_throughput_iops") - private Integer maxThroughputIops = null; - - @JsonProperty("max_throughput_mbps") - private Integer maxThroughputMbps = null; - - @JsonProperty("min_throughput_iops") - private Integer minThroughputIops = null; - + @JsonProperty("fixed") + private Fixed fixed; @JsonProperty("name") private String name = null; - @JsonProperty("uuid") private String uuid = null; + @JsonProperty("svm") + private Svm svm; + @JsonProperty("object_count") + private Integer objectCount; - public Integer getMaxThroughputIops() { - return maxThroughputIops; - } - - public void setMaxThroughputIops(Integer maxThroughputIops) { - this.maxThroughputIops = maxThroughputIops; - } - - public Integer getMaxThroughputMbps() { - return maxThroughputMbps; + public Fixed getFixed() { + return fixed; } - public void setMaxThroughputMbps(Integer maxThroughputMbps) { - this.maxThroughputMbps = maxThroughputMbps; - } - - public Integer getMinThroughputIops() { - return minThroughputIops; - } - - public void setMinThroughputIops(Integer minThroughputIops) { - this.minThroughputIops = minThroughputIops; + public void setFixed(Fixed fixed) { + this.fixed = fixed; } public String getName() { @@ -80,4 +60,57 @@ public String getUuid() { public void setUuid(String uuid) { this.uuid = uuid; } + + public Svm getSvm() { + return svm; + } + + public void setSvm(Svm svm) { + this.svm = svm; + } + + public Integer getObjectCount() { + return objectCount; + } + + public void setObjectCount(Integer objectCount) { + this.objectCount = objectCount; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class Fixed { + @JsonProperty("capacity_shared") + private Boolean capacityShared; + + @JsonProperty("min_throughput_iops") + private Long minThroughputIops; + + @JsonProperty("max_throughput_iops") + private Long maxThroughputIops; + + public Boolean getCapacityShared() { + return capacityShared; + } + + public void setCapacityShared(Boolean capacityShared) { + this.capacityShared = capacityShared; + } + + public Long getMinThroughputIops() { + return minThroughputIops; + } + + public void setMinThroughputIops(Long minThroughputIops) { + this.minThroughputIops = minThroughputIops; + } + + public Long getMaxThroughputIops() { + return maxThroughputIops; + } + + public void setMaxThroughputIops(Long maxThroughputIops) { + this.maxThroughputIops = maxThroughputIops; + } + } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycle.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycle.java index fec594ea0ea6..83b0c0223ea0 100755 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycle.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycle.java @@ -19,20 +19,15 @@ package org.apache.cloudstack.storage.lifecycle; -import org.apache.cloudstack.engine.subsystem.api.storage.Scope; -import com.cloud.agent.api.StoragePoolInfo; -import com.cloud.dc.ClusterVO; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.Storage; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePool; -import com.cloud.storage.StoragePoolAutomation; -import com.cloud.utils.exception.CloudRuntimeException; -import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.inject.Inject; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -40,12 +35,14 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.lifecycle.BasePrimaryDataStoreLifeCycleImpl; +import org.apache.cloudstack.storage.feign.model.Aggregate; import org.apache.cloudstack.storage.feign.model.OntapStorage; import org.apache.cloudstack.storage.feign.model.Volume; import org.apache.cloudstack.storage.provider.StorageProviderFactory; @@ -59,13 +56,21 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.alert.AlertManager; +import com.cloud.capacity.CapacityManager; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolAutomation; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Preconditions; public class OntapPrimaryDatastoreLifecycle extends BasePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle { @Inject private ClusterDao _clusterDao; @@ -76,9 +81,11 @@ public class OntapPrimaryDatastoreLifecycle extends BasePrimaryDataStoreLifeCycl @Inject private StoragePoolAutomation _storagePoolAutomation; @Inject private PrimaryDataStoreDao storagePoolDao; @Inject private StoragePoolDetailsDao storagePoolDetailsDao; + @Inject private AlertManager _alertMgr; + @Inject private CapacityManager _capacityMgr; private static final Logger logger = LogManager.getLogger(OntapPrimaryDatastoreLifecycle.class); - private static final long ONTAP_MIN_VOLUME_SIZE_IN_BYTES = 1677721600L; + private static final long ONTAP_MIN_VOLUME_SIZE_IN_BYTES = 20971520L; /** * Creates primary storage on NetApp storage @@ -96,6 +103,7 @@ public DataStore initialize(Map dsInfos) { String storagePoolName = (String) dsInfos.get("name"); String providerName = (String) dsInfos.get("providerName"); Long capacityBytes = (Long) dsInfos.get("capacityBytes"); + Long capacityIops = (Long) dsInfos.get("capacityIops"); boolean managed = (boolean) dsInfos.get("managed"); String tags = (String) dsInfos.get("tags"); Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @@ -108,6 +116,8 @@ public DataStore initialize(Map dsInfos) { @SuppressWarnings("unchecked") Map details = (Map) dsInfos.get("details"); + validateInitializeInputs(capacityBytes, capacityIops, podId, clusterId, zoneId, storagePoolName, providerName, managed, details); + PrimaryDataStoreParameters parameters = new PrimaryDataStoreParameters(); if (clusterId != null) { ClusterVO clusterVO = _clusterDao.findById(clusterId); @@ -118,8 +128,6 @@ public DataStore initialize(Map dsInfos) { parameters.setHypervisorType(clusterVO.getHypervisorType()); } - capacityBytes = validateInitializeInputs(capacityBytes, podId, clusterId, zoneId, storagePoolName, providerName, managed, details); - details.put(OntapStorageConstants.SIZE, capacityBytes.toString()); ProtocolType protocol = ProtocolType.valueOf(details.get(OntapStorageConstants.PROTOCOL)); @@ -135,17 +143,34 @@ public DataStore initialize(Map dsInfos) { StorageStrategy storageStrategy = StorageProviderFactory.getStrategy(ontapStorage); boolean isValid = storageStrategy.connect(); if (isValid) { - // Get the DataLIF for data access - String dataLIF = storageStrategy.getNetworkInterface(); - if (dataLIF == null || dataLIF.isEmpty()) { - throw new CloudRuntimeException("Failed to retrieve Data LIF from ONTAP, cannot create primary storage"); + details.put(OntapStorageConstants.IS_AFF, Boolean.toString(storageStrategy.isAff())); + if (storageStrategy.getResolvedSvmUuid() != null && !storageStrategy.getResolvedSvmUuid().isEmpty()) { + details.put(OntapStorageConstants.SVM_UUID, storageStrategy.getResolvedSvmUuid()); + } + Aggregate aggregate; + try { + aggregate = storageStrategy.chooseAggregate(storageStrategy.getAggregates(), capacityBytes); + } catch (Exception e) { + logger.error("Exception occurred while choosing aggregate for pool: " + storagePoolName, e); + throw new CloudRuntimeException("Failed to choose ONTAP aggregate for pool: " + storagePoolName + + ". Error: " + e.getMessage(), e); + } + + Map lifResult; + try { + lifResult = storageStrategy.getNetworkInterface(aggregate); + } catch (Exception e) { + logger.error("Exception occurred while retrieving network interface for pool: " + storagePoolName, e); + throw new CloudRuntimeException("Failed to retrieve Data LIF from ONTAP: " + e.getMessage(), e); } - logger.info("Using Data LIF for storage access: " + dataLIF); - details.put(OntapStorageConstants.DATA_LIF, dataLIF); + String dataLif = lifResult.get(OntapStorageConstants.DATA_LIF); + String lifWarning = lifResult.get(OntapStorageConstants.LIF_WARNING); + processDataLifSelection(dataLif, lifWarning, details, storagePoolName, zoneId, podId); + logger.info("Creating ONTAP volume '" + storagePoolName + "' with size: " + capacityBytes + " bytes (" + (capacityBytes / (1024 * 1024 * 1024)) + " GB)"); try { - Volume volume = storageStrategy.createStorageVolume(storagePoolName, capacityBytes); + Volume volume = storageStrategy.createStorageVolume(storagePoolName, capacityBytes, aggregate); if (volume == null) { logger.error("createStorageVolume returned null for volume: " + storagePoolName); throw new CloudRuntimeException("Failed to create ONTAP volume: " + storagePoolName); @@ -174,7 +199,7 @@ public DataStore initialize(Map dsInfos) { logger.info("Setting NFS path for storage pool: " + path + ", port: " + port + " with mount option: vers=3"); break; case ISCSI: - parameters.setType(Storage.StoragePoolType.Iscsi); + parameters.setType(Storage.StoragePoolType.OntapiSCSI); path = storageStrategy.getStoragePath(); port = OntapStorageConstants.ISCSI_PORT; logger.info("Setting iSCSI path for storage pool: " + path + ", port: " + port); @@ -197,21 +222,26 @@ public DataStore initialize(Map dsInfos) { parameters.setProviderName(providerName); parameters.setManaged(managed); parameters.setCapacityBytes(capacityBytes); + parameters.setCapacityIops(capacityIops); parameters.setUsedBytes(0); return _dataStoreHelper.createPrimaryDataStore(parameters); } - private long validateInitializeInputs(Long capacityBytes, Long podId, Long clusterId, Long zoneId, + private void validateInitializeInputs(Long capacityBytes, Long capacityIops, Long podId, Long clusterId, Long zoneId, String storagePoolName, String providerName, boolean managed, Map details) { - // Validate and set capacity if (capacityBytes == null || capacityBytes <= 0) { - logger.warn("capacityBytes not provided or invalid (" + capacityBytes + "), using ONTAP minimum size: " + ONTAP_MIN_VOLUME_SIZE_IN_BYTES); - capacityBytes = ONTAP_MIN_VOLUME_SIZE_IN_BYTES; - } else if (capacityBytes < ONTAP_MIN_VOLUME_SIZE_IN_BYTES) { - logger.warn("capacityBytes (" + capacityBytes + ") is below ONTAP minimum (" + ONTAP_MIN_VOLUME_SIZE_IN_BYTES + "), adjusting to minimum"); - capacityBytes = ONTAP_MIN_VOLUME_SIZE_IN_BYTES; + throw new InvalidParameterValueException("Storage pool capacity is required for ONTAP primary storage and must be at least " + + ONTAP_MIN_VOLUME_SIZE_IN_BYTES + " bytes (20 MB)"); + } + if (capacityBytes < ONTAP_MIN_VOLUME_SIZE_IN_BYTES) { + throw new InvalidParameterValueException("Storage pool capacity " + capacityBytes + " bytes is below the ONTAP minimum volume size of " + + ONTAP_MIN_VOLUME_SIZE_IN_BYTES + " bytes (20 MB)"); + } + // IOPS capacity is optional; when left blank no pool-level IOPS ceiling is enforced. + if (capacityIops != null && capacityIops <= 0) { + throw new InvalidParameterValueException("Storage pool IOPS capacity must be greater than 0"); } // Validate scope @@ -268,8 +298,24 @@ private long validateInitializeInputs(Long capacityBytes, Long podId, Long clust missing.removeAll(providedKeys); throw new CloudRuntimeException("ONTAP primary storage creation failed, missing detail(s): " + missing); } + } + + private void processDataLifSelection(String dataLIF, String lifWarning, Map details, + String storagePoolName, Long zoneId, Long podId) { + if (dataLIF == null || dataLIF.isEmpty()) { + throw new CloudRuntimeException("Failed to retrieve Data LIF from ONTAP, cannot create primary storage"); + } + logger.info("Using Data LIF for storage access: " + dataLIF); + details.put(OntapStorageConstants.DATA_LIF, dataLIF); - return capacityBytes; + // Persist LIF warning as a pool detail and fire a storage alert so the user is informed + if (lifWarning != null) { + details.put(OntapStorageConstants.LIF_WARNING, lifWarning); + logger.warn("LIF selection warning for pool '" + storagePoolName + "': " + lifWarning); + String alertSubject = "ONTAP Storage Pool '" + storagePoolName + "': " + + lifWarning.split(OntapStorageConstants.SEMICOLON)[0].trim(); + OntapStorageUtils.sendStorageAlert(_alertMgr, zoneId, podId, alertSubject, lifWarning); + } } @Override @@ -523,7 +569,62 @@ public boolean migrateToObjectStore(DataStore store) { @Override public void updateStoragePool(StoragePool storagePool, Map details) { + String newCapacityIopsStr = details.get(PrimaryDataStoreLifeCycle.CAPACITY_IOPS); + if (newCapacityIopsStr != null) { + validateUpdatedCapacityIops(storagePool, newCapacityIopsStr); + } + + String newCapacityStr = details.get(PrimaryDataStoreLifeCycle.CAPACITY_BYTES); + if (newCapacityStr == null) { + logger.debug("No capacity change requested for pool: {}, skipping FlexVolume resize", storagePool.getName()); + return; + } + + long currentCapacityBytes = storagePool.getCapacityBytes(); + long newCapacityBytes = Long.parseLong(newCapacityStr); + StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + + String volumeUuid = details.get(OntapStorageConstants.VOLUME_UUID); + if (volumeUuid == null || volumeUuid.isEmpty()) { + logger.error("Volume UUID or name not found in details for pool: {}, cannot resize", storagePool.getName()); + throw new CloudRuntimeException("Volume UUID or name not found in details, cannot resize ONTAP FlexVolume"); + } + + Volume volume = new Volume(); + volume.setUuid(volumeUuid); + volume.setName(details.get(OntapStorageConstants.VOLUME_NAME)); + volume.setSize(newCapacityBytes); + try { + storageStrategy.updateStorageVolume(volume); + logger.info("Successfully resized ONTAP FlexVolume '{}' (UUID: {}) for pool '{}' from {} bytes to {} bytes", + volume.getName(), volume.getUuid(), storagePool.getName(), currentCapacityBytes, newCapacityBytes); + } catch (Exception e) { + logger.error("Exception while resizing FlexVolume for pool: {}. Error: {}", storagePool.getName(), e.getMessage(), e); + throw new CloudRuntimeException("Failed to resize ONTAP FlexVolume for pool: " + storagePool.getName() + ". " + e.getMessage(), e); + } + } + + private void validateUpdatedCapacityIops(StoragePool storagePool, String newCapacityIopsStr) { + long newCapacityIops; + try { + newCapacityIops = Long.parseLong(newCapacityIopsStr); + } catch (NumberFormatException e) { + throw new InvalidParameterValueException("Invalid storage pool IOPS capacity: " + newCapacityIopsStr); + } + if (newCapacityIops <= 0) { + throw new InvalidParameterValueException("Storage pool IOPS capacity must be greater than 0"); + } + StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePool.getId()); + if (storagePoolVO == null) { + throw new InvalidParameterValueException("Storage pool not found for id: " + storagePool.getId()); + } + long allocatedIops = _capacityMgr.getUsedIops(storagePoolVO); + if (newCapacityIops < allocatedIops) { + throw new InvalidParameterValueException(String.format( + "Cannot set IOPS capacity of storage pool %s to %d IOPS because %d IOPS are already allocated", + storagePool.getName(), newCapacityIops, allocatedIops)); + } } @Override diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/listener/OntapHostListener.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/listener/OntapHostListener.java index ecdd3efd2c5c..f750e2904023 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/listener/OntapHostListener.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/listener/OntapHostListener.java @@ -19,30 +19,38 @@ package org.apache.cloudstack.storage.listener; +import java.util.List; +import java.util.Map; + import javax.inject.Inject; -import com.cloud.agent.api.ModifyStoragePoolCommand; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.service.model.AccessGroup; +import org.apache.cloudstack.storage.service.model.ProtocolType; +import org.apache.cloudstack.storage.utils.OntapStorageConstants; +import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; import com.cloud.agent.api.ModifyStoragePoolAnswer; +import com.cloud.agent.api.ModifyStoragePoolCommand; import com.cloud.agent.api.StoragePoolInfo; import com.cloud.alert.AlertManager; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.dao.StoragePoolHostDao; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.DeleteStoragePoolCommand; -import com.cloud.host.Host; -import com.cloud.storage.StoragePool; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; -import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; -import com.cloud.host.dao.HostDao; - -import java.util.Map; public class OntapHostListener implements HypervisorHostListener { protected Logger logger = LogManager.getLogger(getClass()); @@ -63,26 +71,39 @@ public class OntapHostListener implements HypervisorHostListener { @Override public boolean hostConnect(long hostId, long poolId) { - logger.info("Connect to host " + hostId + " from pool " + poolId); + logger.info("hostConnect: Connecting host {} to pool {}", hostId, poolId); Host host = _hostDao.findById(hostId); if (host == null) { - logger.error("host was not found with id : {}", hostId); + logger.error("hostConnect: Host was not found with id: {}", hostId); return false; } if (!host.getHypervisorType().equals(Hypervisor.HypervisorType.KVM)) { - logger.error("ONTAP plugin does not support {} type host currently ", host.getHypervisorType()); + logger.error("hostConnect: ONTAP plugin does not support {} type host currently", host.getHypervisorType()); return false; } StoragePool pool = _storagePoolDao.findById(poolId); if (pool == null) { - logger.error("Failed to connect host - storage pool not found with id: {}", poolId); + logger.error("hostConnect: Failed to connect host - storage pool not found with id: {}", poolId); return false; } - logger.info("Connecting host {} to ONTAP storage pool {}", host.getName(), pool.getName()); + logger.info("hostConnect: Connecting host {} to ONTAP storage pool {}", host.getName(), pool.getName()); try { // Load storage pool details from database to pass mount options and other config to agent Map detailsMap = _storagePoolDetailsDao.listDetailsKeyPairs(poolId); + if (detailsMap == null || detailsMap.isEmpty()) { + logger.error("hostConnect: Failed to load storage pool details for pool id: {}", poolId); + return false; + } + + if (detailsMap.get(OntapStorageConstants.PROTOCOL) == null) { + logger.error("hostConnect: Storage pool details missing required protocol type for pool id: {}", poolId); + return false; + } + + // Update NFS export policy for this connected host when the pool protocol is NFS3. + updateNfsExportPolicyForConnectedHostIfNeeded(poolId, hostId, host, detailsMap); + // Create the ModifyStoragePoolCommand to send to the agent // Note: Always send command even if database entry exists, because agent may have restarted // and lost in-memory pool registration. The command handler is idempotent. @@ -118,7 +139,7 @@ public boolean hostConnect(long hostId, long poolId) { } String localPath = poolInfo.getLocalPath(); - logger.info("Storage pool {} successfully mounted at: {}", pool.getName(), localPath); + logger.info("hostConnect: Storage pool {} successfully mounted at: {}", pool.getName(), localPath); // Update or create the storage_pool_host_ref entry with the correct local_path StoragePoolHostVO storagePoolHost = storagePoolHostDao.findByPoolHost(poolId, hostId); @@ -126,11 +147,11 @@ public boolean hostConnect(long hostId, long poolId) { if (storagePoolHost == null) { storagePoolHost = new StoragePoolHostVO(poolId, hostId, localPath); storagePoolHostDao.persist(storagePoolHost); - logger.info("Created storage_pool_host_ref entry for pool {} and host {}", pool.getName(), host.getName()); + logger.info("hostConnect: Created storage_pool_host_ref entry for pool {} and host {}", pool.getName(), host.getName()); } else { storagePoolHost.setLocalPath(localPath); storagePoolHostDao.update(storagePoolHost.getId(), storagePoolHost); - logger.info("Updated storage_pool_host_ref entry with local_path: {}", localPath); + logger.info("hostConnect: Updated storage_pool_host_ref entry with local_path: {}", localPath); } // Update pool capacity/usage information @@ -139,11 +160,11 @@ public boolean hostConnect(long hostId, long poolId) { poolVO.setCapacityBytes(poolInfo.getCapacityBytes()); poolVO.setUsedBytes(poolInfo.getCapacityBytes() - poolInfo.getAvailableBytes()); _storagePoolDao.update(poolVO.getId(), poolVO); - logger.info("Updated storage pool capacity: {} GB, used: {} GB", poolInfo.getCapacityBytes() / (1024 * 1024 * 1024), (poolInfo.getCapacityBytes() - poolInfo.getAvailableBytes()) / (1024 * 1024 * 1024)); + logger.info("hostConnect: Updated storage pool capacity: {} GB, used: {} GB", poolInfo.getCapacityBytes() / (1024 * 1024 * 1024), (poolInfo.getCapacityBytes() - poolInfo.getAvailableBytes()) / (1024 * 1024 * 1024)); } } catch (Exception e) { - logger.error("Exception while connecting host {} to storage pool {}", host.getName(), pool.getName(), e); + logger.error("hostConnect: Exception while connecting host {} to storage pool {}", host.getName(), pool.getName(), e); // CRITICAL: Don't throw exception - it crashes the agent and causes restart loops // Return false to indicate failure without crashing return false; @@ -151,50 +172,111 @@ public boolean hostConnect(long hostId, long poolId) { return true; } - @Override - public boolean hostDisconnected(long hostId, long poolId) { - logger.info("Disconnect from host " + hostId + " from pool " + poolId); + private void updateNfsExportPolicyForConnectedHostIfNeeded(long poolId, long hostId, Host host, Map detailsMap) { + if (!ProtocolType.NFS3.name().equalsIgnoreCase(detailsMap.get(OntapStorageConstants.PROTOCOL))) { + return; + } - Host hostToremove = _hostDao.findById(hostId); - if (hostToremove == null) { - logger.error("Failed to add host by HostListener as host was not found with id : {}", hostId); - return false; + if (!isNfs3EnabledOnHost(host)) { + throw new CloudRuntimeException("NFS protocol is not enabled on host with id: " + hostId); } - StoragePool pool = _storagePoolDao.findById(poolId); - if (pool == null) { - logger.error("Failed to disconnect host - storage pool not found with id: {}", poolId); + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(poolId); + accessGroup.setHostsToConnect(List.of((HostVO) host)); + + StorageStrategy strategy = OntapStorageUtils.getStrategyByStoragePoolDetails(detailsMap); + strategy.updateAccessGroup(accessGroup); + logger.info("hostConnect: updateNfsExportPolicyForConnectedHostIfNeeded: Updated NFS export policy rules for host {} on storage pool {}", host.getName(), poolId); + } + + private boolean isNfs3EnabledOnHost(Host host) { + if (host == null) { return false; } - logger.info("Disconnecting host {} from ONTAP storage pool {}", hostToremove.getName(), pool.getName()); - try { - DeleteStoragePoolCommand cmd = new DeleteStoragePoolCommand(pool); - Answer answer = _agentMgr.easySend(hostId, cmd); - if (answer != null && answer.getResult()) { - logger.info("Successfully disconnected host {} from ONTAP storage pool {}", hostToremove.getName(), pool.getName()); - return true; - } else { - String errMsg = (answer != null) ? answer.getDetails() : "Unknown error"; - logger.warn("Failed to disconnect host {} from storage pool {}. Error: {}", hostToremove.getName(), pool.getName(), errMsg); - return false; - } - } catch (Exception e) { - logger.error("Exception while disconnecting host {} from storage pool {}", hostToremove.getName(), pool.getName(), e); + String storageIp = host.getStorageIpAddress() != null ? host.getStorageIpAddress().trim() : ""; + if (storageIp.isEmpty() && StringUtils.isBlank(host.getPrivateIpAddress())) { + logger.warn("isNfs3EnabledOnHost: Host {} is not eligible for NFS3 protocol: both storage IP and private IP are empty", + host.getId()); return false; } + + return true; } @Override - public boolean hostAboutToBeRemoved(long hostId) { + public boolean hostDisconnected(long hostId, long poolId) { + logger.info("hostDisconnected: Disconnecting host {} from pool {}", hostId, poolId); + // Note: This is not currently being called for NetApp ONTAP storage plugin. return false; } + @Override + public boolean hostAboutToBeRemoved(long hostId) { + logger.info("hostAboutToBeRemoved: Host {} is about to be removed", hostId); + + Host host = _hostDao.findById(hostId); + if (host == null) { + logger.warn("hostAboutToBeRemoved: Host not found with id: {}, considering it as no-op", hostId); + return true; + } + + List poolHostRefs = storagePoolHostDao.listByHostId(hostId); + if (poolHostRefs == null || poolHostRefs.isEmpty()) { + logger.debug("hostAboutToBeRemoved: No storage pool associations found for host {}", hostId); + return true; + } + + for (StoragePoolHostVO ref : poolHostRefs) { + StoragePoolVO pool = _storagePoolDao.findById(ref.getPoolId()); + if (pool != null) { + removeHostFromOntapPoolIfNeeded(pool, host); + } + } + + logger.info("hostAboutToBeRemoved: Cleaned up ONTAP export policies for host {} about to be removed", hostId); + return true; + } + @Override public boolean hostRemoved(long hostId, long clusterId) { return false; } + private void removeHostFromOntapPoolIfNeeded(StoragePoolVO pool, Host host) { + try { + Map detailsMap = _storagePoolDetailsDao.listDetailsKeyPairs(pool.getId()); + if (detailsMap == null || detailsMap.isEmpty()) { + logger.debug("hostAboutToBeRemoved: removeHostFromOntapPoolIfNeeded: No pool details found for pool id: {}", pool.getId()); + return; + } + + // Skip non-NFS3 pools; Currently, for iSCSI type, iGroup rules are being handled as part of revokeAccess in OntapPrimaryDataStoreDriver, so no need to handle here. + if (!ProtocolType.NFS3.name().equalsIgnoreCase(detailsMap.get(OntapStorageConstants.PROTOCOL))) { + return; + } + + logger.info("hostAboutToBeRemoved: removeHostFromOntapPoolIfNeeded: Removing export policy rule for host {} from storage pool {}", host.getName(), pool.getName()); + if (!isNfs3EnabledOnHost(host)) { + logger.warn("hostAboutToBeRemoved: removeHostFromOntapPoolIfNeeded: Skipping NFS export policy removal for host {} on pool {} as host is not NFS-enabled", + host.getId(), pool.getId()); + return; + } + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(pool.getId()); + accessGroup.setHostsToConnect(List.of((HostVO) host)); + accessGroup.setHostRuleAction(AccessGroup.HostRuleAction.REMOVE); + + StorageStrategy strategy = OntapStorageUtils.getStrategyByStoragePoolDetails(detailsMap); + strategy.updateAccessGroup(accessGroup); + logger.info("hostAboutToBeRemoved: removeHostFromOntapPoolIfNeeded: Removed NFS export policy rules for removed host {} from storage pool {}", host.getName(), pool.getName()); + } catch (Exception e) { + logger.warn("hostAboutToBeRemoved: removeHostFromOntapPoolIfNeeded: Failed to remove NFS export policy rule for host {} from pool {}: {}", host.getId(), pool.getName(), e.getMessage()); + // Continue processing other pools even if one fails + } + } + @Override public boolean hostEnabled(long hostId) { return false; diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java index c13b255c67ea..a19a90c59641 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java @@ -19,25 +19,41 @@ package org.apache.cloudstack.storage.service; -import com.cloud.utils.exception.CloudRuntimeException; -import feign.FeignException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.feign.FeignClientFactory; import org.apache.cloudstack.storage.feign.client.AggregateFeignClient; +import org.apache.cloudstack.storage.feign.client.ClusterFeignClient; +import org.apache.cloudstack.storage.feign.client.EmsFeignClient; import org.apache.cloudstack.storage.feign.client.JobFeignClient; -import org.apache.cloudstack.storage.feign.client.NetworkFeignClient; import org.apache.cloudstack.storage.feign.client.NASFeignClient; +import org.apache.cloudstack.storage.feign.client.NetworkFeignClient; +import org.apache.cloudstack.storage.feign.client.QosFeignClient; import org.apache.cloudstack.storage.feign.client.SANFeignClient; import org.apache.cloudstack.storage.feign.client.SnapshotFeignClient; import org.apache.cloudstack.storage.feign.client.SvmFeignClient; import org.apache.cloudstack.storage.feign.client.VolumeFeignClient; import org.apache.cloudstack.storage.feign.model.Aggregate; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.ClusterNode; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; import org.apache.cloudstack.storage.feign.model.IpInterface; import org.apache.cloudstack.storage.feign.model.IscsiService; import org.apache.cloudstack.storage.feign.model.Job; import org.apache.cloudstack.storage.feign.model.Nas; import org.apache.cloudstack.storage.feign.model.OntapStorage; import org.apache.cloudstack.storage.feign.model.Svm; +import org.apache.cloudstack.storage.feign.model.Version; import org.apache.cloudstack.storage.feign.model.Volume; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import org.apache.cloudstack.storage.service.model.AccessGroup; @@ -45,14 +61,13 @@ import org.apache.cloudstack.storage.service.model.ProtocolType; import org.apache.cloudstack.storage.utils.OntapStorageConstants; import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.HashMap; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import com.cloud.utils.exception.CloudRuntimeException; + +import feign.FeignException; /** * Storage Strategy represents the communication path for all the ONTAP storage options @@ -69,9 +84,12 @@ public abstract class StorageStrategy { protected SvmFeignClient svmFeignClient; protected JobFeignClient jobFeignClient; protected NetworkFeignClient networkFeignClient; + protected QosFeignClient qosFeignClient; protected SANFeignClient sanFeignClient; protected NASFeignClient nasFeignClient; protected SnapshotFeignClient snapshotFeignClient; + protected ClusterFeignClient clusterFeignClient; + protected EmsFeignClient emsFeignClient; protected OntapStorage storage; @@ -79,6 +97,7 @@ public abstract class StorageStrategy { * Presents aggregate object for the unified storage, not eligible for disaggregated */ private List aggregates; + private String resolvedSvmUuid; private static final Logger logger = LogManager.getLogger(StorageStrategy.class); @@ -93,15 +112,194 @@ public StorageStrategy(OntapStorage ontapStorage) { this.svmFeignClient = feignClientFactory.createClient(SvmFeignClient.class, baseURL); this.jobFeignClient = feignClientFactory.createClient(JobFeignClient.class, baseURL); this.networkFeignClient = feignClientFactory.createClient(NetworkFeignClient.class, baseURL); + this.qosFeignClient = feignClientFactory.createClient(QosFeignClient.class, baseURL); this.sanFeignClient = feignClientFactory.createClient(SANFeignClient.class, baseURL); this.nasFeignClient = feignClientFactory.createClient(NASFeignClient.class, baseURL); this.snapshotFeignClient = feignClientFactory.createClient(SnapshotFeignClient.class, baseURL); + this.clusterFeignClient = feignClientFactory.createClient(ClusterFeignClient.class, baseURL); + this.emsFeignClient = feignClientFactory.createClient(EmsFeignClient.class, baseURL); + } + + /** + * Fetches the ONTAP {@link Cluster} (name, uuid, version) and, for ASUP, the rolled-up node + * {@code model} and {@code platformType} from {@code GET /api/cluster/nodes}. Best-effort: + * returns {@code null} if the cluster GET fails. Node fields are left unset if the nodes GET + * fails. Callers must never fail a storage operation because of this. + * + * @return the ONTAP {@link Cluster}, or {@code null} if it cannot be resolved + */ + public Cluster getClusterInfo() { + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + Cluster cluster = clusterFeignClient.getCluster(authHeader, true); + if (cluster != null) { + populateNodeAsupFields(cluster, authHeader); + } + return cluster; + } catch (Exception e) { + logger.warn("getClusterInfo: failed to fetch ONTAP cluster info for storage IP {}: {}", + storage.getStorageIP(), e.getMessage()); + return null; + } + } + + /** + * Sets {@link Cluster#setModel(String)} and {@link Cluster#setPlatformType(String)} from one + * {@code GET /api/cluster/nodes}. Distinct models are joined with a comma. Platform type is + * {@code performance}, {@code capacity}, {@code fas}, or {@code composite} when personalities mix. + * A node that omits {@code is_all_flash_optimized} is left out of the rollup so a partial + * response is not reported as {@code fas}. + */ + private void populateNodeAsupFields(Cluster cluster, String authHeader) { + try { + Map query = new HashMap<>(); + query.put(OntapStorageConstants.FIELDS, OntapStorageConstants.CLUSTER_NODE_ASUP_FIELDS); + OntapResponse response = clusterFeignClient.getClusterNodes(authHeader, query); + if (response == null || response.getRecords() == null || response.getRecords().isEmpty()) { + return; + } + LinkedHashSet models = new LinkedHashSet<>(); + LinkedHashSet platformTypes = new LinkedHashSet<>(); + for (ClusterNode node : response.getRecords()) { + if (node == null) { + continue; + } + if (node.getModel() != null && !node.getModel().isBlank()) { + models.add(node.getModel().trim()); + } + String platformType = classifyNodePlatformType(node); + if (platformType != null) { + platformTypes.add(platformType); + } + } + if (!models.isEmpty()) { + cluster.setModel(String.join(OntapStorageConstants.COMMA, models)); + } + cluster.setPlatformType(rollupPlatformType(platformTypes)); + } catch (Exception e) { + logger.warn("getClusterInfo: failed to fetch ONTAP node hardware for storage IP {}: {}", + storage.getStorageIP(), e.getMessage()); + } + } + + /** + * Extracts a clean, parser-friendly ONTAP version string from a {@link Cluster}. + * + *

Prefers the compact "generation.major.minor" numeric form (e.g. "9.17.1"), which avoids + * the colon/date noise in {@code version.full}. Falls back to the verbose {@code version.full} + * banner only when the numeric fields are unavailable.

+ * + * @param cluster the cluster (may be {@code null}) + * @return the ONTAP version string, or {@code null} if it cannot be resolved + */ + public String getClusterVersion(Cluster cluster) { + if (cluster == null || cluster.getVersion() == null) { + return null; + } + Version version = cluster.getVersion(); + if (version.getGeneration() != null && version.getMajor() != null && version.getMinor() != null) { + return version.getGeneration() + OntapStorageConstants.DOT + version.getMajor() + + OntapStorageConstants.DOT + version.getMinor(); + } + if (version.getFull() != null && !version.getFull().isEmpty()) { + return version.getFull(); + } + return null; + } + + /** + * Classifies one node: not all-flash → {@code fas}; all-flash + capacity → {@code capacity}; + * otherwise all-flash (including performance-optimized or classic AFF) → {@code performance}. + * Returns {@code null} when {@code is_all_flash_optimized} is absent, so unknown hardware is + * not reported as {@code fas}. + */ + private static String classifyNodePlatformType(ClusterNode node) { + if (node.getAllFlashOptimized() == null) { + return null; + } + if (!node.getAllFlashOptimized()) { + return OntapStorageConstants.ASUP_PLATFORM_TYPE_FAS; + } + if (Boolean.TRUE.equals(node.getCapacityOptimized())) { + return OntapStorageConstants.ASUP_PLATFORM_TYPE_CAPACITY; + } + return OntapStorageConstants.ASUP_PLATFORM_TYPE_PERFORMANCE; + } + + private static String rollupPlatformType(LinkedHashSet platformTypes) { + if (platformTypes == null || platformTypes.isEmpty()) { + return null; + } + if (platformTypes.size() == 1) { + return platformTypes.iterator().next(); + } + return OntapStorageConstants.ASUP_PLATFORM_TYPE_COMPOSITE; + } + + /** + * True when every cluster node reports {@code is_all_flash_optimized} (AFF, including C-series). + * A FAS node, or a node that omits the flag, makes this false. Used for min-throughput QoS support. + */ + public boolean isAff() { + Map query = new HashMap<>(); + query.put(OntapStorageConstants.FIELDS, OntapStorageConstants.CLUSTER_NODE_ASUP_FIELDS); + OntapResponse response = clusterFeignClient.getClusterNodes(getAuthHeader(), query); + if (response == null || response.getRecords() == null + || response.getRecords().stream().allMatch(node -> node == null)) { + throw new CloudRuntimeException( + "Unable to determine whether the ONTAP cluster is AFF or FAS"); + } + for (ClusterNode node : response.getRecords()) { + if (node == null || !Boolean.TRUE.equals(node.getAllFlashOptimized())) { + return false; + } + } + return true; + } + + /** + * Pushes a single ASUP (AutoSupport) EMS application-log message to the ONTAP cluster. + * + *

This is strictly best-effort telemetry: any failure is logged and swallowed so that + * it can never affect a storage operation or the periodic scheduler.

+ * + * @param message the EMS message to send + */ + public void sendAsupMessage(EmsApplicationLog message) { + if (message == null) { + return; + } + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + emsFeignClient.sendEmsApplicationLog(authHeader, message); + logger.debug("sendAsupMessage: ASUP EMS message [event-id={}] sent to ONTAP cluster at {}", + message.getEventId(), storage.getStorageIP()); + } catch (Exception e) { + logger.error("sendAsupMessage: failed to send ASUP EMS message [event-id={}] to ONTAP cluster at {}: {}", + message.getEventId(), storage.getStorageIP(), e.getMessage()); + } } - // Connect method to validate ONTAP cluster, credentials, protocol, and SVM + /** + * Validates ONTAP cluster reachability, credentials, SVM state, protocol, and aggregate capacity + * for new FlexVol creation (primary pool provisioning). + */ public boolean connect() { + return connect(true); + } + + /** + * Validates ONTAP cluster reachability and SVM/protocol settings. + * + *

Aggregate free-space checks apply only when {@code validateAggregatesForVolumeCreation} is + * {@code true} (pool provisioning). Snapshot, delete, revert, and grant/revoke paths must use + * {@code false} — they operate on an existing FlexVol and must not compare aggregate space to + * the full pool capacity stored in pool details.

+ */ + public boolean connect(boolean validateAggregatesForVolumeCreation) { logger.info("Attempting to connect to ONTAP cluster at " + storage.getStorageIP() + " and validate SVM " + - storage.getSvmName() + ", protocol " + storage.getProtocol()); + storage.getSvmName() + ", protocol " + storage.getProtocol() + + (validateAggregatesForVolumeCreation ? " (with aggregate validation)" : " (operations only)")); String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); String svmName = storage.getSvmName(); try { @@ -131,47 +329,21 @@ public boolean connect() { logger.error("ISCSI protocol is not enabled on SVM " + svmName); throw new CloudRuntimeException("ISCSI protocol is not enabled on SVM " + svmName); } - List aggrs = svm.getAggregates(); - if (aggrs == null || aggrs.isEmpty()) { - logger.error("No aggregates are assigned to SVM " + svmName); - throw new CloudRuntimeException("No aggregates are assigned to SVM " + svmName); - } - // Collect all online aggregates assigned to the SVM. Capacity-based selection is - // intentionally deferred to createStorageVolume(name, size), which validates the - // available space against the actual requested volume size. - List eligibleAggregates = new ArrayList<>(); - for (Aggregate aggr : aggrs) { - logger.debug("Found aggregate: " + aggr.getName() + " with UUID: " + aggr.getUuid()); - Aggregate aggrResp = aggregateFeignClient.getAggregateByUUID(authHeader, aggr.getUuid()); - if (aggrResp == null) { - logger.warn("Aggregate details response is null for aggregate " + aggr.getName() + ". Skipping."); - continue; - } - if (!Objects.equals(aggrResp.getState(), Aggregate.StateEnum.ONLINE)) { - logger.warn("Aggregate " + aggr.getName() + " is not in online state. Skipping this aggregate."); - continue; - } - logger.debug("Aggregate " + aggr.getName() + " is online and eligible for volume operations."); - eligibleAggregates.add(aggr); - } - if (eligibleAggregates.isEmpty()) { - logger.error("No suitable aggregates found on SVM " + svmName + " for volume operations."); - throw new CloudRuntimeException("No suitable aggregates found on SVM " + svmName + " for volume operations."); + this.resolvedSvmUuid = svm.getUuid(); + + if (validateAggregatesForVolumeCreation) { + validateAndSelectAggregatesForVolumeCreation(authHeader, svmName, svm.getAggregates()); + } else { + logger.debug("Skipping aggregate capacity validation — not required for existing-volume operations"); } - this.aggregates = eligibleAggregates; - logger.info("Found " + eligibleAggregates.size() + " online aggregate(s) on SVM " + svmName + " for volume operations."); logger.info("Successfully connected to ONTAP cluster and validated ONTAP details provided"); + } catch (CloudRuntimeException e) { + throw e; } catch (FeignException.Unauthorized e) { - logger.error("Authentication failed while connecting to ONTAP cluster at " + storage.getStorageIP() + - ". Please verify the username and password.", e); - throw new CloudRuntimeException("Authentication failed: Invalid credentials for ONTAP cluster at " + - storage.getStorageIP() + ". Please verify the username and password."); - } catch (FeignException.Forbidden e) { - logger.error("Authorization failed while connecting to ONTAP cluster at " + storage.getStorageIP() + - ". The user does not have sufficient privileges.", e); - throw new CloudRuntimeException("Authorization failed: User does not have sufficient privileges on ONTAP cluster at " + - storage.getStorageIP() + ". Please verify user permissions."); + String msg = "Authentication failed: Invalid credentials. Please verify the username and password."; + logger.error(msg, e); + throw new CloudRuntimeException(msg, e); } catch (Exception e) { logger.error("Failed to connect to ONTAP cluster: " + e.getMessage(), e); throw new CloudRuntimeException("Failed to connect to ONTAP cluster: " + e.getMessage(), e); @@ -179,20 +351,71 @@ public boolean connect() { return true; } + /** + * ONTAP SVM UUID resolved during the last successful {@link #connect(boolean)} call. + */ + public String getResolvedSvmUuid() { + return resolvedSvmUuid; + } + + /** + * Aggregates eligible for new FlexVol creation, populated by {@link #connect(boolean)} with aggregate validation enabled. + */ + public List getAggregates() { + return aggregates; + } + + public void setAggregates(List aggregates) { + this.aggregates = aggregates; + } + + private void validateAndSelectAggregatesForVolumeCreation(String authHeader, String svmName, List aggrs) { + if (aggrs == null || aggrs.isEmpty()) { + logger.error("No aggregates are assigned to SVM " + svmName); + throw new CloudRuntimeException("No aggregates are assigned to SVM " + svmName); + } + List eligibleAggregates = new ArrayList<>(); + for (Aggregate aggr : aggrs) { + logger.debug("Found aggregate: " + aggr.getName() + " with UUID: " + aggr.getUuid()); + Aggregate aggrResp = aggregateFeignClient.getAggregateByUUID(authHeader, aggr.getUuid(), + Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.AGGREGATE_NODE + + OntapStorageConstants.COMMA + OntapStorageConstants.AGGREGATE_SPACE + + OntapStorageConstants.COMMA + OntapStorageConstants.STATE)); + if (aggrResp == null) { + logger.warn("Aggregate details response is null for aggregate " + aggr.getName() + ". Skipping."); + continue; + } + if (!Objects.equals(aggrResp.getState(), Aggregate.StateEnum.ONLINE)) { + logger.warn("Aggregate " + aggr.getName() + " is not in online state. Skipping this aggregate."); + continue; + } else if (aggrResp.getSpace() == null || aggrResp.getAvailableBlockStorageSpace() == null || + aggrResp.getAvailableBlockStorageSpace() <= storage.getSize().doubleValue()) { + logger.warn("Aggregate " + aggr.getName() + " does not have sufficient available space. Skipping this aggregate."); + continue; + } + logger.info("Selected aggregate: " + aggr.getName() + " for volume operations."); + eligibleAggregates.add(aggr); + } + if (eligibleAggregates.isEmpty()) { + logger.error("No suitable aggregates found on SVM " + svmName + " for volume creation."); + throw new CloudRuntimeException("No suitable aggregates found on SVM " + svmName + " for volume creation."); + } + setAggregates(eligibleAggregates); + } + // Common methods like create/delete etc., should be here /** - * Creates ONTAP Flex-Volume - * Eligible only for Unified ONTAP storage - * throw exception in case of disaggregated ONTAP storage + * Selects the best aggregate for a volume of the given size from the given candidate aggregates. * - * @param volumeName the name of the volume to create - * @param size the size of the volume in bytes - * @return the created Volume object + *

Picks the online aggregate with the largest available block space that can fit + * {@code size}. The returned aggregate includes node information for LIF affinity.

+ * + * @param aggregates candidate aggregates, for example {@link #getAggregates()} + * @param size requested volume size in bytes + * @return the chosen aggregate detail response */ - public Volume createStorageVolume(String volumeName, Long size) { - logger.info("Creating volume: " + volumeName + " of size: " + size + " bytes"); - + public Aggregate chooseAggregate(List aggregates, Long size) { String svmName = storage.getSvmName(); if (aggregates == null || aggregates.isEmpty()) { logger.error("No aggregates available to create volume on SVM " + svmName); @@ -203,23 +426,14 @@ public Volume createStorageVolume(String volumeName, Long size) { } String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); - - // Generate the Create Volume Request - Volume volumeRequest = new Volume(); - Svm svm = new Svm(); - svm.setName(svmName); - Nas nas = new Nas(); - nas.setPath(OntapStorageConstants.SLASH + volumeName); - - volumeRequest.setName(volumeName); - volumeRequest.setSvm(svm); - - // Pick the best aggregate for this specific request (largest available, online, and sufficient space). long maxAvailableAggregateSpaceBytes = -1L; Aggregate aggrChosen = null; for (Aggregate aggr : aggregates) { logger.debug("Found aggregate: " + aggr.getName() + " with UUID: " + aggr.getUuid()); - Aggregate aggrResp = aggregateFeignClient.getAggregateByUUID(authHeader, aggr.getUuid()); + Aggregate aggrResp = aggregateFeignClient.getAggregateByUUID(authHeader, aggr.getUuid(), + Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.AGGREGATE_NODE + + OntapStorageConstants.COMMA + OntapStorageConstants.AGGREGATE_SPACE + + OntapStorageConstants.COMMA + OntapStorageConstants.STATE)); if (aggrResp == null) { logger.warn("Aggregate details response is null for aggregate " + aggr.getName() + ". Skipping."); @@ -239,7 +453,7 @@ public Volume createStorageVolume(String volumeName, Long size) { final long availableBytes = aggrResp.getAvailableBlockStorageSpace().longValue(); logger.debug("Aggregate " + aggr.getName() + " available bytes=" + availableBytes + ", requested=" + size); - if (availableBytes < size) { + if (availableBytes <= size) { logger.warn("Aggregate " + aggr.getName() + " does not have sufficient available space. Required=" + size + " bytes, available=" + availableBytes + " bytes. Skipping this aggregate."); continue; @@ -247,7 +461,7 @@ public Volume createStorageVolume(String volumeName, Long size) { if (availableBytes > maxAvailableAggregateSpaceBytes) { maxAvailableAggregateSpaceBytes = availableBytes; - aggrChosen = aggr; + aggrChosen = aggrResp; } } @@ -255,11 +469,55 @@ public Volume createStorageVolume(String volumeName, Long size) { logger.error("No suitable aggregates found on SVM " + svmName + " for volume creation."); throw new CloudRuntimeException("No suitable aggregates found on SVM " + svmName + " for volume operations."); } - logger.info("Selected aggregate: " + aggrChosen.getName() + " for volume operations."); + if (aggrChosen.getNode() == null || aggrChosen.getNode().getName() == null + || aggrChosen.getNode().getName().isEmpty()) { + logger.error("Selected aggregate " + aggrChosen.getName() + " does not have a node name."); + throw new CloudRuntimeException("Selected aggregate " + aggrChosen.getName() + + " does not have a node name required for LIF affinity."); + } + logger.info("Selected aggregate: " + aggrChosen.getName() + " on node " + + aggrChosen.getNode().getName() + " for volume operations."); + return aggrChosen; + } + + /** + * Creates ONTAP Flex-Volume on the given aggregate. + * Eligible only for Unified ONTAP storage + * throw exception in case of disaggregated ONTAP storage + * + * @param volumeName the name of the volume to create + * @param size the size of the volume in bytes + * @param aggregate the aggregate previously selected via {@link #chooseAggregate(Long)} + * @return the created Volume object + */ + public Volume createStorageVolume(String volumeName, Long size, Aggregate aggregate) { + logger.info("Creating volume: " + volumeName + " of size: " + size + " bytes"); + + String svmName = storage.getSvmName(); + if (size == null || size <= 0) { + throw new CloudRuntimeException("Invalid volume size provided: " + size); + } + if (aggregate == null || aggregate.getName() == null || aggregate.getUuid() == null) { + throw new CloudRuntimeException("Aggregate is required to create volume on SVM " + svmName); + } + + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + + // Generate the Create Volume Request + Volume volumeRequest = new Volume(); + Svm svm = new Svm(); + svm.setName(svmName); + Nas nas = new Nas(); + nas.setPath(OntapStorageConstants.SLASH + volumeName); + + volumeRequest.setName(volumeName); + volumeRequest.setSvm(svm); + + logger.info("Creating volume on aggregate: " + aggregate.getName() + " for volume operations."); Aggregate aggr = new Aggregate(); - aggr.setName(aggrChosen.getName()); - aggr.setUuid(aggrChosen.getUuid()); + aggr.setName(aggregate.getName()); + aggr.setUuid(aggregate.getUuid()); volumeRequest.setAggregates(List.of(aggr)); volumeRequest.setSize(size); volumeRequest.setNas(nas); @@ -271,7 +529,7 @@ public Volume createStorageVolume(String volumeName, Long size) { } String jobUUID = jobResponse.getJob().getUuid(); - Boolean jobSucceeded = jobPollForSuccess(jobUUID,10, 1000); + Boolean jobSucceeded = jobPollForSuccess(jobUUID,OntapStorageConstants.ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES, OntapStorageConstants.ONTAP_FLEXVOL_JOB_POLL_INTERVAL_MS); if (!jobSucceeded) { logger.error("Volume creation job failed for volume: " + volumeName); throw new CloudRuntimeException("Volume creation job failed for volume: " + volumeName); @@ -339,7 +597,25 @@ public Volume createStorageVolume(String volumeName, Long size) { * @return the updated Volume object */ public Volume updateStorageVolume(Volume volume) { - return null; + logger.info("Resizing ONTAP FlexVolume '{}' (UUID: {}) to {} bytes", volume.getName(), volume.getUuid(), volume.getSize()); + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + try { + Volume resizeRequest = new Volume(); + resizeRequest.setSize(volume.getSize()); + JobResponse jobResponse = volumeFeignClient.updateVolume(authHeader, volume.getUuid(), resizeRequest); + pollJobIfPresent(jobResponse, "resize FlexVolume [" + volume.getUuid() + "]", + OntapStorageConstants.ONTAP_VOLUME_JOB_MAX_RETRIES, OntapStorageConstants.ONTAP_VOLUME_JOB_POLL_INTERVAL_MS); + logger.info("FlexVolume '{}' (UUID: {}) resized successfully to {} bytes", volume.getName(), volume.getUuid(), volume.getSize()); + } catch (FeignException e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + String msg = String.format("Cannot resize FlexVolume '%s' (UUID: %s): volume not found on ONTAP (404). ", volume.getName(), volume.getUuid()); + logger.error(msg); + throw new CloudRuntimeException(msg, e); + } + logger.error("Exception while resizing FlexVolume '{}' (UUID: {}): {}", volume.getName(), volume.getUuid(), e.getMessage(), e); + throw new CloudRuntimeException("Failed to resize ONTAP FlexVolume: " + e.getMessage(), e); + } + return volume; } /** @@ -354,13 +630,17 @@ public void deleteStorageVolume(Volume volume) { String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); try { JobResponse jobResponse = volumeFeignClient.deleteVolume(authHeader, volume.getUuid()); - Boolean jobSucceeded = jobPollForSuccess(jobResponse.getJob().getUuid(), 10, 1000); + Boolean jobSucceeded = jobPollForSuccess(jobResponse.getJob().getUuid(), OntapStorageConstants.ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES, OntapStorageConstants.ONTAP_FLEXVOL_JOB_POLL_INTERVAL_MS); if (!jobSucceeded) { logger.error("Volume deletion job failed for volume: " + volume.getName()); throw new CloudRuntimeException("Volume deletion job failed for volume: " + volume.getName()); } logger.info("Volume deleted successfully: " + volume.getName()); - } catch (FeignException.FeignClientException e) { + } catch (FeignException e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + logger.warn("deleteStorageVolume: Volume '{}' not found in ONTAP, treating as no-op", volume.getName()); + return; + } logger.error("Exception while deleting volume: ", e); throw new CloudRuntimeException("Failed to delete volume: " + e.getMessage()); } @@ -368,15 +648,38 @@ public void deleteStorageVolume(Volume volume) { } /** - * Gets ONTAP Flex-Volume - * Eligible only for Unified ONTAP storage - * throw exception in case of disaggregated ONTAP storage + * Gets an ONTAP FlexVolume by UUID. + * Eligible only for Unified ONTAP storage. + * Throws exception in case of disaggregated ONTAP storage. + * + *

Callers pass ONTAP query parameters when they need a partial record. For example, + * {@code fields=space.used} returns only used space instead of the full volume. + * A null or empty map returns the default volume representation.

* - * @param volume the volume to retrieve - * @return the retrieved Volume object + * @param uuid the UUID of the volume to retrieve + * @param queryParams ONTAP query parameters applied to {@code GET /storage/volumes/{uuid}}; + * may be null + * @return the retrieved Volume object, or null if not found */ - public Volume getStorageVolume(Volume volume) { - return null; + public Volume getStorageVolume(String uuid, Map queryParams) { + if (StringUtils.isBlank(uuid)) { + throw new CloudRuntimeException("Cannot fetch ONTAP volume: UUID is null or empty"); + } + Map params = queryParams != null ? queryParams : Collections.emptyMap(); + logger.info("getStorageVolume: Fetching ONTAP volume [{}] with query params {}", uuid, params); + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + try { + Volume fetchedVolume = volumeFeignClient.getVolumeByUUID(authHeader, uuid, params); + logger.info("getStorageVolume: Volume [{}] fetched successfully", uuid); + return fetchedVolume; + } catch (FeignException e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + logger.warn("getStorageVolume: Volume [{}] not found in ONTAP", uuid); + return null; + } + logger.error("getStorageVolume: Exception while fetching volume [{}]: ", uuid, e); + throw new CloudRuntimeException("Failed to fetch volume: " + e.getMessage()); + } } /** @@ -430,12 +733,27 @@ public String getStoragePath() { /** - * Get the network ip interface + * Selects the best available data LIF for storage I/O, preferring one homed on the same node + * as the given aggregate to avoid inter-node traffic. + * + *

Selection order:

+ *
    + *
  1. LIF whose {@code location.home_node} matches the aggregate's node — no warning
    2. + *
  2. LIF currently running on that node (e.g. after failover) — returned with a warning
    3. + *
  3. Any UP and enabled LIF — returned with a warning
    4. + *
* - * @return the network interface ip as a String + * @param aggregate the aggregate previously selected via {@link #chooseAggregate(Long)}; + * must include a node name for LIF affinity + * @return map with {@link OntapStorageConstants#DATA_LIF} set to the LIF IP address, and + * optionally {@link OntapStorageConstants#LIF_WARNING} when a non-ideal LIF was selected */ - - public String getNetworkInterface() { + public Map getNetworkInterface(Aggregate aggregate) { + if (aggregate == null || aggregate.getNode() == null || aggregate.getNode().getName() == null + || aggregate.getNode().getName().isEmpty()) { + throw new CloudRuntimeException("Aggregate with a node name is required to select a network interface"); + } + String aggregateNode = aggregate.getNode().getName(); String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); try { Map queryParams = new HashMap<>(); @@ -453,36 +771,99 @@ public String getNetworkInterface() { throw new CloudRuntimeException("Unsupported protocol: " + storage.getProtocol()); } } - queryParams.put(OntapStorageConstants.FIELDS, OntapStorageConstants.IP_ADDRESS); + queryParams.put(OntapStorageConstants.FIELDS, + OntapStorageConstants.IP_ADDRESS + OntapStorageConstants.COMMA + + OntapStorageConstants.STATE + OntapStorageConstants.COMMA + + OntapStorageConstants.LIF_ENABLED + OntapStorageConstants.COMMA + + OntapStorageConstants.LIF_LOCATION_HOME_NODE + OntapStorageConstants.COMMA + + OntapStorageConstants.LIF_LOCATION_NODE); queryParams.put(OntapStorageConstants.RETURN_RECORDS, OntapStorageConstants.TRUE); OntapResponse response = networkFeignClient.getNetworkIpInterfaces(authHeader, queryParams); - if (response != null && response.getRecords() != null && !response.getRecords().isEmpty()) { - IpInterface ipInterface = null; - // For simplicity, return the first interface's name (Of IPv4 type for NFS3) - if (storage.getProtocol() == ProtocolType.ISCSI) { - ipInterface = response.getRecords().get(0); - } else if (storage.getProtocol() == ProtocolType.NFS3) { - for (IpInterface iface : response.getRecords()) { - if (iface.getIp().getAddress().contains(".")) { - ipInterface = iface; - break; - } + if (response == null || response.getRecords() == null || response.getRecords().isEmpty()) { + throw new CloudRuntimeException("No network interfaces found for SVM " + storage.getSvmName() + + " for protocol " + storage.getProtocol()); + } + + IpInterface currentNodeInterface = null; + IpInterface fallbackInterface = null; + + for (IpInterface iface : response.getRecords()) { + if (iface == null || iface.getIp() == null) { + continue; + } + if (!Boolean.TRUE.equals(iface.getEnabled()) || !OntapStorageConstants.LIF_STATE_UP.equals(iface.getState())) { + continue; + } + if (!isIPv4Address(iface.getIp().getAddress())) { + continue; + } + // LIF is homed on the aggregate's node + String homeNode = iface.getLocation() != null && iface.getLocation().getHomeNode() != null + ? iface.getLocation().getHomeNode().getName() : null; + if (aggregateNode.equals(homeNode)) { + return networkInterfaceResult(iface.getIp().getAddress(), null); + } + // LIF has failed over and is currently running on the aggregate's node + // (home_node differs). Keep as a candidate; returned with a warning if no match is found earlier. + if (currentNodeInterface == null) { + String currentNode = iface.getLocation() != null && iface.getLocation().getNode() != null + ? iface.getLocation().getNode().getName() : null; + if (aggregateNode.equals(currentNode)) { + currentNodeInterface = iface; } } + if (fallbackInterface == null) { + fallbackInterface = iface; + } + } - logger.info("Retrieved network interface: " + ipInterface.getIp().getAddress()); - return ipInterface.getIp().getAddress(); - } else { - throw new CloudRuntimeException("No network interfaces found for SVM " + storage.getSvmName() + - " for protocol " + storage.getProtocol()); + if (currentNodeInterface == null && fallbackInterface == null) { + throw new CloudRuntimeException("No operationally UP and enabled LIF found for SVM '" + + storage.getSvmName() + "' with protocol " + storage.getProtocol() + + " — all " + response.getRecords().size() + " LIF(s) are either administratively disabled or operationally down"); } - } catch (FeignException.FeignClientException e) { + + if (currentNodeInterface != null) { + String ip = currentNodeInterface.getIp().getAddress(); + String warning = "No home-node LIF found for aggregate node '" + aggregateNode + + "'; using LIF '" + ip + "' currently running on that node (home node LIF may be down)."; + logger.warn(warning); + return networkInterfaceResult(ip, warning); + } + + String ip = fallbackInterface.getIp().getAddress(); + String warning = "No operational LIF found on aggregate's home node '" + aggregateNode + + "'; using fallback LIF '" + ip + "' on a different node." + + " I/O will traverse an inter-node path, increasing latency."; + logger.warn(warning); + return networkInterfaceResult(ip, warning); + } catch (CloudRuntimeException e) { + throw e; + } catch (Exception e) { logger.error("Exception while retrieving network interfaces: ", e); throw new CloudRuntimeException("Failed to retrieve network interfaces: " + e.getMessage()); } } + private Map networkInterfaceResult(String address, String warning) { + Map result = new HashMap<>(); + result.put(OntapStorageConstants.DATA_LIF, address); + if (warning != null) { + result.put(OntapStorageConstants.LIF_WARNING, warning); + } + return result; + } + + /** + * Returns true if the given IP address string is an IPv4 address. + * IPv6 addresses contain colons; IPv4 addresses do not. + * To extend LIF selection to support IPv6, update this method and its call site in getNetworkInterface(). + */ + private boolean isIPv4Address(String address) { + return address != null && !address.contains(":"); + } + /** * Method encapsulates the behavior based on the opted protocol in subclasses. * it is going to mimic @@ -495,6 +876,27 @@ public String getNetworkInterface() { */ abstract public CloudStackVolume createCloudStackVolume(CloudStackVolume cloudstackVolume); + /** + * Creates the protocol-specific backend object that caches a template on this pool. + * + *

iSCSI creates an empty LUN ({@code /vol/<flexVol>/cs_tmpl_<id>}) sized to + * {@code sizeInBytes}. NFS is a no-op on the array: the KVM agent later writes the qcow2 + * into the mounted FlexVolume.

+ * + *

Returns a {@link CloudStackVolume} so the driver can map it to {@code CreateCmdResult} + * and update {@code template_spool_ref}. SAN populates {@code lun}; NAS returns an empty + * volume (no LUN / file yet).

+ * + * @param storagePool CloudStack primary storage pool (one FlexVolume) + * @param templateInfo template being cached + * @param details pool details (SVM, protocol, etc.) + * @param sizeInBytes virtual size for the cache object (required for SAN; ignored for NAS) + * @return created cache identity, or an empty {@link CloudStackVolume} when nothing is + * pre-created on the array + */ + abstract public CloudStackVolume createTemplateCache(StoragePoolVO storagePool, TemplateInfo templateInfo, + Map details, long sizeInBytes); + /** * Method encapsulates the behavior based on the opted protocol in subclasses. * it is going to mimic @@ -505,7 +907,7 @@ public String getNetworkInterface() { * @param cloudstackVolume the CloudStack volume to update * @return the updated CloudStackVolume object */ - abstract CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume); + public abstract CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume); /** * Method encapsulates the behavior based on the opted protocol in subclasses. @@ -519,14 +921,42 @@ public String getNetworkInterface() { abstract public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume); /** - * Method encapsulates the behavior based on the opted protocol in subclasses. + * Creates a space-efficient clone of an existing object inside the same FlexVolume. * it is going to mimic * cloneLun for iSCSI, FC protocols * cloneFile for NFS3.0 and NFS4.1 protocols * cloneNameSpace for Nvme/TCP and Nvme/FC protocol - * @param cloudstackVolume the CloudStack volume to copy + * + *

ONTAP requires the source and the destination to live in the same FlexVolume, which + * holds because a CloudStack primary storage pool maps one-to-one onto a FlexVolume.

+ * + *

The source may be an active object or one inside a FlexVolume snapshot:

+ *
    + *
  • SAN — {@code lun.clone.source.name} set to + * {@code /vol/<fv>/.snapshot/<snap>/<lun>}
    • + *
  • NAS — {@code file.path} set to + * {@code ./.snapshot/<snapshot-name>/<source-file>} and sent as + * {@code source_path} on {@code POST /api/storage/file/clone}
    • + *
+ * + * @param cloudstackVolume describes the clone to create; the source is carried in the + * protocol-specific clone reference (for SAN, {@code lun.clone.source}) + * @return the created CloudStackVolume, populated with the backend identity of the clone */ - abstract public void copyCloudStackVolume(CloudStackVolume cloudstackVolume); + abstract public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume); + + /** + * Grows an existing backend object to {@code sizeInBytes}. + * + *

Needed after cloning a cached template, because a clone inherits the size of its source + * while the service offering may ask for a larger disk.

+ * + *

When the request already carries a QoS policy on the LUN or file, that policy is applied + * as part of this resize. iSCSI sends one LUN update with the new size and the policy. + * NFS grows the file on the host, then updates the file policy. A request with no policy + * changes size only.

+ */ + abstract public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes); /** * Method encapsulates the behavior based on the opted protocol in subclasses. @@ -540,15 +970,13 @@ public String getNetworkInterface() { abstract public CloudStackVolume getCloudStackVolume(Map cloudStackVolumeMap); /** - * Reverts a CloudStack volume to a snapshot using protocol-specific ONTAP APIs. + * Reverts a CloudStack volume to a snapshot using ONTAP CLI-based Single File Snap Restore (SFSR). * - *

This method encapsulates the snapshot revert behavior based on protocol:

- *
    - *
  • iSCSI/FC: Uses {@code POST /api/storage/luns/{lun.uuid}/restore} - * to restore LUN data from the FlexVolume snapshot.
    • - *
  • NFS: Uses {@code POST /api/storage/volumes/{vol.uuid}/snapshots/{snap.uuid}/files/{path}/restore} - * to restore a single file from the FlexVolume snapshot.
    • - *
+ *

Both NFS and iSCSI use the CLI passthrough API: + * {@code POST /api/private/cli/volume/snapshot/restore-file}

+ * + *

Callers should invoke {@link #executeCliSfsrRestore(JobResponse, String)} after this + * method returns to poll the async job when present, or treat a missing job as synchronous success.

* * @param snapshotName The ONTAP FlexVolume snapshot name * @param flexVolUuid The FlexVolume UUID containing the snapshot @@ -590,7 +1018,7 @@ public abstract JobResponse revertSnapshotForCloudStackVolume(String snapshotNam * @param accessGroup the access group to update * @return the updated AccessGroup object */ - abstract AccessGroup updateAccessGroup(AccessGroup accessGroup); + public abstract AccessGroup updateAccessGroup(AccessGroup accessGroup); /** * Method encapsulates the behavior based on the opted protocol in subclasses @@ -650,16 +1078,128 @@ public String getAuthHeader() { return OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); } + public VolumeQosPolicy createVolumeQosPolicy(String policyName, Long minIops, Long maxIops) { + VolumeQosPolicy policy = buildVolumeQosPolicy(policyName, minIops, maxIops); + Svm svm = new Svm(); + svm.setName(storage.getSvmName()); + policy.setSvm(svm); + try { + JobResponse response = qosFeignClient.createPolicy(getAuthHeader(), policy); + pollJobIfPresent(response, "create QoS policy [" + policyName + "]"); + } catch (FeignException e) { + if (e.status() != 409) { + throw new CloudRuntimeException("Failed to create ONTAP QoS policy [" + policyName + "]: " + + e.getMessage(), e); + } + logger.info("QoS policy [{}] already exists; using the existing policy", policyName); + } + + VolumeQosPolicy createdPolicy = getVolumeQosPolicy(policyName); + return createdPolicy; + } + + public void deleteVolumeQosPolicy(String policyUuid) { + if (policyUuid == null || policyUuid.isEmpty()) { + return; + } + VolumeQosPolicy policy = getVolumeQosPolicyByUuid(policyUuid); + if (policy == null) { + logger.info("QoS policy [{}] not found on ONTAP; skipping delete", policyUuid); + return; + } + if (policy.getObjectCount() != null && policy.getObjectCount() > 0) { + logger.info("QoS policy [{}] still has object_count={}; skipping delete", + policyUuid, policy.getObjectCount()); + return; + } + try { + JobResponse response = qosFeignClient.deletePolicy(getAuthHeader(), policyUuid); + pollJobIfPresent(response, "delete QoS policy [" + policyUuid + "]"); + } catch (Exception e) { + if ((e instanceof FeignException && ((FeignException) e).status() == 409) + || OntapStorageUtils.isOntapObjectNotFoundError(e)) { + logger.info("QoS policy [{}] was not deleted on ONTAP (already absent or conflict): {}", + policyUuid, e.getMessage()); + return; + } + throw new CloudRuntimeException("Failed to delete ONTAP QoS policy [" + policyUuid + "]: " + + e.getMessage(), e); + } + } + + private VolumeQosPolicy getVolumeQosPolicyByUuid(String policyUuid) { + Map queryParams = new HashMap<>(); + queryParams.put(OntapStorageConstants.FIELDS, OntapStorageConstants.QOS_POLICY_OBJECT_COUNT_FIELDS); + try { + VolumeQosPolicy policy = qosFeignClient.getPolicy(getAuthHeader(), policyUuid, queryParams); + if (policy == null || StringUtils.isEmpty(policy.getUuid())) { + throw new CloudRuntimeException("Failed to fetch ONTAP QoS policy [" + policyUuid + + "]: empty response"); + } + return policy; + } catch (FeignException e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + return null; + } + throw new CloudRuntimeException("Failed to fetch ONTAP QoS policy [" + policyUuid + "]: " + + e.getMessage(), e); + } + } + + private VolumeQosPolicy getVolumeQosPolicy(String policyName) { + Map queryParams = new HashMap<>(); + queryParams.put(OntapStorageConstants.NAME, policyName); + queryParams.put(OntapStorageConstants.SVM_DOT_NAME, storage.getSvmName()); + try { + OntapResponse response = qosFeignClient.getPolicies(getAuthHeader(), queryParams); + if (response == null || response.getRecords() == null || response.getRecords().size() <= 0) { + throw new CloudRuntimeException("Unable to get ONTAP QoS policy [" + policyName + "] after creation"); + } + VolumeQosPolicy policy = response.getRecords().get(0); + if (policy == null || StringUtils.isEmpty(policy.getUuid())) { + throw new CloudRuntimeException("Failed to fetch ONTAP QoS policy [" + policyName + + "]: empty response"); + } + return policy; + } catch (FeignException e) { + throw new CloudRuntimeException("Failed to fetch ONTAP QoS policy [" + policyName + "]: " + + e.getMessage(), e); + } + } + + private VolumeQosPolicy buildVolumeQosPolicy(String policyName, Long minIops, Long maxIops) { + VolumeQosPolicy.Fixed fixed = new VolumeQosPolicy.Fixed(); + fixed.setCapacityShared(false); + // ONTAP rejects a policy whose throughput limit is zero, so only unlimited-side values are omitted. + if (minIops != null && minIops > 0) { + fixed.setMinThroughputIops(minIops); + } + if (maxIops != null && maxIops > 0) { + fixed.setMaxThroughputIops(maxIops); + } + + VolumeQosPolicy policy = new VolumeQosPolicy(); + policy.setName(policyName); + policy.setFixed(fixed); + return policy; + } + /** * Polls an ONTAP async job for successful completion. * * @param jobUUID UUID of the ONTAP job to poll * @param maxRetries maximum number of poll attempts - * @param sleepTimeInMilliSecs seconds to sleep between poll attempts + * @param sleepTimeInMilliSecs sleep between poll attempts * @return true if the job completed successfully */ public Boolean jobPollForSuccess(String jobUUID, int maxRetries, int sleepTimeInMilliSecs) { - //Create URI for GET Job API + return jobPollUntilSuccess(jobUUID, maxRetries, sleepTimeInMilliSecs) != null; + } + + /** + * Polls an ONTAP async job until it succeeds and returns the completed job record. + */ + public Job jobPollUntilSuccess(String jobUUID, int maxRetries, int sleepTimeInMilliSecs) { int jobRetryCount = 0; Job jobResp = null; try { @@ -684,14 +1224,125 @@ public Boolean jobPollForSuccess(String jobUUID, int maxRetries, int sleepTimeIn jobRetryCount++; Thread.sleep(sleepTimeInMilliSecs); } - if (jobResp == null || !jobResp.getState().equals(OntapStorageConstants.JOB_SUCCESS)) { - return false; - } + return jobResp; } catch (FeignException.FeignClientException e) { throw new CloudRuntimeException("Failed to fetch job status: " + e.getMessage()); } catch (InterruptedException e) { - throw new RuntimeException(e); + Thread.currentThread().interrupt(); + throw new CloudRuntimeException("Interrupted while polling ONTAP job " + jobUUID, e); + } + } + + /** + * Polls an ONTAP async job when the API response includes a job reference. + * + *

When no job is returned (common for CLI passthrough SFSR on synchronous completion), + * the operation is treated as successful after HTTP 2xx.

+ * + * @param response ONTAP job response (may be null or without a job) + * @param operationName label for logging and error messages + */ + public void pollJobIfPresent(JobResponse response, String operationName) { + pollJobIfPresent(response, operationName, + OntapStorageConstants.ONTAP_CG_JOB_MAX_RETRIES, + OntapStorageConstants.ONTAP_CG_JOB_POLL_INTERVAL_MS); + } + + /** + * Polls an ONTAP async job when present, using caller-supplied retry settings. + */ + public void pollJobIfPresent(JobResponse response, String operationName, + int maxRetries, int pollIntervalMs) { + if (response == null || response.getJob() == null || response.getJob().getUuid() == null) { + logger.debug("pollJobIfPresent: No async job returned for operation [{}], continuing without polling", + operationName); + return; + } + jobPollForSuccess(response.getJob().getUuid(), maxRetries, pollIntervalMs); + } + + /** + * Polls an ONTAP async job when present and returns the completed job (for extracting created resource UUIDs). + */ + public Job pollJobIfPresentAndGetCompletedJob(JobResponse response, String operationName) { + return pollJobIfPresentAndGetCompletedJob(response, operationName, + OntapStorageConstants.ONTAP_CG_JOB_MAX_RETRIES, + OntapStorageConstants.ONTAP_CG_JOB_POLL_INTERVAL_MS); + } + + public Job pollJobIfPresentAndGetCompletedJob(JobResponse response, String operationName, + int maxRetries, int pollIntervalMs) { + if (response == null || response.getJob() == null || response.getJob().getUuid() == null) { + logger.debug("pollJobIfPresentAndGetCompletedJob: No async job for operation [{}]", operationName); + return null; + } + return jobPollUntilSuccess(response.getJob().getUuid(), maxRetries, pollIntervalMs); + } + + /** + * Completes CLI-based SFSR ({@code restore-file}) orchestration: poll job when returned, + * otherwise accept synchronous success. + */ + public void executeCliSfsrRestore(JobResponse response, String operationName) { + pollJobIfPresent(response, operationName, + OntapStorageConstants.ONTAP_SFSR_JOB_MAX_RETRIES, + OntapStorageConstants.ONTAP_SFSR_JOB_POLL_INTERVAL_MS); + } + + /** + * Deletes a FlexVolume snapshot on ONTAP for a CloudStack volume snapshot. + * + *

ONTAP volume snapshots (NFS and iSCSI) are FlexVol-level snapshots created by + * {@code POST /storage/volumes/{uuid}/snapshots} during take. Delete uses the matching + * REST {@code DELETE /storage/volumes/{uuid}/snapshots/{snapshot.uuid}} API regardless + * of whether the CloudStack volume is a file (NFS) or LUN (iSCSI). Protocol-specific + * subclasses ({@code UnifiedNASStrategy}, {@code UnifiedSANStrategy}) inherit this + * implementation; revert/restore remains protocol-specific via SFSR CLI.

+ * + *

Called from {@link org.apache.cloudstack.storage.driver.OntapPrimaryDatastoreDriver} + * during the standard delete chain — not from a separate ONTAP snapshot strategy.

+ * + * @param flexVolUuid ONTAP FlexVolume UUID + * @param snapshotUuid ONTAP FlexVolume snapshot UUID + * @param snapshotName ONTAP FlexVolume snapshot name (for logging) + */ + public void deleteFlexVolSnapshotForCloudStackVolume(String flexVolUuid, String snapshotUuid, String snapshotName) { + if (flexVolUuid == null || flexVolUuid.isEmpty() || snapshotUuid == null || snapshotUuid.isEmpty()) { + throw new CloudRuntimeException("FlexVolume UUID and snapshot UUID are required to delete an ONTAP snapshot"); + } + + logger.info("deleteFlexVolSnapshotForCloudStackVolume: issuing ONTAP REST delete for snapshot [{}] " + + "(uuid={}) on FlexVol [{}]", snapshotName, snapshotUuid, flexVolUuid); + + try { + JobResponse jobResponse = snapshotFeignClient.deleteSnapshot(getAuthHeader(), flexVolUuid, snapshotUuid); + + if (jobResponse == null || jobResponse.getJob() == null) { + logger.debug("deleteFlexVolSnapshotForCloudStackVolume: no async job returned for snapshot [{}] " + + "(uuid={}); treating HTTP success as completion", snapshotName, snapshotUuid); + } else { + logger.debug("deleteFlexVolSnapshotForCloudStackVolume: polling ONTAP delete job [{}] for snapshot [{}]", + jobResponse.getJob().getUuid(), snapshotName); + } + + pollJobIfPresent(jobResponse, "delete FlexVol snapshot [" + snapshotName + "] uuid [" + snapshotUuid + "]", + OntapStorageConstants.ONTAP_SNAPSHOT_DELETE_JOB_MAX_RETRIES, + OntapStorageConstants.ONTAP_SNAPSHOT_DELETE_JOB_POLL_INTERVAL_MS); + + logger.info("deleteFlexVolSnapshotForCloudStackVolume: ONTAP FlexVol snapshot [{}] (uuid={}) removed from [{}]", + snapshotName, snapshotUuid, flexVolUuid); + } catch (Exception e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + logger.warn("deleteFlexVolSnapshotForCloudStackVolume: ONTAP snapshot [{}] (uuid={}) on FlexVol [{}] " + + "already absent; treating delete as success: {}", snapshotName, snapshotUuid, flexVolUuid, + e.getMessage()); + return; + } + if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException) e; + } + throw new CloudRuntimeException("Failed to delete ONTAP FlexVol snapshot [" + snapshotName + "]: " + + e.getMessage(), e); } - return true; } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedNASStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedNASStrategy.java index 198957ca5db8..7973412feb5e 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedNASStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedNASStrategy.java @@ -19,48 +19,62 @@ package org.apache.cloudstack.storage.service; -import com.cloud.agent.api.Answer; -import com.cloud.host.HostVO; -import com.cloud.storage.Storage; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.utils.exception.CloudRuntimeException; -import feign.FeignException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.CliSnapshotRestoreRequest; import org.apache.cloudstack.storage.feign.model.ExportPolicy; import org.apache.cloudstack.storage.feign.model.ExportRule; +import org.apache.cloudstack.storage.feign.model.FileCloneRequest; import org.apache.cloudstack.storage.feign.model.FileInfo; import org.apache.cloudstack.storage.feign.model.Job; import org.apache.cloudstack.storage.feign.model.Nas; import org.apache.cloudstack.storage.feign.model.OntapStorage; import org.apache.cloudstack.storage.feign.model.Svm; import org.apache.cloudstack.storage.feign.model.Volume; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; -import org.apache.cloudstack.storage.feign.model.CliSnapshotRestoreRequest; import org.apache.cloudstack.storage.service.model.AccessGroup; import org.apache.cloudstack.storage.service.model.CloudStackVolume; -import org.apache.cloudstack.storage.volume.VolumeObject; import org.apache.cloudstack.storage.utils.OntapStorageConstants; import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.apache.cloudstack.storage.volume.VolumeObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.ResizeVolumeCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.host.HostVO; +import com.cloud.storage.ResizeVolumePayload; +import com.cloud.storage.Storage; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; + +import feign.FeignException; public class UnifiedNASStrategy extends NASStrategy { private static final Logger logger = LogManager.getLogger(UnifiedNASStrategy.class); @Inject private VolumeDao volumeDao; @Inject private EndPointSelector epSelector; @Inject private StoragePoolDetailsDao storagePoolDetailsDao; + @Inject private PrimaryDataStoreDao primaryDataStoreDao; public UnifiedNASStrategy(OntapStorage ontapStorage) { super(ontapStorage); @@ -83,18 +97,69 @@ public CloudStackVolume createCloudStackVolume(CloudStackVolume cloudstackVolume logger.error("createCloudStackVolume: " + errMsg); throw new CloudRuntimeException(errMsg); } + if (cloudstackVolume.getFile() != null && cloudstackVolume.getFile().getQosPolicy() != null) { + try { + updateCloudStackVolume(cloudstackVolume); + } catch (RuntimeException qosError) { + logger.error("createCloudStackVolume: QoS attach failed; deleting leftover NFS volume file", qosError); + try { + Answer cleanup = deleteVolumeOnKVMHost(cloudstackVolume.getVolumeInfo()); + if (cleanup == null || !cleanup.getResult()) { + logger.error("createCloudStackVolume: leftover NFS file may remain after QoS attach failure: {}", + cleanup != null ? cleanup.getDetails() : "null answer"); + } + } catch (Exception cleanupError) { + logger.error("createCloudStackVolume: failed to delete leftover NFS volume file after QoS attach failure", + cleanupError); + } + throw qosError; + } + } return cloudstackVolume; - }catch (Exception e) { + } catch (CloudRuntimeException e) { + throw e; + } catch (Exception e) { logger.error("createCloudStackVolume: error occured " + e); throw new CloudRuntimeException(e); } } + /** + * NFS template cache: nothing is pre-created on the array. The KVM agent writes the qcow2 + * into the mounted FlexVolume; the framework records {@code install_path} afterward. + */ @Override - CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume) { + public CloudStackVolume createTemplateCache(StoragePoolVO storagePool, TemplateInfo templateInfo, + Map details, long sizeInBytes) { + logger.info("createTemplateCache: NFS pool [{}], template [{}] will be written directly to the mounted FlexVolume", + storagePool.getId(), templateInfo.getId()); return null; } + @Override + public CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume) { + if (cloudstackVolume == null || cloudstackVolume.getVolumeInfo() == null + || cloudstackVolume.getFlexVolumeUuid() == null || cloudstackVolume.getFile() == null) { + throw new CloudRuntimeException("Invalid NFS volume request"); + } + VolumeQosPolicy qosPolicy = cloudstackVolume.getFile().getQosPolicy(); + if (qosPolicy == null) { + return cloudstackVolume; + } + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicy); + String filePath = cloudstackVolume.getVolumeInfo().getUuid(); + try { + nasFeignClient.updateFile(getAuthHeader(), cloudstackVolume.getFlexVolumeUuid(), filePath, fileInfo); + } catch (FeignException e) { + throw new CloudRuntimeException("Failed to update NFS volume file: " + e.getMessage(), e); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to update NFS volume file: " + e.getMessage(), e); + } + logger.info("Updated NFS volume file [{}] with QoS policy [{}]", filePath, qosPolicy.getName()); + return cloudstackVolume; + } + @Override public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) { logger.info("deleteCloudStackVolume: Delete cloudstack volume " + cloudstackVolume); @@ -112,9 +177,114 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) { } } + /** + * Clones a file inside the FlexVolume using ONTAP's file clone API. + * + *

The source is taken from {@code file.path} and the destination from + * {@code destinationPath}, both relative to the root of the FlexVolume backing the pool.

+ */ @Override - public void copyCloudStackVolume(CloudStackVolume cloudstackVolume) { + public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume) { + if (cloudstackVolume == null || cloudstackVolume.getFile() == null + || cloudstackVolume.getFile().getPath() == null || cloudstackVolume.getDestinationPath() == null) { + logger.error("cloneCloudStackVolume: File clone failed. Invalid request: {}", cloudstackVolume); + throw new CloudRuntimeException("Failed to clone file, invalid request"); + } + if (cloudstackVolume.getDatastoreId() == null) { + throw new CloudRuntimeException("Failed to clone file, no datastore id in the request"); + } + + Map details = storagePoolDetailsDao.listDetailsKeyPairs(Long.parseLong(cloudstackVolume.getDatastoreId())); + String flexVolUuid = details.get(OntapStorageConstants.VOLUME_UUID); + String flexVolName = details.get(OntapStorageConstants.VOLUME_NAME); + if (flexVolUuid == null || flexVolUuid.isEmpty()) { + throw new CloudRuntimeException("Failed to clone file, FlexVolume uuid is missing from pool poolDetails"); + } + String sourcePath = cloudstackVolume.getFile().getPath(); + String destinationPath = cloudstackVolume.getDestinationPath(); + + logger.info("cloneCloudStackVolume: Cloning file [{}] to [{}] in FlexVol [{}]", + sourcePath, destinationPath, flexVolName); + try { + FileCloneRequest request = new FileCloneRequest(flexVolUuid, flexVolName, sourcePath, destinationPath); + JobResponse jobResponse = nasFeignClient.cloneFile(getAuthHeader(), request); + pollJobIfPresent(jobResponse, "clone file [" + sourcePath + "] to [" + destinationPath + "]"); + + updateCloudStackVolumeMetadata(cloudstackVolume.getDatastoreId(), cloudstackVolume.getVolumeInfo()); + + FileInfo clonedFile = new FileInfo(); + clonedFile.setPath(destinationPath); + CloudStackVolume clonedCloudStackVolume = new CloudStackVolume(); + clonedCloudStackVolume.setFile(clonedFile); + clonedCloudStackVolume.setDatastoreId(cloudstackVolume.getDatastoreId()); + clonedCloudStackVolume.setVolumeInfo(cloudstackVolume.getVolumeInfo()); + return clonedCloudStackVolume; + } catch (FeignException e) { + logger.error("FeignException occurred while cloning file [{}], Status: {}, Exception: {}", + sourcePath, e.status(), e.getMessage()); + throw new CloudRuntimeException("Failed to clone file: " + e.getMessage()); + } catch (Exception e) { + logger.error("Exception occurred while cloning file [{}], Exception: {}", sourcePath, e.getMessage()); + throw new CloudRuntimeException("Failed to clone file: " + e.getMessage()); + } + } + + /** + * Grows the cloned qcow2 to the requested size via a host-side {@code qemu-img resize}. + */ + @Override + public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes) { + if (cloudstackVolume == null || cloudstackVolume.getVolumeInfo() == null) { + logger.error("resizeCloudStackVolume: Resize failed. Invalid request: {}", cloudstackVolume); + throw new CloudRuntimeException("Failed to resize file, invalid request"); + } + if (sizeInBytes <= 0) { + throw new CloudRuntimeException("Failed to resize file, invalid size " + sizeInBytes); + } + + DataObject volumeInfo = cloudstackVolume.getVolumeInfo(); + Answer answer = resizeVolumeOnKVMHost(volumeInfo, sizeInBytes); + if (answer == null || !answer.getResult()) { + String errMsg = answer != null ? answer.getDetails() : "Failed to resize qcow2 on KVM host"; + logger.error("resizeCloudStackVolume: " + errMsg); + throw new CloudRuntimeException(errMsg); + } + if (cloudstackVolume.getFile() != null && cloudstackVolume.getFile().getQosPolicy() != null) { + updateCloudStackVolume(cloudstackVolume); + } + logger.info("resizeCloudStackVolume: Resized volume [{}] to {} bytes", volumeInfo.getUuid(), sizeInBytes); + } + + private Answer resizeVolumeOnKVMHost(DataObject volumeInfo, long sizeInBytes) { + VolumeObject volumeObject = (VolumeObject) volumeInfo; + VolumeVO volume = volumeDao.findById(volumeObject.getId()); + if (volume == null) { + throw new CloudRuntimeException("Volume not found with id: " + volumeObject.getId()); + } + + StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); + if (storagePool == null) { + throw new CloudRuntimeException("Storage Pool not found for id: " + volume.getPoolId()); + } + + // instanceName is set by VolumeApiServiceImpl.orchestrateResizeVolume() before calling the + // driver — it is the VM instance name when attached, or "none" when the volume is detached. + ResizeVolumePayload resizePayload = volumeObject.getpayload() instanceof ResizeVolumePayload + ? (ResizeVolumePayload) volumeObject.getpayload() + : null; + String instanceName = resizePayload != null ? resizePayload.instanceName : "none"; + ResizeVolumeCommand cmd = new ResizeVolumeCommand(volume.getPath(), new StorageFilerTO(storagePool), + volume.getSize(), sizeInBytes, false, instanceName); + + EndPoint ep = epSelector.select(volumeInfo); + if (ep == null) { + String errMsg = "No remote endpoint to send ResizeVolumeCommand, check if host is up"; + logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + logger.info("resizeVolumeOnKVMHost: Sending command to endpoint: {}", ep.getHostAddr()); + return ep.sendMessage(cmd); } @Override @@ -148,7 +318,7 @@ public AccessGroup createAccessGroup(AccessGroup accessGroup) { logger.info("createAccessGroup: ExportPolicy created: {}, now attaching this policy to storage pool volume", createdPolicy.getName()); // attach export policy to volume of storage pool assignExportPolicyToVolume(volumeUUID,createdPolicy.getName()); - // save the export policy details in storage pool details + // save the export policy poolDetails in storage pool poolDetails storagePoolDetailsDao.addDetail(accessGroup.getStoragePoolId(), OntapStorageConstants.EXPORT_POLICY_ID, String.valueOf(createdPolicy.getId()), true); storagePoolDetailsDao.addDetail(accessGroup.getStoragePoolId(), OntapStorageConstants.EXPORT_POLICY_NAME, createdPolicy.getName(), true); logger.info("Successfully assigned exportPolicy {} to volume {}", policyRequest.getName(), volumeName); @@ -176,12 +346,15 @@ public void deleteAccessGroup(AccessGroup accessGroup) { String exportPolicyId = details.get(OntapStorageConstants.EXPORT_POLICY_ID); try { - nasFeignClient.deleteExportPolicyById(authHeader,exportPolicyId); + nasFeignClient.deleteExportPolicyById(authHeader, exportPolicyId); logger.info("deleteAccessGroup: Successfully deleted export policy '{}'", exportPolicyName); - } catch (Exception e) { + } catch (FeignException e) { + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + logger.warn("deleteAccessGroup: Export policy '{}' not found in ONTAP, treating as no-op", exportPolicyName); + return; + } logger.error("deleteAccessGroup: Failed to delete export policy. Exception: {}", e.getMessage(), e); throw new CloudRuntimeException("Failed to delete export policy: " + e.getMessage(), e); - } } catch (Exception e) { logger.error("deleteAccessGroup: Failed to delete export policy. Exception: {}", e.getMessage(), e); @@ -191,7 +364,134 @@ public void deleteAccessGroup(AccessGroup accessGroup) { @Override public AccessGroup updateAccessGroup(AccessGroup accessGroup) { - return null; + if (accessGroup == null) { + throw new CloudRuntimeException("Invalid accessGroup object - accessGroup is null"); + } + // Check if an AccessGroup was constructed without associating it to a storage pool. + if (accessGroup.getStoragePoolId() == null) { + throw new CloudRuntimeException("Invalid accessGroup object - storagePoolId is null"); + } + // At least one host is required regardless of ADD or REMOVE action. + // An empty list means there is nothing to add to or remove from the export policy client list. + if (accessGroup.getHostsToConnect() == null || accessGroup.getHostsToConnect().isEmpty()) { + throw new CloudRuntimeException("Invalid accessGroup object - hostsToConnect is null or empty"); + } + + Map details = storagePoolDetailsDao.listDetailsKeyPairs(accessGroup.getStoragePoolId()); + if (details == null || details.isEmpty()) { + throw new CloudRuntimeException("No storage pool poolDetails found for storagePoolId: " + accessGroup.getStoragePoolId()); + } + String exportPolicyId = details.get(OntapStorageConstants.EXPORT_POLICY_ID); + if (exportPolicyId == null || exportPolicyId.isEmpty()) { + throw new CloudRuntimeException("No export policy found for storagePoolId: " + accessGroup.getStoragePoolId()); + } + + + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + ExportPolicy existingPolicy = nasFeignClient.getExportPolicyById(authHeader, exportPolicyId); + // Check if the export policy was deleted externally on ONTAP or the stored ID is stale. + if (existingPolicy == null) { + throw new CloudRuntimeException("Failed to fetch existing export policy with id: " + exportPolicyId); + } + + List rules = existingPolicy.getRules(); + if (rules == null || rules.isEmpty()) { + throw new CloudRuntimeException("Export policy " + existingPolicy.getName() + + " has no rules — unexpected state, the plugin always creates a rule at pool registration"); + } + + ExportRule targetRule = rules.get(0); + + Set hostMatches = new HashSet<>(); + for (HostVO host : accessGroup.getHostsToConnect()) { + String hostStorageIp = host.getStorageIpAddress() != null ? host.getStorageIpAddress().trim() : null; + String ip = (hostStorageIp != null && !hostStorageIp.isEmpty()) ? hostStorageIp + : (host.getPrivateIpAddress() != null ? host.getPrivateIpAddress().trim() : null); + // Occurs when a CloudStack host has neither a storage IP nor a private IP configured + // (misconfigured or partially registered host). Skip it to avoid inserting a broken + // or empty match entry into the ONTAP export rule. + if (ip == null || ip.isEmpty()) { + logger.warn("updateAccessGroup: Host {} has no storage/private IP, skipping export rule update", host.getName()); + continue; + } + hostMatches.add(ip + "/32"); + } + + // Occurs when every host in hostsToConnect had no valid IP (all were skipped above). + // There is nothing to add or remove, so skip the ONTAP API call and return early. + if (hostMatches.isEmpty()) { + accessGroup.setPolicy(existingPolicy); + return accessGroup; + } + + boolean updated = false; + // Differentiates between removing hosts (e.g., host decommissioned or removed from the cluster) + // and the default ADD path (e.g., new host being connected to the storage pool). + List exportClients = targetRule.getClients(); + // Existing rules can legitimately have no clients yet; treat that as an empty list. + if (exportClients == null) { + exportClients = new ArrayList<>(); + targetRule.setClients(exportClients); + } + + if (AccessGroup.HostRuleAction.REMOVE.equals(accessGroup.getHostRuleAction())) { + updated = exportClients.removeIf(c -> c != null && c.getMatch() != null && hostMatches.contains(c.getMatch())); + // None of the requested host IPs were present in the policy — log for diagnostics + // so operators can investigate whether the policy state is already correct or stale. + if (!updated) { + logger.info("updateAccessGroup: No matching host IPs found in export policy {} for removal", existingPolicy.getName()); + } + } else { + Set existingMatches = new HashSet<>(); + for (ExportRule.ExportClient exportClient : exportClients) { + // Skips null client entries or entries with a null match field that may have been + // inserted externally on ONTAP. Avoids polluting the dedup set with null values + // which would cause subsequent hosts to be incorrectly treated as duplicates. + if (exportClient != null && exportClient.getMatch() != null) { + existingMatches.add(exportClient.getMatch()); + } + } + + for (String match : hostMatches) { + // Set.add() returns false when the element was already present, acting as a dedup check. + // Prevents inserting a duplicate client match entry for a host that is already allowed + // in the export policy — ONTAP may reject or behave unpredictably with duplicate matches. + if (existingMatches.add(match)) { + ExportRule.ExportClient exportClient = new ExportRule.ExportClient(); + exportClient.setMatch(match); + exportClients.add(exportClient); + updated = true; + } + } + } + + // Occurs when the export policy is already in the desired state: + // ADD path — all provided host IPs were already present (all were duplicates). + // REMOVE path — none of the provided host IPs matched any existing entry. + // In both cases, skip the ONTAP PATCH call to avoid an unnecessary round-trip. + if (!updated) { + // Only log the "nothing to add" message for the ADD path; the REMOVE no-op + // is already logged above in its own branch to avoid double-logging. + if (!AccessGroup.HostRuleAction.REMOVE.equals(accessGroup.getHostRuleAction())) { + logger.info("updateAccessGroup: No new host IPs to add to export policy {}", existingPolicy.getName()); + } + accessGroup.setPolicy(existingPolicy); + return accessGroup; + } + + ExportPolicy updateRequest = new ExportPolicy(); + updateRequest.setRules(rules); + nasFeignClient.updateExportPolicy(authHeader, exportPolicyId, updateRequest); + + existingPolicy.setRules(rules); + accessGroup.setPolicy(existingPolicy); + logger.info("updateAccessGroup: Successfully updated export policy {} with new host client rules", existingPolicy.getName()); + return accessGroup; + } catch (Exception e) { + logger.error("updateAccessGroup: Failed to update export policy for pool {}", accessGroup.getStoragePoolId(), e); + throw new CloudRuntimeException("Failed to update export policy for NFS host connection: " + e.getMessage(), e); + } } @Override @@ -256,7 +556,7 @@ private void assignExportPolicyToVolume(String volumeUuid, String policyName) { volumeUpdate.setNas(nas); try { - JobResponse jobResponse = volumeFeignClient.updateVolumeRebalancing(authHeader, volumeUuid, volumeUpdate); + JobResponse jobResponse = volumeFeignClient.updateVolume(authHeader, volumeUuid, volumeUpdate); if (jobResponse == null || jobResponse.getJob() == null) { throw new CloudRuntimeException("Failed to attach policy " + policyName + "to volume " + volumeUuid); } @@ -307,10 +607,10 @@ private ExportPolicy createExportPolicyRequest(AccessGroup accessGroup,String sv List exportClients = new ArrayList<>(); List hosts = accessGroup.getHostsToConnect(); for (HostVO host : hosts) { - String hostStorageIp = host.getStorageIpAddress(); + String hostStorageIp = host.getStorageIpAddress() != null ? host.getStorageIpAddress().trim() : null; String ip = (hostStorageIp != null && !hostStorageIp.isEmpty()) ? hostStorageIp - : host.getPrivateIpAddress(); + : (host.getPrivateIpAddress() != null ? host.getPrivateIpAddress().trim() : null); String ipToUse = ip + "/32"; ExportRule.ExportClient exportClient = new ExportRule.ExportClient(); exportClient.setMatch(ipToUse); @@ -409,6 +709,28 @@ private Answer deleteVolumeOnKVMHost(DataObject volumeInfo) { } } + /** + * Deletes a file from a FlexVolume, treating an already-absent file as success. + */ + public void deleteFileByPath(String flexVolUuid, String filePath) { + logger.info("deleteFileByPath: Deleting file [{}] from FlexVol [{}]", filePath, flexVolUuid); + try { + nasFeignClient.deleteFile(getAuthHeader(), flexVolUuid, filePath); + logger.debug("deleteFileByPath: Deleted file [{}]", filePath); + } catch (FeignException e) { + if (e.status() == 404) { + logger.warn("deleteFileByPath: File [{}] does not exist (status 404), skipping deletion", filePath); + return; + } + logger.error("FeignException occurred while deleting file [{}], Status: {}, Exception: {}", + filePath, e.status(), e.getMessage()); + throw new CloudRuntimeException("Failed to delete file: " + e.getMessage()); + } catch (Exception e) { + logger.error("Exception occurred while deleting file [{}], Exception: {}", filePath, e.getMessage()); + throw new CloudRuntimeException("Failed to delete file: " + e.getMessage()); + } + } + private FileInfo getFile(String volumeUuid, String filePath) { logger.info("Get File: {} for volume: {}", filePath, volumeUuid); diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java index 2b0e65f9f7ea..c6310352b468 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java @@ -19,17 +19,24 @@ package org.apache.cloudstack.storage.service; -import com.cloud.host.HostVO; -import com.cloud.utils.exception.CloudRuntimeException; -import feign.FeignException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.CliSnapshotRestoreRequest; import org.apache.cloudstack.storage.feign.model.Igroup; import org.apache.cloudstack.storage.feign.model.Initiator; -import org.apache.cloudstack.storage.feign.model.Svm; -import org.apache.cloudstack.storage.feign.model.OntapStorage; import org.apache.cloudstack.storage.feign.model.Lun; import org.apache.cloudstack.storage.feign.model.LunMap; -import org.apache.cloudstack.storage.feign.model.CliSnapshotRestoreRequest; +import org.apache.cloudstack.storage.feign.model.LunSpace; +import org.apache.cloudstack.storage.feign.model.OntapStorage; +import org.apache.cloudstack.storage.feign.model.Svm; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import org.apache.cloudstack.storage.service.model.AccessGroup; @@ -38,18 +45,24 @@ import org.apache.cloudstack.storage.utils.OntapStorageConstants; import org.apache.cloudstack.storage.utils.OntapStorageUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; + +import com.cloud.host.HostVO; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.exception.CloudRuntimeException; + +import feign.FeignException; public class UnifiedSANStrategy extends SANStrategy { private static final Logger logger = LogManager.getLogger(UnifiedSANStrategy.class); @Inject private StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + private VolumeDetailsDao volumeDetailsDao; public UnifiedSANStrategy(OntapStorage ontapStorage) { super(ontapStorage); @@ -78,6 +91,7 @@ public CloudStackVolume createCloudStackVolume(CloudStackVolume cloudstackVolume throw new CloudRuntimeException("Failed to create Lun: " + cloudstackVolume.getLun().getName()); } Lun lun = createdLun.getRecords().get(0); + validateCreatedLun(lun, cloudstackVolume.getLun().getName(), "createCloudStackVolume"); logger.debug("createCloudStackVolume: LUN created successfully. Lun: {}", lun); CloudStackVolume createdCloudStackVolume = new CloudStackVolume(); @@ -86,16 +100,124 @@ public CloudStackVolume createCloudStackVolume(CloudStackVolume cloudstackVolume } catch (FeignException e) { logger.error("FeignException occurred while creating LUN: {}, Status: {}, Exception: {}", cloudstackVolume.getLun().getName(), e.status(), e.getMessage()); - throw new CloudRuntimeException("Failed to create Lun: " + e.getMessage()); + throw new CloudRuntimeException("Failed to create Lun: " + e.getMessage(), e); + } catch (CloudRuntimeException e) { + throw e; } catch (Exception e) { logger.error("Exception occurred while creating LUN: {}, Exception: {}", cloudstackVolume.getLun().getName(), e.getMessage()); - throw new CloudRuntimeException("Failed to create Lun: " + e.getMessage()); + throw new CloudRuntimeException("Failed to create Lun: " + e.getMessage(), e); } } + /** + * Creates an empty LUN that caches a template on this pool's FlexVolume. + * + *

Sized to the template virtual disk size (what KVM writes after {@code qemu-img convert} + * to RAW). On create failure, best-effort deletes any leftover LUN before rethrowing.

+ */ @Override - CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume) { - return null; + public CloudStackVolume createTemplateCache(StoragePoolVO storagePool, TemplateInfo templateInfo, + Map details, long sizeInBytes) { + if (sizeInBytes <= 0) { + throw new CloudRuntimeException("Unknown virtual size for template [" + templateInfo.getId() + + "]; cannot size the template LUN on pool [" + storagePool.getId() + "]. The template size in vm_template" + + " is unset; verify the template was registered/seeded with its virtual size (virtualsize in" + + " template.properties on secondary storage)."); + } + + CloudStackVolume request = buildTemplateLunRequest(storagePool, details, templateInfo.getId(), sizeInBytes); + String lunName = request.getLun().getName(); + CloudStackVolume created = null; + try { + created = createCloudStackVolume(request); + Lun lun = created.getLun(); + logger.info("createTemplateCache: Created template cache LUN [{}] (uuid [{}], {} bytes) on pool [{}] for template [{}]", + lun.getName(), lun.getUuid(), sizeInBytes, storagePool.getId(), templateInfo.getId()); + return created; + } catch (Exception e) { + bestEffortDeleteTemplateCacheLun(details.get(OntapStorageConstants.SVM_NAME), lunName, + created != null && created.getLun() != null ? created.getLun().getUuid() : null); + if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException) e; + } + throw new CloudRuntimeException("Failed to create template cache LUN for template [" + templateInfo.getId() + + "]: " + e.getMessage(), e); + } + } + + private CloudStackVolume buildTemplateLunRequest(StoragePoolVO storagePool, Map details, + long templateId, long sizeInBytes) { + Svm svm = new Svm(); + svm.setName(details.get(OntapStorageConstants.SVM_NAME)); + + Lun lunRequest = new Lun(); + lunRequest.setSvm(svm); + lunRequest.setName(OntapStorageUtils.getLunName(storagePool.getName(), + OntapStorageConstants.TEMPLATE_LUN_PREFIX + templateId)); + lunRequest.setOsType(Lun.OsTypeEnum.valueOf( + OntapStorageUtils.getOSTypeFromHypervisor(storagePool.getHypervisor().name()))); + LunSpace lunSpace = new LunSpace(); + lunSpace.setSize(sizeInBytes); + lunRequest.setSpace(lunSpace); + + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lunRequest); + return request; + } + + private void bestEffortDeleteTemplateCacheLun(String svmName, String lunName, String lunUuid) { + try { + String uuid = lunUuid; + if (uuid == null || uuid.isEmpty()) { + Map lookup = Map.of( + OntapStorageConstants.NAME, lunName, + OntapStorageConstants.SVM_DOT_NAME, svmName); + CloudStackVolume existing = getCloudStackVolume(lookup); + if (existing == null || existing.getLun() == null || existing.getLun().getUuid() == null) { + logger.warn("bestEffortDeleteTemplateCacheLun: LUN [{}] not found on SVM [{}]; nothing to delete", + lunName, svmName); + return; + } + uuid = existing.getLun().getUuid(); + } + Lun lun = new Lun(); + lun.setUuid(uuid); + lun.setName(lunName); + CloudStackVolume deleteRequest = new CloudStackVolume(); + deleteRequest.setLun(lun); + deleteCloudStackVolume(deleteRequest); + logger.info("bestEffortDeleteTemplateCacheLun: Removed leftover template cache LUN [{}] on SVM [{}]", + lunName, svmName); + } catch (Exception cleanupEx) { + logger.warn("bestEffortDeleteTemplateCacheLun: Failed to remove leftover template cache LUN [{}] on SVM [{}]: {}", + lunName, svmName, cleanupEx.getMessage()); + } + } + + @Override + public CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume) { + if (cloudstackVolume == null || cloudstackVolume.getLun() == null + || cloudstackVolume.getLun().getUuid() == null) { + throw new CloudRuntimeException("Invalid iSCSI volume request"); + } + VolumeQosPolicy qosPolicy = cloudstackVolume.getLun().getQosPolicy(); + if (qosPolicy == null) { + return cloudstackVolume; + } + Lun lunUpdate = new Lun(); + lunUpdate.setQosPolicy(qosPolicy); + try { + JobResponse response = sanFeignClient.updateLun( + getAuthHeader(), cloudstackVolume.getLun().getUuid(), lunUpdate); + pollJobIfPresent(response, "update LUN [" + cloudstackVolume.getLun().getUuid() + "]"); + } catch (FeignException e) { + throw new CloudRuntimeException("Failed to update LUN: " + e.getMessage(), e); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to update LUN: " + e.getMessage(), e); + } + logger.info("Updated LUN [{}] with QoS policy [{}]", + cloudstackVolume.getLun().getUuid(), qosPolicy.getName()); + return cloudstackVolume; } @Override @@ -109,7 +231,8 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) { String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); Map queryParams = Map.of("allow_delete_while_mapped", "true"); try { - sanFeignClient.deleteLun(authHeader, cloudstackVolume.getLun().getUuid(), queryParams); + JobResponse response = sanFeignClient.deleteLun(authHeader, cloudstackVolume.getLun().getUuid(), queryParams); + pollJobIfPresent(response, "delete Lun [" + cloudstackVolume.getLun().getName() + "]"); } catch (FeignException feignEx) { if (feignEx.status() == 404) { logger.warn("deleteCloudStackVolume: Lun {} does not exist (status 404), skipping deletion", cloudstackVolume.getLun().getName()); @@ -124,8 +247,128 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) { } } + /** + * Clones a LUN inside the FlexVolume using ONTAP's {@code clone.source} form of LUN create. + * + *

The resulting LUN shares blocks with its source and is created in constant time. + * It is a sis-clone, so it stays readable after the source LUN is deleted.

+ */ @Override - public void copyCloudStackVolume(CloudStackVolume cloudstackVolume) {} + public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume) { + if (cloudstackVolume == null || cloudstackVolume.getLun() == null) { + logger.error("cloneCloudStackVolume: LUN clone failed. Invalid request: {}", cloudstackVolume); + throw new CloudRuntimeException("Failed to clone Lun, invalid request"); + } + Lun lunRequest = cloudstackVolume.getLun(); + if (lunRequest.getClone() == null || lunRequest.getClone().getSource() == null) { + logger.error("cloneCloudStackVolume: LUN clone failed. No clone source in request for Lun {}", lunRequest.getName()); + throw new CloudRuntimeException("Failed to clone Lun, no clone source provided"); + } + logger.trace("cloneCloudStackVolume: Cloning Lun {} from source name={} uuid={}", + lunRequest.getName(), + lunRequest.getClone().getSource().getName(), + lunRequest.getClone().getSource().getUuid()); + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + OntapResponse clonedLun = sanFeignClient.createLun(authHeader, true, lunRequest); + if (clonedLun == null || CollectionUtils.isEmpty(clonedLun.getRecords())) { + logger.error("cloneCloudStackVolume: LUN clone returned no records for Lun {}", lunRequest.getName()); + throw new CloudRuntimeException("Failed to clone Lun: " + lunRequest.getName()); + } + Lun lun = clonedLun.getRecords().get(0); + validateCreatedLun(lun, lunRequest.getName(), "cloneCloudStackVolume"); + logger.debug("cloneCloudStackVolume: LUN cloned successfully. Lun: {}", lun); + + CloudStackVolume clonedCloudStackVolume = new CloudStackVolume(); + clonedCloudStackVolume.setLun(lun); + clonedCloudStackVolume.setVolumeInfo(cloudstackVolume.getVolumeInfo()); + return clonedCloudStackVolume; + } catch (FeignException e) { + logger.error("FeignException occurred while cloning LUN: {}, Status: {}, Exception: {}", + lunRequest.getName(), e.status(), e.getMessage()); + throw new CloudRuntimeException("Failed to clone Lun: " + e.getMessage()); + } catch (CloudRuntimeException e) { + throw e; + } catch (Exception e) { + logger.error("Exception occurred while cloning LUN: {}, Exception: {}", lunRequest.getName(), e.getMessage()); + throw new CloudRuntimeException("Failed to clone Lun: " + e.getMessage()); + } + } + + /** + * Ensures ONTAP returned a usable LUN identity from create/clone. Callers in the datastore + * driver rely on non-null name and uuid, so reject incomplete records at the Feign boundary. + */ + private void validateCreatedLun(Lun lun, String requestName, String operation) { + if (lun == null || lun.getName() == null || lun.getUuid() == null) { + logger.error("{}: ONTAP returned incomplete LUN for {}", operation, requestName); + throw new CloudRuntimeException("ONTAP returned incomplete LUN for: " + requestName); + } + } + + /** + * Grows an existing LUN to {@code sizeInBytes}. + * + *

Needed after cloning a cached template, because a clone inherits the size of its source + * while the service offering may ask for a larger disk.

+ */ + @Override + public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes) { + if (cloudstackVolume == null || cloudstackVolume.getVolumeInfo() == null) { + logger.error("resizeCloudStackVolume: Lun resize failed. Invalid request: {}", cloudstackVolume); + throw new CloudRuntimeException("Failed to resize Lun, invalid request"); + } + if (sizeInBytes <= 0) { + throw new CloudRuntimeException("Failed to resize Lun, invalid size " + sizeInBytes); + } + + // Resolve LUN UUID from volume details when not pre-populated on the cloudstackVolume + // Keep a QoS policy already set on the request so size and QoS go out in one LUN update. + VolumeQosPolicy qosPolicy = cloudstackVolume.getLun() != null ? cloudstackVolume.getLun().getQosPolicy() : null; + if (cloudstackVolume.getLun() == null || cloudstackVolume.getLun().getUuid() == null) { + logger.debug("LUN details not present on cloudstackVolume, resolving UUID from volume details"); + long volumeId = cloudstackVolume.getVolumeInfo().getId(); + VolumeDetailVO lunUuidDetail = volumeDetailsDao.findDetail(volumeId, OntapStorageConstants.LUN_DOT_UUID); + if (lunUuidDetail == null || StringUtils.isBlank(lunUuidDetail.getValue())) { + throw new CloudRuntimeException("LUN UUID not found in volume details for volume " + volumeId); + } + Lun resolvedLun = new Lun(); + resolvedLun.setUuid(lunUuidDetail.getValue()); + cloudstackVolume.setLun(resolvedLun); + } else { + logger.debug("resizeCloudStackVolume: LUN UUID [{}] already present on cloudstackVolume, skipping volume details lookup", cloudstackVolume.getLun().getUuid()); + } + + String lunUuid = cloudstackVolume.getLun().getUuid(); + logger.trace("resizeCloudStackVolume: Resizing Lun {} to {} bytes", lunUuid, sizeInBytes); + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + LunSpace lunSpace = new LunSpace(); + lunSpace.setSize(sizeInBytes); + Lun lunPatch = new Lun(); + lunPatch.setSpace(lunSpace); + if (qosPolicy != null) { + lunPatch.setQosPolicy(qosPolicy); + } + JobResponse response = sanFeignClient.updateLun(authHeader, lunUuid, lunPatch); + pollJobIfPresent(response, "resize Lun [" + lunUuid + "]"); + logger.debug("resizeCloudStackVolume: Lun {} resized to {} bytes", lunUuid, sizeInBytes); + } catch (FeignException e) { + logger.error("FeignException occurred while resizing LUN [{}], Status: {}, Exception: {}", + lunUuid, e.status(), e.getMessage()); + if (OntapStorageUtils.isOntapObjectNotFoundError(e)) { + throw new CloudRuntimeException(String.format( + "LUN [%s] no longer exists on ONTAP; it may have been deleted externally. " + + "Verify the LUN is present before retrying the resize.", lunUuid)); + } + throw new CloudRuntimeException(String.format( + "Failed to resize LUN [%s]: %s",lunUuid, e.getMessage())); + } catch (Exception e) { + logger.error("Exception occurred while resizing LUN [{}]: {}", lunUuid, e.getMessage()); + throw new CloudRuntimeException(String.format( + "Unexpected error while resizing LUN [%s]: %s", lunUuid, e.getMessage())); + } + } @Override public CloudStackVolume getCloudStackVolume(Map values) { @@ -160,10 +403,41 @@ public CloudStackVolume getCloudStackVolume(Map values) { return null; } logger.error("FeignException occurred while fetching Lun, Status: {}, Exception: {}", e.status(), e.getMessage()); - throw new CloudRuntimeException("Failed to fetch Lun details: " + e.getMessage()); + throw new CloudRuntimeException("Failed to fetch Lun poolDetails: " + e.getMessage()); } catch (Exception e) { logger.error("Exception occurred while fetching Lun, Exception: {}", e.getMessage()); - throw new CloudRuntimeException("Failed to fetch Lun details: " + e.getMessage()); + throw new CloudRuntimeException("Failed to fetch Lun poolDetails: " + e.getMessage()); + } + } + + public String getLunSerialNumber(String svmName, String lunName) { + logger.trace("getLunSerialNumber: fetching serial number of LUN [{}] on SVM [{}]", lunName, svmName); + if (svmName == null || svmName.isEmpty() || lunName == null || lunName.isEmpty()) { + throw new CloudRuntimeException("Failed to get LUN serial number, invalid svm or LUN name"); + } + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + Map queryParams = Map.of( + OntapStorageConstants.SVM_DOT_NAME, svmName, + OntapStorageConstants.NAME, lunName, + OntapStorageConstants.FIELDS, OntapStorageConstants.SERIAL_NUMBER); + OntapResponse lunResponse = sanFeignClient.getLunResponse(authHeader, queryParams); + if (lunResponse == null || lunResponse.getRecords() == null || lunResponse.getRecords().isEmpty()) { + logger.warn("getLunSerialNumber: LUN [{}] on SVM [{}] not found", lunName, svmName); + return null; + } + return lunResponse.getRecords().get(0).getSerialNumber(); + } catch (FeignException e) { + if (e.status() == 404) { + logger.warn("getLunSerialNumber: LUN [{}] on SVM [{}] not found (status 404)", lunName, svmName); + return null; + } + logger.error("FeignException while fetching serial number of LUN [{}], Status: {}, Exception: {}", + lunName, e.status(), e.getMessage()); + throw new CloudRuntimeException("Failed to fetch LUN serial number: " + e.getMessage()); + } catch (Exception e) { + logger.error("Exception while fetching serial number of LUN [{}]: {}", lunName, e.getMessage()); + throw new CloudRuntimeException("Failed to fetch LUN serial number: " + e.getMessage()); } } @@ -174,9 +448,9 @@ public AccessGroup createAccessGroup(AccessGroup accessGroup) { logger.error("createAccessGroup: Igroup creation failed. Invalid request: {}", accessGroup); throw new CloudRuntimeException("Failed to create Igroup, invalid request"); } - // Get StoragePool details + // Get StoragePool poolDetails if (accessGroup.getStoragePoolId() == null) { - throw new CloudRuntimeException("Failed to create Igroup, invalid datastore details in the request"); + throw new CloudRuntimeException("Failed to create Igroup, invalid datastore poolDetails in the request"); } if (accessGroup.getHostsToConnect() == null || accessGroup.getHostsToConnect().isEmpty()) { throw new CloudRuntimeException("Failed to create Igroup, no hosts to connect provided in the request"); @@ -185,7 +459,7 @@ public AccessGroup createAccessGroup(AccessGroup accessGroup) { String igroupName = null; try { Map dataStoreDetails = storagePoolDetailsDao.listDetailsKeyPairs(accessGroup.getStoragePoolId()); - logger.trace("createAccessGroup: Successfully fetched datastore details."); + logger.trace("createAccessGroup: Successfully fetched datastore poolDetails."); // Generate Igroup request Igroup igroupRequest = new Igroup(); @@ -261,9 +535,9 @@ public void deleteAccessGroup(AccessGroup accessGroup) { logger.error("deleteAccessGroup: Igroup deletion failed. Invalid request: {}", accessGroup); throw new CloudRuntimeException("Failed to delete Igroup, invalid request"); } - // Get StoragePool details + // Get StoragePool poolDetails if (accessGroup.getStoragePoolId() == null) { - throw new CloudRuntimeException("Failed to delete Igroup, invalid datastore details in the request"); + throw new CloudRuntimeException("Failed to delete Igroup, invalid datastore poolDetails in the request"); } try { String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); @@ -370,10 +644,10 @@ public AccessGroup getAccessGroup(Map values) { return null; } logger.error("FeignException occurred while fetching Igroup, Status: {}, Exception: {}", e.status(), e.getMessage()); - throw new CloudRuntimeException("Failed to fetch Igroup details: " + e.getMessage()); + throw new CloudRuntimeException("Failed to fetch Igroup poolDetails: " + e.getMessage()); } catch (Exception e) { logger.error("Exception occurred while fetching Igroup, Exception: {}", e.getMessage()); - throw new CloudRuntimeException("Failed to fetch Igroup details: " + e.getMessage()); + throw new CloudRuntimeException("Failed to fetch Igroup poolDetails: " + e.getMessage()); } } @@ -419,7 +693,7 @@ public String enableLogicalAccess(Map values) { throw feignEx; } } - // Get the LunMap details + // Get the LunMap poolDetails OntapResponse lunMapResponse = null; try { lunMapResponse = sanFeignClient.getLunMapResponse(authHeader, @@ -434,12 +708,12 @@ public String enableLogicalAccess(Map values) { lunNumber = lunMapResponse.getRecords().get(0).getLogicalUnitNumber().toString(); } else { - logger.error("enableLogicalAccess: Failed to fetch LunMap details for Lun: {} and igroup: {}. LunMap response is null or empty.", lunName, igroupName); - throw new CloudRuntimeException("Failed to fetch LunMap details for Lun: " + lunName + " and igroup: " + igroupName); + logger.error("enableLogicalAccess: Failed to fetch LunMap poolDetails for Lun: {} and igroup: {}. LunMap response is null or empty.", lunName, igroupName); + throw new CloudRuntimeException("Failed to fetch LunMap poolDetails for Lun: " + lunName + " and igroup: " + igroupName); } } catch (Exception e) { - logger.error("enableLogicalAccess: Failed to fetch LunMap details for Lun: {} and igroup: {}, Exception: {}", lunName, igroupName, e); - throw new CloudRuntimeException("Failed to fetch LunMap details for Lun: " + lunName + " and igroup: " + igroupName); + logger.error("enableLogicalAccess: Failed to fetch LunMap poolDetails for Lun: {} and igroup: {}, Exception: {}", lunName, igroupName, e); + throw new CloudRuntimeException("Failed to fetch LunMap poolDetails for Lun: " + lunName + " and igroup: " + igroupName); } logger.trace("enableLogicalAccess: LunMap created successfully, LunMap: {}", lunMapResponse.getRecords().get(0)); } catch (Exception e) { diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/model/AccessGroup.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/model/AccessGroup.java index 9815724fc1aa..8b1aa24fe5d9 100755 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/model/AccessGroup.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/model/AccessGroup.java @@ -19,21 +19,28 @@ package org.apache.cloudstack.storage.service.model; -import com.cloud.host.HostVO; +import java.util.List; + import org.apache.cloudstack.engine.subsystem.api.storage.Scope; import org.apache.cloudstack.storage.feign.model.ExportPolicy; import org.apache.cloudstack.storage.feign.model.Igroup; -import java.util.List; +import com.cloud.host.HostVO; public class AccessGroup { + public enum HostRuleAction { + ADD, + REMOVE + } + private Igroup igroup; private ExportPolicy exportPolicy; private List hostsToConnect; private Long storagePoolId; private Scope scope; + private HostRuleAction hostRuleAction = HostRuleAction.ADD; public Igroup getIgroup() { @@ -74,4 +81,12 @@ public Scope getScope() { public void setScope(Scope scope) { this.scope = scope; } + + public HostRuleAction getHostRuleAction() { + return hostRuleAction; + } + + public void setHostRuleAction(HostRuleAction hostRuleAction) { + this.hostRuleAction = hostRuleAction; + } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapConfigurationManager.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapConfigurationManager.java new file mode 100644 index 000000000000..e93a1757de2f --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapConfigurationManager.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.utils; + +import com.cloud.exception.InvalidParameterValueException; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.ValidatedConfigKey; +import org.apache.commons.lang3.StringUtils; + +import java.util.function.Consumer; + +/** + * Single registration point for ONTAP plugin {@link ConfigKey}s (same pattern as + * Linstor/StorPool). Callers read live values via {@code Key.value()}; they must not + * construct new {@code ConfigKey} instances. + */ +public class OntapConfigurationManager implements Configurable { + public static final ValidatedConfigKey AsupIntervalHours = new ValidatedConfigKey<>( + OntapStorageConstants.ADVANCED_CONFIG_KEY_CATEGORY, Integer.class, + OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY, + String.valueOf(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS), + OntapStorageConstants.ASUP_INTERVAL_DESCRIPTION, + true, ConfigKey.Scope.Global, null, asupIntervalValidator()); + + public static final ConfigKey[] CONFIG_KEYS = new ConfigKey[] { + AsupIntervalHours + }; + + /** + * {@link ValidatedConfigKey#validateValue(String)} always passes the raw config string, + * even when the key type is {@link Integer}. Adapt that to {@code Consumer}. + */ + @SuppressWarnings("unchecked") + private static Consumer asupIntervalValidator() { + Consumer validator = OntapConfigurationManager::validateAsupInterval; + return (Consumer) (Consumer) validator; + } + + /** + * Rejects {@code ontap.autosupport.interval} values that are not {@link + * OntapStorageConstants#ASUP_DISABLED_INTERVAL_HOURS} (disabled) or integers in + * [{@link OntapStorageConstants#ASUP_MIN_INTERVAL_HOURS}, + * {@link OntapStorageConstants#ASUP_MAX_INTERVAL_HOURS}] hours. + * Invoked by {@link ValidatedConfigKey} when the setting is saved in Global Settings. + * {@code raw} is the saved string (or null). + */ + private static void validateAsupInterval(Object raw) { + String value = raw == null ? null : String.valueOf(raw).trim(); + if (StringUtils.isBlank(value)) { + throw new InvalidParameterValueException(asupIntervalRangeMessage()); + } + final int parsed; + try { + parsed = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new InvalidParameterValueException( + OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY + " must be an integer. " + + asupIntervalRangeMessage()); + } + if (parsed == OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS) { + return; + } + if (parsed < OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS + || parsed > OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS) { + throw new InvalidParameterValueException(asupIntervalRangeMessage()); + } + } + + private static String asupIntervalRangeMessage() { + return String.format( + "%s must be %d to disable, or between %d hours and %d hours (1 week).", + OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY, + OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS, + OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS, + OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS); + } + + @Override + public String getConfigComponentName() { + return OntapConfigurationManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return CONFIG_KEYS; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java index e5224237e526..c12fe7325761 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java @@ -36,6 +36,7 @@ public class OntapStorageConstants { public static final String SIZE = "size"; public static final String PROTOCOL = "protocol"; public static final String SVM_NAME = "svmName"; + public static final String SVM_UUID = "svmUUID"; public static final String USERNAME = "username"; public static final String PASSWORD = "password"; public static final String DATA_LIF = "dataLIF"; @@ -58,19 +59,46 @@ public class OntapStorageConstants { public static final String JOB_SUCCESS = "success"; public static final String TRUE = "true"; - public static final String FALSE = "false"; // Query params public static final String NAME = "name"; public static final String FIELDS = "fields"; + public static final String SERIAL_NUMBER = "serial_number"; + + /** + * NAA format designator. + * {@code 6} = "IEEE Registered Extended" (128-bit WWID). + */ + public static final String NAA_IEEE_REG_EXT = "6"; + /** + * NetApp's IEEE-registered OUI. Lookup: https://standards-oui.ieee.org/ → "NetApp, Inc." + */ + public static final String NETAPP_OUI = "00a098"; + /** + * the 4-bit pad nibble + *

The full WWID is {@code NETAPP_NAA_PREFIX} + {@code serial_number_hex} (12 ASCII + * bytes of the LUN serial, hex-encoded), yielding 32 hex chars / 128 bits.

+ */ + public static final String NETAPP_NAA_PREFIX = NAA_IEEE_REG_EXT + NETAPP_OUI + "0"; + public static final String SPACE_USED = "space.used"; + public static final String CLUSTER_NODE_ASUP_FIELDS = "model,is_all_flash_optimized,is_capacity_optimized,is_perf_optimized"; public static final String INITIATORS = "initiators"; public static final String AGGREGATES = "aggregates"; public static final String STATE = "state"; + public static final String AGGREGATE_NODE = "node"; + public static final String AGGREGATE_SPACE = "space"; public static final String DATA_NFS = "data_nfs"; public static final String DATA_ISCSI = "data_iscsi"; public static final String IP_ADDRESS = "ip.address"; + public static final String LIF_ENABLED = "enabled"; + public static final String LIF_STATE_UP = "up"; + public static final String LIF_LOCATION_HOME_NODE = "location.home_node.name"; + public static final String LIF_LOCATION_NODE = "location.node.name"; + public static final String LIF_WARNING = "ONTAP_LIF_WARNING"; public static final String SERVICES = "services"; public static final String RETURN_RECORDS = "return_records"; + public static final String SVM = "svm"; + public static final String VOLUMES = "volumes"; public static final int JOB_MAX_RETRIES = 100; public static final int CREATE_VOLUME_CHECK_SLEEP_TIME = 2000; @@ -80,9 +108,16 @@ public class OntapStorageConstants { public static final String SEMICOLON = ";"; public static final String COMMA = ","; public static final String HYPHEN = "-"; + public static final String DOT = "."; public static final String VOLUME_PATH_PREFIX = "/vol/"; + /** + * Path segment inserted after the FlexVol name when identifying a LUN inside a FlexVol snapshot + * for {@code POST /api/storage/luns} clone ({@code /vol/<fv>/.snapshot/<snap>/<lun>}). + */ + public static final String SNAPSHOT_PATH_SEGMENT = "/.snapshot/"; + public static final String ONTAP_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9_]*$"; public static final String KVM = "KVM"; @@ -91,6 +126,16 @@ public class OntapStorageConstants { public static final String LUN_DOT_NAME = "lun.name"; public static final String IQN = "iqn"; public static final String LUN_DOT_UUID = "lun.uuid"; + public static final String LUN_DOT_SERIAL_NUMBER = "lun.serial_number"; + public static final String QOS_POLICY_UUID = "qosPolicyUuid"; + public static final String IS_AFF = "isAFF"; + public static final String MIN_IOPS_NOT_SUPPORTED_ON_FAS = "Minimum IOPS must be 0 for FAS. Only a Maximum IOPS limit is supported"; + public static final String QOS_POLICY_NONE = "none"; + public static final String QOS_POLICY_NAME_PREFIX = "cs_"; + public static final String QOS_POLICY_NAME_TO = "to_"; + public static final String QOS_POLICY_NAME_IOPS = "iops_"; + public static final String UUID = "uuid"; + public static final String QOS_POLICY_OBJECT_COUNT_FIELDS = "uuid,name,object_count"; public static final String LOGICAL_UNIT_NUMBER = "logical_unit_number"; public static final String IGROUP_DOT_NAME = "igroup.name"; public static final String IGROUP_DOT_UUID = "igroup.uuid"; @@ -104,9 +149,114 @@ public class OntapStorageConstants { public static final String VOLUME_PATH = "volume_path"; public static final String PRIMARY_POOL_ID = "primary_pool_id"; public static final String ONTAP_SNAP_SIZE = "ontap_snap_size"; + /** + * Optional {@code snapshot_details} keys: min/max IOPS from the source volume at take-snapshot + * time. Persisted only when the volume has configured values; applied to volumes created from + * the snapshot in a later change (see TODO on create-from-snapshot). + */ + public static final String MIN_IOPS = "min_iops"; + public static final String MAX_IOPS = "max_iops"; public static final String FILE_PATH = "file_path"; public static final int MAX_SNAPSHOT_NAME_LENGTH = 255; + public static final String ONTAP_TEMP_CG_PREFIX = "cs-temp-cg-"; + /** ONTAP CG API: action required when referencing existing FlexVols in a consistency group. */ + public static final String CG_VOLUME_PROVISIONING_ACTION_ADD = "add"; + public static final int ONTAP_CG_JOB_MAX_RETRIES = 60; + public static final int ONTAP_CG_JOB_POLL_INTERVAL_MS = 2000; + public static final int ONTAP_CG_SNAPSHOT_RESOLVE_MAX_RETRIES = 30; + public static final int ONTAP_CG_SNAPSHOT_RESOLVE_POLL_INTERVAL_MS = 1000; + public static final int ONTAP_SFSR_JOB_MAX_RETRIES = 60; + public static final int ONTAP_SFSR_JOB_POLL_INTERVAL_MS = 2000; + public static final int ONTAP_SNAPSHOT_DELETE_JOB_MAX_RETRIES = 30; + public static final int ONTAP_SNAPSHOT_DELETE_JOB_POLL_INTERVAL_MS = 2000; + /** Retry settings for FlexVolume create/resize/delete job polling. */ + public static final int ONTAP_VOLUME_JOB_MAX_RETRIES = 10; + public static final int ONTAP_VOLUME_JOB_POLL_INTERVAL_MS = 1000; + public static final int ONTAP_FLEXVOL_JOB_POLL_INTERVAL_MS = 2000; + public static final int ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES = 30; /** vm_snapshot_details key for ONTAP FlexVolume-level VM snapshots. */ public static final String ONTAP_FLEXVOL_SNAPSHOT = "ontapFlexVolSnapshot"; + + /** Name prefix of the LUN that caches a template on the FlexVol, suffixed with the template id. */ + public static final String TEMPLATE_LUN_PREFIX = "cs_tmpl_"; + + /** + * Key of the {@code volume_details} row that {@code StorageSystemDataMotionStrategy} writes + * immediately before {@code createAsync} when a volume is to be cloned from a template already + * cached on this pool. The value is the CloudStack template id. The literal must stay in sync + * with the string used by the orchestrator. + */ + public static final String CLONE_OF_TEMPLATE = "cloneOfTemplate"; + + /** + * Key of the {@code volume_details} row that {@code StorageSystemDataMotionStrategy} writes + * immediately before {@code createAsync} when a volume is to be created from a CloudStack + * snapshot already present on this pool. The value is the CloudStack snapshot id. The literal + * must stay in sync with the string used by the orchestrator. + */ + public static final String CLONE_OF_SNAPSHOT = "cloneOfSnapshot"; + + /** + * Driver capability read by {@code StorageSystemDataMotionStrategy} when copying a snapshot to + * secondary storage. The literal must stay in sync with the string used by the orchestrator. + */ + public static final String CAN_DIRECT_ATTACH_SNAPSHOT = "CAN_DIRECT_ATTACH_SNAPSHOT"; + + /** + * Name prefix of the temporary volume a snapshot is cloned into for createTemplate(snapshotid), + * suffixed with the CloudStack snapshot id and volume id. + */ + public static final String TEMP_SNAPSHOT_COPY_NAME_PREFIX = "cs_tmp_snap_"; + + // ASUP (AutoSupport) / EMS telemetry + public static final String ADVANCED_CONFIG_KEY_CATEGORY = "Advanced"; + public static final String ASUP_CATEGORY = "provisioning"; + public static final String ASUP_SEVERITY = "notice"; + public static final String ASUP_EVENT_SOURCE = "CloudStack ONTAP plugin"; + public static final String ASUP_EVENT_ID_HEARTBEAT = "0"; + public static final String ASUP_EVENT_ID_STORAGE_POOL = "1"; + public static final String ASUP_UNKNOWN = "unknown"; + /** Event-id 0/1 JSON field: short human-readable description of the message. */ + public static final String ASUP_MESSAGE = "message"; + public static final String ASUP_HEARTBEAT_MESSAGE = "CloudStack connected to Unified ONTAP cluster"; + public static final String ASUP_POOL_MESSAGE = "CloudStack storage pool backed by Unified ONTAP volume"; + public static final String ASUP_POOL_NAME = "poolName"; + public static final String ASUP_POOL_STATUS = "poolStatus"; + public static final String ASUP_PROTOCOL = "protocol"; + public static final String ASUP_SVM = "svm"; + public static final String ASUP_ONTAP_VOLUME_UUID = "ontapVolumeUuid"; + public static final String ASUP_CLOUDSTACK_VERSION = "cloudstackVersion"; + public static final String ASUP_PLATFORM = "platform"; + public static final String ASUP_ONTAP_VERSION = "ontapVersion"; + public static final String ASUP_ONTAP_CLUSTER_MODEL = "ontapClusterModel"; + public static final String ASUP_ONTAP_PLATFORM_TYPE = "ontapPlatformType"; + public static final String ASUP_PLATFORM_TYPE_PERFORMANCE = "performance"; + public static final String ASUP_PLATFORM_TYPE_CAPACITY = "capacity"; + public static final String ASUP_PLATFORM_TYPE_FAS = "fas"; + public static final String ASUP_PLATFORM_TYPE_COMPOSITE = "composite"; + public static final String ASUP_CLUSTER_UUID = "clusterUuid"; + public static final String ASUP_MANAGEMENT_SERVER_COUNT = "managementServerCount"; + /** Event-id 0 field: VM snapshots spanning multiple ONTAP pools (consistency group). */ + public static final String ASUP_SNAPSHOT_ACROSS_POOL = "snapshot_across_pool"; + public static final String ASUP_MULTI_PRIMARY_STORAGE_POOL_VM = "multiPrimaryStoragePoolVm"; + public static final String ASUP_ROOT_DISK_COUNT = "rootDiskCount"; + public static final String ASUP_DATA_DISK_COUNT = "dataDiskCount"; + public static final String ASUP_TOTAL_LOGICAL_SIZE_BYTES = "totalLogicalSizeBytes"; + public static final String ASUP_VOLUME_SNAPSHOT_COUNT = "volumeSnapshotCount"; + public static final String ASUP_VM_SNAPSHOT_COUNT = "vmSnapshotCount"; + public static final String ASUP_GLOBAL_LOCK_NAME = "ontap.autosupport.push"; + public static final String ASUP_INTERVAL_CONFIG_KEY = "ontap.autosupport.interval"; + /** {@code ontap.autosupport.interval} value that disables telemetry reporting. */ + public static final int ASUP_DISABLED_INTERVAL_HOURS = 0; + public static final int ASUP_MIN_INTERVAL_HOURS = 4; + public static final int ASUP_MAX_INTERVAL_HOURS = 168; // once a week + public static final int ASUP_DEFAULT_INTERVAL_HOURS = 24; // once a day + + public static final String ASUP_INTERVAL_DESCRIPTION = + "Interval in hours between telemetry reports sent to the ONTAP cluster. " + + ASUP_DISABLED_INTERVAL_HOURS + " disables reporting. Allowed range is " + + ASUP_MIN_INTERVAL_HOURS + " hours to " + ASUP_MAX_INTERVAL_HOURS + + " hours (1 week). Default is " + ASUP_DEFAULT_INTERVAL_HOURS + + " hours (once a day). Changes take effect immediately and do not require a management server restart."; } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageUtils.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageUtils.java index 8a74e77b3371..5155f9f868f6 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageUtils.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageUtils.java @@ -19,25 +19,23 @@ package org.apache.cloudstack.storage.utils; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.utils.StringUtils; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import feign.FeignException; import org.apache.cloudstack.storage.feign.model.Lun; -import org.apache.cloudstack.storage.feign.model.LunSpace; import org.apache.cloudstack.storage.feign.model.OntapStorage; -import org.apache.cloudstack.storage.feign.model.Svm; import org.apache.cloudstack.storage.provider.StorageProviderFactory; import org.apache.cloudstack.storage.service.StorageStrategy; -import org.apache.cloudstack.storage.service.model.CloudStackVolume; import org.apache.cloudstack.storage.service.model.ProtocolType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.util.Base64Utils; -import java.nio.charset.StandardCharsets; -import java.util.Map; +import com.cloud.alert.AlertManager; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.StringUtils; +import com.cloud.utils.exception.CloudRuntimeException; public class OntapStorageUtils { @@ -57,48 +55,6 @@ public static String generateAuthHeader (String username, String password) { return BASIC + StringUtils.SPACE + new String(encodedBytes); } - public static CloudStackVolume createCloudStackVolumeRequestByProtocol(StoragePoolVO storagePool, Map details, DataObject volumeObject) { - CloudStackVolume cloudStackVolumeRequest = null; - - String protocol = details.get(OntapStorageConstants.PROTOCOL); - ProtocolType protocolType = ProtocolType.valueOf(protocol); - switch (protocolType) { - case NFS3: - cloudStackVolumeRequest = new CloudStackVolume(); - cloudStackVolumeRequest.setDatastoreId(String.valueOf(storagePool.getId())); - cloudStackVolumeRequest.setVolumeInfo(volumeObject); - break; - case ISCSI: - Svm svm = new Svm(); - svm.setName(details.get(OntapStorageConstants.SVM_NAME)); - cloudStackVolumeRequest = new CloudStackVolume(); - Lun lunRequest = new Lun(); - lunRequest.setSvm(svm); - - LunSpace lunSpace = new LunSpace(); - lunSpace.setSize(volumeObject.getSize()); - lunRequest.setSpace(lunSpace); - //Lun name is full path like in unified "/vol/VolumeName/LunName" - String lunName = volumeObject.getName().replace(OntapStorageConstants.HYPHEN, OntapStorageConstants.UNDERSCORE); - if(!isValidName(lunName)) { - String errMsg = "createAsync: Invalid dataObject name [" + lunName + "]. It must start with a letter and can only contain letters, digits, and underscores, and be up to 200 characters long."; - throw new InvalidParameterValueException(errMsg); - } - String lunFullName = getLunName(storagePool.getName(), lunName); - lunRequest.setName(lunFullName); - - String osType = getOSTypeFromHypervisor(storagePool.getHypervisor().name()); - lunRequest.setOsType(Lun.OsTypeEnum.valueOf(osType)); - - cloudStackVolumeRequest.setLun(lunRequest); - break; - default: - throw new CloudRuntimeException("Unsupported protocol " + protocol); - - } - return cloudStackVolumeRequest; - } - public static boolean isValidName(String name) { // Check for null and length constraint first if (name == null || name.length() > 200) { @@ -119,17 +75,40 @@ public static String getOSTypeFromHypervisor(String hypervisorType) { } } + public static void sendStorageAlert(AlertManager alertMgr, Long zoneId, Long podId, String subject, String body) { + if (alertMgr != null) { + alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_STORAGE_MISC, zoneId != null ? zoneId : 0L, podId, subject, body); + } + } + + /** + * Returns a connected {@link StorageStrategy} for operations on an existing pool (snapshots, + * delete, revert, grant/revoke). Does not require aggregate free space for the full pool size. + */ public static StorageStrategy getStrategyByStoragePoolDetails(Map details) { + return getStrategyByStoragePoolDetails(details, false); + } + + /** + * Same as {@link #getStrategyByStoragePoolDetails(Map)} but does not call + * {@link StorageStrategy#connect}. ASUP uses this for cluster GET and EMS POST only. + */ + public static StorageStrategy resolveStrategyFromPoolDetails(Map details) { if (details == null || details.isEmpty()) { - logger.error("getStrategyByStoragePoolDetails: Storage pool details are null or empty"); + logger.error("resolveStrategyFromPoolDetails: Storage pool details are null or empty"); throw new CloudRuntimeException("Storage pool details are null or empty"); } String protocol = details.get(OntapStorageConstants.PROTOCOL); OntapStorage ontapStorage = new OntapStorage(details.get(OntapStorageConstants.USERNAME), details.get(OntapStorageConstants.PASSWORD), details.get(OntapStorageConstants.STORAGE_IP), details.get(OntapStorageConstants.SVM_NAME), Long.parseLong(details.get(OntapStorageConstants.SIZE)), ProtocolType.valueOf(protocol)); - StorageStrategy storageStrategy = StorageProviderFactory.getStrategy(ontapStorage); - boolean isValid = storageStrategy.connect(); + return StorageProviderFactory.getStrategy(ontapStorage); + } + + public static StorageStrategy getStrategyByStoragePoolDetails(Map details, + boolean validateAggregatesForVolumeCreation) { + StorageStrategy storageStrategy = resolveStrategyFromPoolDetails(details); + boolean isValid = storageStrategy.connect(validateAggregatesForVolumeCreation); if (isValid) { logger.info("Connection to Ontap SVM [{}] successful", details.get(OntapStorageConstants.SVM_NAME)); return storageStrategy; @@ -159,4 +138,146 @@ public static String getLunName(String volName, String lunName) { return OntapStorageConstants.VOLUME_PATH_PREFIX + volName + OntapStorageConstants.SLASH + lunName; } + /** + * Converts a path stored in CloudStack (absolute LUN {@code /vol/<flexVol>/...} or already + * relative NFS file path) into the FlexVol-relative path expected by + * {@code POST /api/storage/file/clone}. + */ + public static String toFlexVolRelativePath(String path, String flexVolName) { + if (path == null || path.isEmpty()) { + return path; + } + if (flexVolName != null && !flexVolName.isEmpty()) { + String prefix = OntapStorageConstants.VOLUME_PATH_PREFIX + flexVolName + OntapStorageConstants.SLASH; + if (path.startsWith(prefix)) { + return path.substring(prefix.length()); + } + } + // Already relative (typical NFS uuid path) or unexpected absolute form — strip a leading slash. + return path.startsWith(OntapStorageConstants.SLASH) ? path.substring(1) : path; + } + + /** + * Builds the ONTAP LUN clone source name that points at a LUN inside a FlexVol snapshot. + * + *

Format required by {@code POST /api/storage/luns} when cloning from a snapshot: + * {@code /vol/<flexVol>/.snapshot/<snapshotName>/<relativeLunPath>}.

+ * + *

{@code clone.source.uuid} cannot identify a snapshot-resident LUN; name must be used.

+ */ + public static String toLunCloneSourcePathInSnapshot(String lunPath, String flexVolName, String snapshotName) { + if (flexVolName == null || flexVolName.isEmpty()) { + throw new InvalidParameterValueException("FlexVolume name is required to build a snapshot LUN path"); + } + if (snapshotName == null || snapshotName.isEmpty()) { + throw new InvalidParameterValueException("Snapshot name is required to build a snapshot LUN path"); + } + String relativeLunPath = toFlexVolRelativePath(lunPath, flexVolName); + if (relativeLunPath == null || relativeLunPath.isEmpty()) { + throw new InvalidParameterValueException("LUN path is required to build a snapshot LUN path"); + } + return OntapStorageConstants.VOLUME_PATH_PREFIX + flexVolName + + OntapStorageConstants.SNAPSHOT_PATH_SEGMENT + snapshotName + + OntapStorageConstants.SLASH + relativeLunPath; + } + + /** + * Builds an ONTAP-safe name token from user-provided snapshot text. + */ + public static String getOntapSnapshotName(String cloudStackSnapshotName) { + if (cloudStackSnapshotName == null || cloudStackSnapshotName.trim().isEmpty()) { + throw new InvalidParameterValueException("Snapshot name cannot be null or blank"); + } + String normalized = cloudStackSnapshotName.replaceAll("[^a-zA-Z0-9_]", "_"); + if (normalized.isEmpty()) { + normalized = "snapshot"; + } + if (!Character.isLetter(normalized.charAt(0))) { + normalized = "s_" + normalized; + } + if (normalized.length() > OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH) { + normalized = normalized.substring(0, OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH); + } + return normalized; + } + + /** + * Builds an ONTAP-safe snapshot name that preserves the CloudStack UI snapshot name + * and appends a uniqueness suffix. + */ + public static String buildOntapSnapshotName(String cloudStackSnapshotName, String uniquenessSuffix) { + String normalizedBase = (cloudStackSnapshotName == null || cloudStackSnapshotName.trim().isEmpty()) + ? "snapshot" + : getOntapSnapshotName(cloudStackSnapshotName); + String suffix = (uniquenessSuffix == null || uniquenessSuffix.isEmpty()) + ? "" + : "_" + uniquenessSuffix.replaceAll("[^a-zA-Z0-9_]", "_"); + int maxLength = OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH; + int maxBaseLength = maxLength - suffix.length(); + if (maxBaseLength <= 0) { + return normalizedBase.substring(0, maxLength); + } + if (normalizedBase.length() > maxBaseLength) { + normalizedBase = normalizedBase.substring(0, maxBaseLength); + } + return normalizedBase + suffix; + } + + /** + * Extracts a resource UUID from an ONTAP job description path. + * + *

Example: {@code POST /api/application/consistency-groups/{cg}/snapshots/{uuid}} + * with {@code pathSegment} {@code "/snapshots/"} returns the snapshot UUID.

+ */ + public static String extractUuidFromOntapJobDescription(String description, String pathSegment) { + if (description == null || pathSegment == null || pathSegment.isEmpty()) { + return null; + } + int idx = description.indexOf(pathSegment); + if (idx < 0) { + return null; + } + String remainder = description.substring(idx + pathSegment.length()).trim(); + if (remainder.isEmpty()) { + return null; + } + int queryIdx = remainder.indexOf('?'); + if (queryIdx >= 0) { + remainder = remainder.substring(0, queryIdx); + } + int slashIdx = remainder.indexOf('/'); + if (slashIdx >= 0) { + remainder = remainder.substring(0, slashIdx); + } + return remainder.isEmpty() ? null : remainder; + } + + /** + * Returns true when the exception indicates the ONTAP Object was already removed. + * Delete workflows treat a missing backend object as idempotent success. + */ + public static boolean isOntapObjectNotFoundError(Throwable error) { + if (error == null) { + return false; + } + if(error instanceof FeignException) { + FeignException feignException = (FeignException) error; + if (feignException.status() == 404) { + return true; + } + } + String message = error.getMessage(); + if (message != null) { + String lower = message.toLowerCase(); + if (lower.contains("404") || lower.contains("not found") || lower.contains("does not exist") + || lower.contains("entry doesn't exist")) { + return true; + } + } else { + logger.warn("Error message is null for exception: {}", error.getClass().getName()); + return false; + } + return false; + } + } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java index 7fa80a0b3fae..b27e4ba6db9a 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java @@ -20,21 +20,29 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.StringUtils; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.VMSnapshotOptions; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.feign.client.SnapshotFeignClient; -import org.apache.cloudstack.storage.feign.model.CliSnapshotRestoreRequest; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroup; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroupSnapshot; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroupVolume; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroupVolumeProvisioningOptions; import org.apache.cloudstack.storage.feign.model.FlexVolSnapshot; +import org.apache.cloudstack.storage.feign.model.Svm; +import org.apache.cloudstack.storage.feign.model.Job; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import org.apache.cloudstack.storage.service.StorageStrategy; @@ -72,25 +80,22 @@ import org.apache.cloudstack.storage.utils.OntapStorageConstants; /** - * VM Snapshot strategy for NetApp ONTAP managed storage using FlexVolume-level snapshots. + * VM Snapshot strategy for NetApp ONTAP managed storage using temporary consistency-group orchestration. * *

This strategy handles VM-level (instance) snapshots for VMs whose volumes - * reside on ONTAP managed primary storage. Instead of creating per-file clones - * (the old approach), it takes ONTAP FlexVolume-level snapshots via the - * ONTAP REST API ({@code POST /api/storage/volumes/{uuid}/snapshots}).

- * - *

Key Advantage:

- *

When multiple CloudStack disks (ROOT + DATA) reside on the same ONTAP - * FlexVolume, a single FlexVolume snapshot atomically captures all of them. - * This is both faster and more storage-efficient than per-file clones.

+ * reside on ONTAP managed primary storage. When VM volumes span multiple FlexVols, + * snapshot creation is coordinated through a temporary ONTAP consistency group (CG) + * and two-phase snapshot flow (start + commit). When all volumes share a single FlexVol, + * a direct FlexVol snapshot is used instead.

* *

Flow:

*
    *
  1. Group all VM volumes by their parent FlexVolume UUID
    2. *
  2. Freeze the VM via QEMU guest agent ({@code fsfreeze}) — if quiesce requested
    3. - *
  3. For each unique FlexVolume, create one ONTAP snapshot
    4. + *
  4. If VM spans multiple FlexVolumes: create temporary CG, start + commit CG snapshot (two-phase)
    5. + *
  5. If VM spans a single FlexVolume: create one FlexVol snapshot directly (no CG overhead)
    6. *
  6. Thaw the VM
    7. - *
  7. Record FlexVolume → snapshot UUID mappings in {@code vm_snapshot_details}
    8. + *
  8. Resolve FlexVolume → snapshot UUID mappings and persist in {@code vm_snapshot_details}
    9. *
* *

Metadata in vm_snapshot_details:

@@ -251,12 +256,14 @@ boolean allVolumesOnOntapManagedStorage(long vmId) { /** * Takes a VM-level snapshot by freezing the VM, creating ONTAP FlexVolume-level - * snapshots (one per unique FlexVolume), and then thawing the VM. + * snapshot(s), and then thawing the VM. * *

Volumes are grouped by their parent FlexVolume UUID (from storage pool details). - * For each unique FlexVolume, exactly one ONTAP snapshot is created via - * {@code POST /api/storage/volumes/{uuid}/snapshots}. This means if a VM has - * ROOT and DATA disks on the same FlexVolume, only one snapshot is created.

+ * When the VM spans more than one unique FlexVolume, a temporary ONTAP + * consistency group is used with two-phase snapshot semantics (start + commit) so + * all FlexVols are captured at the same point in time. When all VM volumes reside + * on a single FlexVolume, a direct per-FlexVol snapshot is taken instead — + * CG orchestration is unnecessary in that case.

* *

Memory Snapshots Not Supported: This strategy only supports disk-only * (crash-consistent) snapshots. Memory snapshots (snapshotmemory=true) are rejected @@ -286,7 +293,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { FreezeThawVMAnswer thawAnswer = null; long startFreeze = 0; - // Track which FlexVolume snapshots were created (for rollback) + // Track which FlexVolume snapshots were created (for rollback and detail persistence) List createdSnapshots = new ArrayList<>(); boolean result = false; @@ -338,7 +345,8 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { CreateVMSnapshotCommand ccmd = new CreateVMSnapshotCommand( userVm.getInstanceName(), userVm.getUuid(), target, volumeTOs, guestOS.getDisplayName()); - logger.info("takeVMSnapshot: Creating ONTAP FlexVolume VM Snapshot for VM [{}] with quiesce={}", userVm.getInstanceName(), quiesceVm); + logger.info("takeVMSnapshot: Creating ONTAP VM snapshot for VM [{}] with quiesce={}", + userVm.getInstanceName(), quiesceVm); // Prepare volume info list and calculate sizes for (VolumeObjectTO volumeObjectTO : volumeTOs) { @@ -375,56 +383,20 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { userVm.getInstanceName(), quiesceVm, vmIsRunning); } - // ── Step 2: Create FlexVolume-level snapshots ── + // ── Step 2: Create FlexVolume-level snapshot(s) ── try { String snapshotNameBase = buildSnapshotName(vmSnapshot); - for (Map.Entry entry : flexVolGroups.entrySet()) { - String flexVolUuid = entry.getKey(); - FlexVolGroupInfo groupInfo = entry.getValue(); - long startSnapshot = System.nanoTime(); - - // Build storage strategy from pool details to get the feign client - StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(groupInfo.poolDetails); - SnapshotFeignClient snapshotClient = storageStrategy.getSnapshotFeignClient(); - String authHeader = storageStrategy.getAuthHeader(); - - // Use the same snapshot name for all FlexVolumes in this VM snapshot - // (each FlexVolume gets its own independent snapshot with this name) - FlexVolSnapshot snapshotRequest = new FlexVolSnapshot(snapshotNameBase, - "CloudStack VM snapshot " + vmSnapshot.getName() + " for VM " + userVm.getInstanceName()); - - logger.info("takeVMSnapshot: Creating ONTAP FlexVolume snapshot [{}] on FlexVol UUID [{}] covering {} volume(s)", - snapshotNameBase, flexVolUuid, groupInfo.volumeIds.size()); - - JobResponse jobResponse = snapshotClient.createSnapshot(authHeader, flexVolUuid, snapshotRequest); - if (jobResponse == null || jobResponse.getJob() == null) { - throw new CloudRuntimeException("Failed to initiate FlexVolume snapshot on FlexVol UUID [" + flexVolUuid + "]"); - } - - // Poll for job completion - Boolean jobSucceeded = storageStrategy.jobPollForSuccess(jobResponse.getJob().getUuid(), 30, 2000); - if (!jobSucceeded) { - throw new CloudRuntimeException("FlexVolume snapshot job failed on FlexVol UUID [" + flexVolUuid + "]"); - } - - // Retrieve the created snapshot UUID by name - String snapshotUuid = resolveSnapshotUuid(snapshotClient, authHeader, flexVolUuid, snapshotNameBase); - - String protocol = groupInfo.poolDetails.get(OntapStorageConstants.PROTOCOL); - - // Create one detail per CloudStack volume in this FlexVol group (for single-file restore during revert) - for (Long volumeId : groupInfo.volumeIds) { - String volumePath = resolveVolumePathOnOntap(volumeId, protocol, groupInfo.poolDetails); - FlexVolSnapshotDetail detail = new FlexVolSnapshotDetail( - flexVolUuid, snapshotUuid, snapshotNameBase, volumePath, groupInfo.poolId, protocol); - createdSnapshots.add(detail); - } - - logger.info("takeVMSnapshot: ONTAP FlexVolume snapshot [{}] (uuid={}) on FlexVol [{}] completed in {} ms. Covers volumes: {}", - snapshotNameBase, snapshotUuid, flexVolUuid, - TimeUnit.MILLISECONDS.convert(System.nanoTime() - startSnapshot, TimeUnit.NANOSECONDS), - groupInfo.volumeIds); + // CG orchestration is only required when VM disks span multiple FlexVols. + // A single FlexVol already provides atomic capture for all volumes on that FlexVol. + if (flexVolGroups.size() > 1) { + logger.info("takeVMSnapshot: VM [{}] spans {} FlexVol(s); using temporary CG two-phase snapshot flow", + userVm.getInstanceName(), flexVolGroups.size()); + createVmSnapshotsViaTemporaryCg(vmSnapshot, userVm, flexVolGroups, snapshotNameBase, createdSnapshots); + } else { + logger.info("takeVMSnapshot: VM [{}] spans a single FlexVol; using direct FlexVol snapshot flow", + userVm.getInstanceName()); + createVmSnapshotsViaSingleFlexVol(vmSnapshot, userVm, flexVolGroups, snapshotNameBase, createdSnapshots); } } finally { // ── Step 3: Thaw the VM (only if it was frozen, always even on error) ── @@ -456,7 +428,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { answer.setVolumeTOs(volumeTOs); processAnswer(vmSnapshotVO, userVm, answer, null); - logger.info("takeVMSnapshot: ONTAP FlexVolume VM Snapshot [{}] created successfully for VM [{}] ({} FlexVol snapshot(s))", + logger.info("takeVMSnapshot: ONTAP VM Snapshot [{}] created successfully for VM [{}] ({} detail row(s))", vmSnapshot.getName(), userVm.getInstanceName(), createdSnapshots.size()); long newChainSize = 0; @@ -668,16 +640,140 @@ Map groupVolumesByFlexVol(List volumeT } /** - * Builds a deterministic, ONTAP-safe snapshot name for a VM snapshot. - * Format: {@code vmsnap__} + * Creates VM snapshot artifacts via direct FlexVol snapshot API. + * + *

Used when all VM volumes map to a single FlexVol. In that case a CG is not + * needed because one FlexVol snapshot already captures every disk atomically.

*/ - String buildSnapshotName(VMSnapshot vmSnapshot) { - String name = "vmsnap_" + vmSnapshot.getId() + "_" + System.currentTimeMillis(); - // ONTAP snapshot names: max 255 chars, must start with letter, only alphanumeric and underscores - if (name.length() > OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH) { - name = name.substring(0, OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH); + void createVmSnapshotsViaSingleFlexVol(VMSnapshot vmSnapshot, UserVm userVm, + Map flexVolGroups, + String snapshotNameBase, + List createdSnapshots) { + for (Map.Entry entry : flexVolGroups.entrySet()) { + String flexVolUuid = entry.getKey(); + FlexVolGroupInfo groupInfo = entry.getValue(); + long startSnapshot = System.nanoTime(); + + StorageStrategy storageStrategy = resolveStorageStrategy(groupInfo.poolDetails); + SnapshotFeignClient snapshotClient = storageStrategy.getSnapshotFeignClient(); + String authHeader = storageStrategy.getAuthHeader(); + + FlexVolSnapshot snapshotRequest = new FlexVolSnapshot(snapshotNameBase, + "CloudStack VM snapshot " + vmSnapshot.getName() + " for VM " + userVm.getInstanceName()); + + logger.info("takeVMSnapshot: [FlexVol] Creating snapshot [{}] on FlexVol UUID [{}] covering {} volume(s)", + snapshotNameBase, flexVolUuid, groupInfo.volumeIds.size()); + + JobResponse jobResponse = snapshotClient.createSnapshot(authHeader, flexVolUuid, snapshotRequest); + if (jobResponse == null || jobResponse.getJob() == null) { + throw new CloudRuntimeException("Failed to initiate FlexVolume snapshot on FlexVol UUID [" + flexVolUuid + "]"); + } + + Boolean jobSucceeded = storageStrategy.jobPollForSuccess(jobResponse.getJob().getUuid(), 30, 2000); + if (!jobSucceeded) { + throw new CloudRuntimeException("FlexVolume snapshot job failed on FlexVol UUID [" + flexVolUuid + "]"); + } + + String snapshotUuid = resolveSnapshotUuid(snapshotClient, authHeader, flexVolUuid, snapshotNameBase); + String protocol = groupInfo.poolDetails.get(OntapStorageConstants.PROTOCOL); + + for (Long volumeId : groupInfo.volumeIds) { + String volumePath = resolveVolumePathOnOntap(volumeId, protocol, groupInfo.poolDetails); + createdSnapshots.add(new FlexVolSnapshotDetail( + flexVolUuid, snapshotUuid, snapshotNameBase, volumePath, groupInfo.poolId, protocol)); + } + + logger.debug("takeVMSnapshot: [FlexVol] Snapshot [{}] (uuid={}) on FlexVol [{}] completed in {} ms. Covers volumes: {}", + snapshotNameBase, snapshotUuid, flexVolUuid, + TimeUnit.MILLISECONDS.convert(System.nanoTime() - startSnapshot, TimeUnit.NANOSECONDS), + groupInfo.volumeIds); } - return name; + } + + /** + * Creates VM snapshot artifacts via temporary consistency-group two-phase flow. + * + *

Used when VM volumes span multiple FlexVols and require a consistent + * point-in-time capture across all participating FlexVolumes.

+ */ + void createVmSnapshotsViaTemporaryCg(VMSnapshot vmSnapshot, UserVm userVm, + Map flexVolGroups, + String snapshotNameBase, + List createdSnapshots) { + String tempCgName = buildTemporaryConsistencyGroupName(vmSnapshot); + String tempCgUuid = null; + String cgSnapshotUuid = null; + long cgFlowStart = System.nanoTime(); + + // All volumes in a VM snapshot belong to ONTAP-managed pools and share the same ONTAP credentials. + // Use any one FlexVol group to obtain strategy/client objects for this operation. + FlexVolGroupInfo referenceGroup = flexVolGroups.values().iterator().next(); + StorageStrategy storageStrategy = resolveStorageStrategy(referenceGroup.poolDetails); + SnapshotFeignClient snapshotClient = storageStrategy.getSnapshotFeignClient(); + String authHeader = storageStrategy.getAuthHeader(); + + try { + logger.info("takeVMSnapshot: [CG:Create] Creating temporary consistency group [{}] for VM [{}] over {} FlexVol(s)", + tempCgName, userVm.getInstanceName(), flexVolGroups.size()); + tempCgUuid = createTemporaryConsistencyGroup(snapshotClient, storageStrategy, authHeader, tempCgName, + resolveConsistencyGroupScope(flexVolGroups), flexVolGroups.keySet()); + + logger.info("takeVMSnapshot: [CG:Start] Starting phase-1 snapshot [{}] for temporary consistency group [{}]", + snapshotNameBase, tempCgUuid); + cgSnapshotUuid = resolveStartedConsistencyGroupSnapshotUuid(snapshotClient, storageStrategy, + authHeader, tempCgUuid, snapshotNameBase); + + logger.info("takeVMSnapshot: [CG:Commit] Committing phase-2 snapshot [{}] (uuid={}) for temporary consistency group [{}]", + snapshotNameBase, cgSnapshotUuid, tempCgUuid); + commitConsistencyGroupSnapshot(snapshotClient, storageStrategy, authHeader, tempCgUuid, cgSnapshotUuid); + + // Resolve per-FlexVol snapshot UUIDs and build one detail entry per CloudStack volume. + for (Map.Entry entry : flexVolGroups.entrySet()) { + String flexVolUuid = entry.getKey(); + FlexVolGroupInfo groupInfo = entry.getValue(); + String snapshotUuid = resolveSnapshotUuid(snapshotClient, authHeader, flexVolUuid, snapshotNameBase); + String protocol = groupInfo.poolDetails.get(OntapStorageConstants.PROTOCOL); + + for (Long volumeId : groupInfo.volumeIds) { + String volumePath = resolveVolumePathOnOntap(volumeId, protocol, groupInfo.poolDetails); + createdSnapshots.add(new FlexVolSnapshotDetail( + flexVolUuid, snapshotUuid, snapshotNameBase, volumePath, groupInfo.poolId, protocol)); + } + + logger.debug("takeVMSnapshot: [CG:Resolve] Snapshot [{}] resolved to FlexVol snapshot uuid [{}] for FlexVol [{}], volumes={}", + snapshotNameBase, snapshotUuid, flexVolUuid, groupInfo.volumeIds); + } + } finally { + // CG is only a transaction boundary; remove it after commit/failure and keep snapshots intact. + if (tempCgUuid != null) { + try { + logger.info("takeVMSnapshot: [CG:Cleanup] Deleting temporary consistency group [{}]", tempCgUuid); + deleteTemporaryConsistencyGroup(snapshotClient, storageStrategy, authHeader, tempCgUuid); + } catch (Exception cleanupEx) { + logger.warn("takeVMSnapshot: Failed to delete temporary consistency group [{}]: {}", + tempCgUuid, cleanupEx.getMessage()); + } + } + } + + logger.info("takeVMSnapshot: Temporary consistency-group two-phase flow completed for VM [{}] in {} ms. CG snapshot uuid={}, detail rows={}", + userVm.getInstanceName(), + TimeUnit.MILLISECONDS.convert(System.nanoTime() - cgFlowStart, TimeUnit.NANOSECONDS), + cgSnapshotUuid, createdSnapshots.size()); + } + + /** + * Builds an ONTAP-safe snapshot name from the CloudStack VM snapshot UI name. + */ + String buildSnapshotName(VMSnapshot vmSnapshot) { + return OntapStorageUtils.buildOntapSnapshotName(vmSnapshot.getName(), "vm" + vmSnapshot.getId()); + } + + /** + * Wrapper for static utility to simplify unit testing and keep call sites explicit. + */ + StorageStrategy resolveStorageStrategy(Map poolDetails) { + return OntapStorageUtils.getStrategyByStoragePoolDetails(poolDetails); } /** @@ -695,6 +791,227 @@ String resolveSnapshotUuid(SnapshotFeignClient client, String authHeader, return response.getRecords().get(0).getUuid(); } + /** + * Builds a deterministic temporary CG name for the VM snapshot transaction. + */ + String buildTemporaryConsistencyGroupName(VMSnapshot vmSnapshot) { + return OntapStorageConstants.ONTAP_TEMP_CG_PREFIX + vmSnapshot.getId(); + } + + /** + * Validates and returns the ONTAP scope for a temporary consistency group. + * + *

CG membership requires all FlexVols on the same ONTAP management endpoint and SVM. + * SVM name alone is not sufficient — different clusters may reuse names such as {@code vs0}. + * Identity uses {@code storageIP} plus {@code svmUUID} when persisted, otherwise + * {@code storageIP} plus {@code svmName} for legacy pools.

+ */ + ConsistencyGroupScope resolveConsistencyGroupScope(Map flexVolGroups) { + ConsistencyGroupScope scope = null; + for (FlexVolGroupInfo group : flexVolGroups.values()) { + ConsistencyGroupScope candidate = consistencyGroupScopeFromPoolDetails(group.poolDetails, group.poolId); + if (scope == null) { + scope = candidate; + } else if (!scope.matches(candidate)) { + throw new CloudRuntimeException("ONTAP consistency groups require all VM volumes on the same " + + "ONTAP cluster and SVM. Found [" + scope + "] and [" + candidate + "]"); + } + } + return scope; + } + + ConsistencyGroupScope consistencyGroupScopeFromPoolDetails(Map poolDetails, long poolId) { + String storageIp = poolDetails.get(OntapStorageConstants.STORAGE_IP); + if (StringUtils.isBlank(storageIp)) { + throw new CloudRuntimeException("ONTAP storage management IP not found in pool details for pool [" + + poolId + "]"); + } + String svmName = poolDetails.get(OntapStorageConstants.SVM_NAME); + if (StringUtils.isBlank(svmName)) { + throw new CloudRuntimeException("SVM name not found in pool details for pool [" + poolId + "]"); + } + String svmUuid = poolDetails.get(OntapStorageConstants.SVM_UUID); + return new ConsistencyGroupScope(storageIp.trim(), svmName.trim(), + svmUuid != null && !svmUuid.trim().isEmpty() ? svmUuid.trim() : null); + } + + /** + * Creates a temporary consistency group for the involved FlexVol UUIDs and returns its UUID. + */ + String createTemporaryConsistencyGroup(SnapshotFeignClient client, StorageStrategy storageStrategy, + String authHeader, String cgName, ConsistencyGroupScope cgScope, + Set flexVolUuids) { + if (cgScope == null) { + throw new CloudRuntimeException("ONTAP consistency group scope is required to create CG [" + cgName + "]"); + } + + List volumeRefs = new ArrayList<>(); + for (String flexVolUuid : flexVolUuids) { + ConsistencyGroupVolumeProvisioningOptions provisioningOptions = + new ConsistencyGroupVolumeProvisioningOptions(OntapStorageConstants.CG_VOLUME_PROVISIONING_ACTION_ADD); + + ConsistencyGroupVolume volumeRef = new ConsistencyGroupVolume(); + volumeRef.setUuid(flexVolUuid); + volumeRef.setProvisioningOptions(provisioningOptions); + volumeRefs.add(volumeRef); + } + + ConsistencyGroup payload = new ConsistencyGroup(); + payload.setName(cgName); + payload.setSvm(cgScope.toOntapSvm()); + payload.setVolumes(volumeRefs); + + JobResponse response = client.createConsistencyGroup(authHeader, payload); + storageStrategy.pollJobIfPresent(response, "create temporary consistency group " + cgName); + + String cgUuid = resolveConsistencyGroupUuidByName(client, authHeader, cgName, cgScope); + if (cgUuid == null || cgUuid.isEmpty()) { + throw new CloudRuntimeException("Unable to resolve temporary consistency group UUID for [" + cgName + "]"); + } + return cgUuid; + } + + /** + * Starts phase-1 of the two-phase CG snapshot and returns the CG snapshot UUID when ONTAP exposes it in the job record. + */ + String startConsistencyGroupSnapshot(SnapshotFeignClient client, StorageStrategy storageStrategy, + String authHeader, String cgUuid, String snapshotName) { + ConsistencyGroupSnapshot payload = new ConsistencyGroupSnapshot(snapshotName, "start"); + JobResponse response = client.createConsistencyGroupSnapshot(authHeader, cgUuid, payload); + Job completedJob = storageStrategy.pollJobIfPresentAndGetCompletedJob(response, + "start CG snapshot " + snapshotName + " for " + cgUuid); + if (completedJob == null) { + return null; + } + String snapshotUuid = OntapStorageUtils.extractUuidFromOntapJobDescription( + completedJob.getDescription(), "/snapshots/"); + if (snapshotUuid != null) { + logger.info("takeVMSnapshot: [CG:Start] Resolved CG snapshot UUID [{}] from ONTAP job for snapshot [{}]", + snapshotUuid, snapshotName); + } + return snapshotUuid; + } + + /** + * Commits phase-2 of the started CG snapshot. + */ + void commitConsistencyGroupSnapshot(SnapshotFeignClient client, StorageStrategy storageStrategy, + String authHeader, String cgUuid, String snapshotUuid) { + ConsistencyGroupSnapshot payload = new ConsistencyGroupSnapshot(); + payload.setAction("commit"); + JobResponse response = client.commitConsistencyGroupSnapshot(authHeader, cgUuid, snapshotUuid, payload); + storageStrategy.pollJobIfPresent(response, "commit CG snapshot " + snapshotUuid + " for " + cgUuid); + } + + /** + * Deletes the temporary consistency group used as a transaction boundary. + */ + void deleteTemporaryConsistencyGroup(SnapshotFeignClient client, StorageStrategy storageStrategy, + String authHeader, String cgUuid) { + JobResponse response = client.deleteConsistencyGroup(authHeader, cgUuid); + storageStrategy.pollJobIfPresent(response, "delete temporary consistency group " + cgUuid); + } + + /** + * Resolves consistency group UUID by name within the given ONTAP cluster/SVM scope. + */ + String resolveConsistencyGroupUuidByName(SnapshotFeignClient client, String authHeader, + String cgName, ConsistencyGroupScope cgScope) { + Map query = new HashMap<>(); + query.put("name", cgName); + cgScope.applySvmQueryFilter(query); + query.put("fields", "uuid,name"); + OntapResponse response = client.getConsistencyGroups(authHeader, query); + if (response == null || response.getRecords() == null || response.getRecords().isEmpty()) { + return null; + } + ConsistencyGroup consistencyGroup = response.getRecords().get(0); + return consistencyGroup != null ? consistencyGroup.getUuid() : null; + } + + /** + * Resolves the started CG snapshot UUID after phase-1, using the job record when available and polling GET otherwise. + */ + String resolveStartedConsistencyGroupSnapshotUuid(SnapshotFeignClient client, StorageStrategy storageStrategy, + String authHeader, String cgUuid, String snapshotName) { + String snapshotUuidFromJob = startConsistencyGroupSnapshot(client, storageStrategy, authHeader, cgUuid, snapshotName); + if (snapshotUuidFromJob != null && !snapshotUuidFromJob.isEmpty()) { + return snapshotUuidFromJob; + } + return resolveConsistencyGroupSnapshotUuid(client, storageStrategy, authHeader, cgUuid, snapshotName); + } + + /** + * Resolves consistency group snapshot UUID by name with retries (ONTAP list can lag behind job success). + */ + String resolveConsistencyGroupSnapshotUuid(SnapshotFeignClient client, StorageStrategy storageStrategy, + String authHeader, String cgUuid, String snapshotName) { + int maxRetries = OntapStorageConstants.ONTAP_CG_SNAPSHOT_RESOLVE_MAX_RETRIES; + int pollIntervalMs = OntapStorageConstants.ONTAP_CG_SNAPSHOT_RESOLVE_POLL_INTERVAL_MS; + + for (int attempt = 1; attempt <= maxRetries; attempt++) { + String snapshotUuid = lookupConsistencyGroupSnapshotUuid(client, authHeader, cgUuid, snapshotName); + if (snapshotUuid != null) { + if (attempt > 1) { + logger.info("takeVMSnapshot: [CG:Resolve] CG snapshot [{}] resolved on attempt {}/{}", + snapshotName, attempt, maxRetries); + } + return snapshotUuid; + } + if (attempt < maxRetries) { + logger.debug("takeVMSnapshot: [CG:Resolve] CG snapshot [{}] not yet visible in CG [{}], retry {}/{}", + snapshotName, cgUuid, attempt, maxRetries); + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CloudRuntimeException("Interrupted while resolving CG snapshot [" + snapshotName + "]", e); + } + } + } + + throw new CloudRuntimeException("Unable to resolve consistency group snapshot UUID for snapshot [" + + snapshotName + "] in CG [" + cgUuid + "] after " + maxRetries + " attempts"); + } + + /** + * Single GET attempt: try to match by name, + * then fall back to listing all CG snapshots in this group (And it would be one + * always since workflow is keep deleting the CG). + */ + String lookupConsistencyGroupSnapshotUuid(SnapshotFeignClient client, String authHeader, + String cgUuid, String snapshotName) { + Map query = new HashMap<>(); + query.put("name", snapshotName); + query.put("fields", "uuid,name"); + OntapResponse response = client.getConsistencyGroupSnapshots(authHeader, cgUuid, query); + String snapshotUuid = findConsistencyGroupSnapshotUuidInRecords(response, snapshotName); + if (snapshotUuid != null) { + return snapshotUuid; + } + + Map listAllQuery = new HashMap<>(); + listAllQuery.put("fields", "uuid,name"); + OntapResponse allSnapshots = client.getConsistencyGroupSnapshots(authHeader, cgUuid, listAllQuery); + return findConsistencyGroupSnapshotUuidInRecords(allSnapshots, snapshotName); + } + + private String findConsistencyGroupSnapshotUuidInRecords(OntapResponse response, + String snapshotName) { + if (response == null || response.getRecords() == null || response.getRecords().isEmpty()) { + return null; + } + for (ConsistencyGroupSnapshot record : response.getRecords()) { + if (record != null && snapshotName.equals(record.getName())) { + String uuid = record.getUuid(); + if (uuid != null && !uuid.isEmpty()) { + return uuid; + } + } + } + return null; + } + /** * Resolves the ONTAP-side path of a CloudStack volume within its FlexVolume. * @@ -735,17 +1052,13 @@ String resolveVolumePathOnOntap(Long volumeId, String protocol, Map poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(detail.poolId); - StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(poolDetails); - SnapshotFeignClient client = storageStrategy.getSnapshotFeignClient(); - String authHeader = storageStrategy.getAuthHeader(); + StorageStrategy storageStrategy = resolveStorageStrategy(poolDetails); logger.info("rollbackFlexVolSnapshot: Rolling back FlexVol snapshot [{}] (uuid={}) on FlexVol [{}]", detail.snapshotName, detail.snapshotUuid, detail.flexVolUuid); - JobResponse jobResponse = client.deleteSnapshot(authHeader, detail.flexVolUuid, detail.snapshotUuid); - if (jobResponse != null && jobResponse.getJob() != null) { - storageStrategy.jobPollForSuccess(jobResponse.getJob().getUuid(), 10, 2000); - } + storageStrategy.deleteFlexVolSnapshotForCloudStackVolume( + detail.flexVolUuid, detail.snapshotUuid, detail.snapshotName); } catch (Exception e) { logger.error("rollbackFlexVolSnapshot: Rollback of FlexVol snapshot failed: {}", e.getMessage(), e); } @@ -757,36 +1070,52 @@ void rollbackFlexVolSnapshot(FlexVolSnapshotDetail detail) { *

Since there is one detail row per CloudStack volume, multiple rows may reference * the same FlexVol + snapshot combination. This method deduplicates to delete each * underlying ONTAP snapshot only once.

+ * + *

Detail rows are removed only after the underlying ONTAP snapshot delete succeeds + * (or was already deleted for the same FlexVol+snapshot pair in this pass). If delete + * throws, the detail row is retained so a retry can still find the ONTAP snapshot.

*/ void deleteFlexVolSnapshots(List flexVolDetails) { - // Track which FlexVol+Snapshot pairs have already been deleted Map deletedSnapshots = new HashMap<>(); + CloudRuntimeException deleteFailure = null; for (VMSnapshotDetailsVO detailVO : flexVolDetails) { FlexVolSnapshotDetail detail = FlexVolSnapshotDetail.parse(detailVO.getValue()); String dedupeKey = detail.flexVolUuid + "::" + detail.snapshotUuid; - // Only delete the ONTAP snapshot once per FlexVol+Snapshot pair - if (!deletedSnapshots.containsKey(dedupeKey)) { - Map poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(detail.poolId); - StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(poolDetails); - SnapshotFeignClient client = storageStrategy.getSnapshotFeignClient(); - String authHeader = storageStrategy.getAuthHeader(); + try { + if (!deletedSnapshots.containsKey(dedupeKey)) { + Map poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(detail.poolId); + StorageStrategy storageStrategy = resolveStorageStrategy(poolDetails); + + logger.info("deleteFlexVolSnapshots: Deleting ONTAP FlexVol snapshot [{}] (uuid={}) on FlexVol [{}]", + detail.snapshotName, detail.snapshotUuid, detail.flexVolUuid); - logger.info("deleteFlexVolSnapshots: Deleting ONTAP FlexVol snapshot [{}] (uuid={}) on FlexVol [{}]", - detail.snapshotName, detail.snapshotUuid, detail.flexVolUuid); + storageStrategy.deleteFlexVolSnapshotForCloudStackVolume( + detail.flexVolUuid, detail.snapshotUuid, detail.snapshotName); - JobResponse jobResponse = client.deleteSnapshot(authHeader, detail.flexVolUuid, detail.snapshotUuid); - if (jobResponse != null && jobResponse.getJob() != null) { - storageStrategy.jobPollForSuccess(jobResponse.getJob().getUuid(), 30, 2000); + deletedSnapshots.put(dedupeKey, Boolean.TRUE); + logger.info("deleteFlexVolSnapshots: Deleted ONTAP FlexVol snapshot [{}] on FlexVol [{}]", + detail.snapshotName, detail.flexVolUuid); + } + } catch (Exception e) { + logger.error("deleteFlexVolSnapshots: Failed to delete ONTAP FlexVol snapshot [{}] (uuid={}) " + + "on FlexVol [{}] for detail [{}]: {}", + detail.snapshotName, detail.snapshotUuid, detail.flexVolUuid, detailVO.getId(), e.getMessage(), e); + if (deleteFailure == null) { + deleteFailure = e instanceof CloudRuntimeException + ? (CloudRuntimeException) e + : new CloudRuntimeException("Failed to delete ONTAP FlexVol snapshot: " + e.getMessage(), e); + } + } finally { + if (deletedSnapshots.containsKey(dedupeKey)) { + vmSnapshotDetailsDao.remove(detailVO.getId()); } - - deletedSnapshots.put(dedupeKey, Boolean.TRUE); - logger.info("deleteFlexVolSnapshots: Deleted ONTAP FlexVol snapshot [{}] on FlexVol [{}]", detail.snapshotName, detail.flexVolUuid); } + } - // Always remove the DB detail row - vmSnapshotDetailsDao.remove(detailVO.getId()); + if (deleteFailure != null) { + throw deleteFailure; } } @@ -818,41 +1147,24 @@ void revertFlexVolSnapshots(List flexVolDetails) { } Map poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(detail.poolId); - StorageStrategy storageStrategy = OntapStorageUtils.getStrategyByStoragePoolDetails(poolDetails); - SnapshotFeignClient snapshotClient = storageStrategy.getSnapshotFeignClient(); - String authHeader = storageStrategy.getAuthHeader(); + StorageStrategy storageStrategy = resolveStorageStrategy(poolDetails); - // Get SVM name and FlexVolume name from pool details - String svmName = poolDetails.get(OntapStorageConstants.SVM_NAME); String flexVolName = poolDetails.get(OntapStorageConstants.VOLUME_NAME); - - if (svmName == null || svmName.isEmpty()) { - throw new CloudRuntimeException("SVM name not found in pool details for pool [" + detail.poolId + "]"); - } if (flexVolName == null || flexVolName.isEmpty()) { throw new CloudRuntimeException("FlexVolume name not found in pool details for pool [" + detail.poolId + "]"); } - // The path must start with "/" for the ONTAP CLI API String ontapFilePath = detail.volumePath.startsWith("/") ? detail.volumePath : "/" + detail.volumePath; logger.info("revertFlexVolSnapshots: Restoring volume [{}] from FlexVol snapshot [{}] on FlexVol [{}] (protocol={})", ontapFilePath, detail.snapshotName, flexVolName, detail.protocol); - // Use CLI-based restore API: POST /api/private/cli/volume/snapshot/restore-file - CliSnapshotRestoreRequest restoreRequest = new CliSnapshotRestoreRequest( - svmName, flexVolName, detail.snapshotName, ontapFilePath); - - JobResponse jobResponse = snapshotClient.restoreFileFromSnapshotCli(authHeader, restoreRequest); + JobResponse jobResponse = storageStrategy.revertSnapshotForCloudStackVolume( + detail.snapshotName, detail.flexVolUuid, detail.snapshotUuid, + detail.volumePath, null, flexVolName); - if (jobResponse != null && jobResponse.getJob() != null) { - Boolean success = storageStrategy.jobPollForSuccess(jobResponse.getJob().getUuid(), 60, 2000); - if (!success) { - throw new CloudRuntimeException("Snapshot file restore failed for volume path [" + - ontapFilePath + "] from snapshot [" + detail.snapshotName + - "] on FlexVol [" + flexVolName + "]"); - } - } + storageStrategy.executeCliSfsrRestore(jobResponse, + "VM snapshot file restore for path [" + ontapFilePath + "] from snapshot [" + detail.snapshotName + "]"); logger.info("revertFlexVolSnapshots: Successfully restored volume [{}] from snapshot [{}] on FlexVol [{}]", ontapFilePath, detail.snapshotName, flexVolName); @@ -877,6 +1189,69 @@ static class FlexVolGroupInfo { } } + /** + * Identifies the ONTAP cluster management endpoint and SVM for CG operations. + */ + static class ConsistencyGroupScope { + final String storageIp; + final String svmName; + final String svmUuid; + + ConsistencyGroupScope(String storageIp, String svmName, String svmUuid) { + this.storageIp = storageIp; + this.svmName = svmName; + this.svmUuid = svmUuid; + } + + boolean matches(ConsistencyGroupScope other) { + return other != null && identityKey().equals(other.identityKey()); + } + + String identityKey() { + if (svmUuid != null && !svmUuid.isEmpty()) { + return storageIp + "|" + svmUuid; + } + return storageIp + "|" + svmName; + } + + Map toOntapSvmReference() { + Svm svm = toOntapSvm(); + Map svmRef = new LinkedHashMap<>(); + if (svm.getUuid() != null && !svm.getUuid().isEmpty()) { + svmRef.put("uuid", svm.getUuid()); + } else { + svmRef.put("name", svm.getName()); + } + return svmRef; + } + + Svm toOntapSvm() { + Svm svm = new Svm(); + if (svmUuid != null && !svmUuid.isEmpty()) { + svm.setUuid(svmUuid); + } else { + svm.setName(svmName); + } + return svm; + } + + void applySvmQueryFilter(Map query) { + if (svmUuid != null && !svmUuid.isEmpty()) { + query.put("svm.uuid", svmUuid); + } else { + query.put("svm.name", svmName); + } + } + + @Override + public String toString() { + if (svmUuid != null && !svmUuid.isEmpty()) { + return "storageIP=" + storageIp + ", svmUUID=" + svmUuid + ", svmName=" + svmName; + } + return "storageIP=" + storageIp + ", svmName=" + svmName; + } + } + /** * Holds the metadata for a single volume's FlexVolume snapshot entry (used during create and for * serialization/deserialization to/from vm_snapshot_details). diff --git a/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml b/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml index bb907871469c..bfbb430b7c17 100644 --- a/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml +++ b/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml @@ -33,4 +33,10 @@ + + + + diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/asup/OntapAsupManagerTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/asup/OntapAsupManagerTest.java new file mode 100644 index 000000000000..700ae8f83e2b --- /dev/null +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/asup/OntapAsupManagerTest.java @@ -0,0 +1,747 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.asup; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.server.ManagementService; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; +import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.utils.OntapConfigurationManager; +import org.apache.cloudstack.storage.utils.OntapStorageConstants; +import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class OntapAsupManagerTest { + + // ── DAOs ────────────────────────────────────────────────────────────────── + @Mock private PrimaryDataStoreDao storagePoolDao; + @Mock private StoragePoolDetailsDao storagePoolDetailsDao; + @Mock private VolumeDao volumeDao; + @Mock private SnapshotDao snapshotDao; + @Mock private VMSnapshotDao vmSnapshotDao; + @Mock private ManagementService managementService; + @Mock private ManagementServerHostDao managementServerHostDao; + + @InjectMocks + private OntapAsupManager asupManager; + + // ── Common fixtures ────────────────────────────────────────────────────── + private StoragePoolVO pool; + private Map poolDetails; + private StorageStrategy mockStrategy; + private Cluster mockCluster; + + @BeforeEach + void setUp() { + pool = mock(StoragePoolVO.class); + lenient().when(pool.getId()).thenReturn(1L); + lenient().when(pool.getName()).thenReturn("ontap-pool-1"); + lenient().when(pool.getStatus()).thenReturn(StoragePoolStatus.Up); + + poolDetails = new HashMap<>(); + poolDetails.put(OntapStorageConstants.STORAGE_IP, "192.168.1.10"); + poolDetails.put(OntapStorageConstants.PROTOCOL, "NFS3"); + poolDetails.put(OntapStorageConstants.SVM_NAME, "svm1"); + poolDetails.put(OntapStorageConstants.VOLUME_UUID, "fv-uuid-1"); + poolDetails.put(OntapStorageConstants.VOLUME_NAME, "fv-name-1"); + + mockStrategy = mock(StorageStrategy.class); + + mockCluster = mock(Cluster.class); + lenient().when(mockCluster.getUuid()).thenReturn("cluster-uuid-1"); + lenient().when(mockCluster.getName()).thenReturn("ontap-cluster-1"); + lenient().when(mockCluster.getModel()).thenReturn("AFF-A400"); + lenient().when(mockCluster.getPlatformType()) + .thenReturn(OntapStorageConstants.ASUP_PLATFORM_TYPE_PERFORMANCE); + lenient().when(managementService.getVersion()).thenReturn("4.23.0.0-SNAPSHOT"); + lenient().when(managementServerHostDao.listAll()).thenReturn(Collections.emptyList()); + } + + // ────────────────────────────────────────────────────────────────────────── + // pushAsupTelemetry – no pools + // ────────────────────────────────────────────────────────────────────────── + + @Test + void pushAsupTelemetry_noOntapPools_sendsNoMessages() { + when(storagePoolDao.findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME)) + .thenReturn(Collections.emptyList()); + + asupManager.pushAsupTelemetry(); + + verify(mockStrategy, never()).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Message count / event-id routing + // ────────────────────────────────────────────────────────────────────────── + + @Test + void pushAsupForStoragePool_newCluster_sendsHeartbeatThenPoolMessage() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + // heartbeat (event-id 0) + pool (event-id 1) = 2 messages + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + + List msgs = cap.getAllValues(); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_HEARTBEAT, msgs.get(0).getEventId()); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL, msgs.get(1).getEventId()); + assertTrue(msgs.get(0).getEventDescription().contains("\"managementServerCount\":0"), + msgs.get(0).getEventDescription()); + assertTrue(msgs.get(0).getEventDescription().contains("\"ontapClusterModel\":\"AFF-A400\""), + msgs.get(0).getEventDescription()); + assertTrue(msgs.get(0).getEventDescription().contains("\"ontapPlatformType\":\"performance\""), + msgs.get(0).getEventDescription()); + } + + @Test + void pushAsupForStoragePool_cachedStorageIp_sendsOnlyPoolMessage() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + Map clientsByStorageIp = new HashMap<>(); + clientsByStorageIp.put("192.168.1.10", new OntapAsupManager.AsupClusterClient(mockStrategy, mockCluster)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + asupManager.pushAsupForStoragePool(pool, clientsByStorageIp); + u.verify(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any()), never()); + } + + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(1)).sendAsupMessage(cap.capture()); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL, cap.getValue().getEventId()); + verify(mockStrategy, never()).getClusterInfo(); + } + + @Test + void heartbeat_clusterModelUnknownWhenNodesGetReturnsNull() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(mockCluster.getModel()).thenReturn(null); + when(mockCluster.getPlatformType()).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + String heartbeat = cap.getAllValues().get(0).getEventDescription(); + assertTrue(heartbeat.contains("\"ontapClusterModel\":\"unknown\""), heartbeat); + assertTrue(heartbeat.contains("\"ontapPlatformType\":\"unknown\""), heartbeat); + } + + @Test + void pushAsupForStoragePool_strategyThrows_doesNotPropagateException() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())) + .thenThrow(new RuntimeException("connection refused")); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + verify(mockStrategy, never()).sendAsupMessage(any()); + } + + @Test + void pushAsupForStoragePool_poolDetailsEmpty_skipsPool() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(Collections.emptyMap()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())) + .thenThrow(new RuntimeException("no details")); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + verify(mockStrategy, never()).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Pool message — content verification + // ────────────────────────────────────────────────────────────────────────── + + @Test + void poolMessage_containsPoolNameClusterUuidAndSnapshotKeys() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("ontap-pool-1"), "should contain pool name"); + assertTrue(desc.contains("\"poolStatus\":\"Up\""), "should contain pool status"); + assertTrue(desc.contains("cluster-uuid-1"), "should contain cluster UUID"); + assertTrue(desc.contains("volumeSnapshotCount"), "should contain volumeSnapshotCount"); + assertTrue(desc.contains("vmSnapshotCount"), "should contain vmSnapshotCount"); + assertTrue(desc.contains("\"multiPrimaryStoragePoolVm\":false"), "desc=" + desc); + } + + @Test + void poolMessage_includesMaintenanceStatus() { + when(pool.getStatus()).thenReturn(StoragePoolStatus.Maintenance); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"poolStatus\":\"Maintenance\""), "desc=" + desc); + } + + @Test + void poolMessage_volumeSnapshots_zeroWhenNoVolumes() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"volumeSnapshotCount\":0"), "desc=" + desc); + assertTrue(desc.contains("\"vmSnapshotCount\":0"), "desc=" + desc); + } + + @Test + void poolMessage_multiPrimaryStoragePoolVm_trueWhenDaoReportsSpan() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + when(volumeDao.hasMultiPrimaryStoragePoolVm(1L)).thenReturn(true); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"multiPrimaryStoragePoolVm\":true"), "desc=" + desc); + } + + @Test + void poolMessage_volumeSnapshots_countExcludesDestroyed() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + // instanceId=null → no VM IDs → vmSnapshotDao never called + VolumeVO vol = mockVolume(10L, null, 10_737_418_240L); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.singletonList(vol)); + + // 2 active + 1 Destroyed → only 2 counted + SnapshotVO s1 = makeSnapshot(1L, 10L, Snapshot.State.BackedUp); + SnapshotVO s2 = makeSnapshot(2L, 10L, Snapshot.State.Creating); + SnapshotVO s3 = makeSnapshot(3L, 10L, Snapshot.State.Destroyed); + when(snapshotDao.searchByVolumes(anyList())).thenReturn(Arrays.asList(s1, s2, s3)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"volumeSnapshotCount\":2"), "Destroyed must be excluded; desc=" + desc); + verify(volumeDao, times(1)).findNonDestroyedVolumesByPoolId(eq(1L), isNull()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Pool message — VM-snapshot fields (vmSnapshotCount) + // ────────────────────────────────────────────────────────────────────────── + + @Test + void poolMessage_vmSnapshots_countsActiveSnapshotsAcrossDistinctVMs() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + VolumeVO vol1 = mockVolume(10L, 100L, 10_737_418_240L); // vm 100 + VolumeVO vol2 = mockVolume(20L, 200L, 10_737_418_240L); // vm 200 + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())) + .thenReturn(Arrays.asList(vol1, vol2)); + when(snapshotDao.searchByVolumes(anyList())).thenReturn(Collections.emptyList()); + + // 2 active VM snapshots; 1 is Expunging (should be excluded) + VMSnapshotVO vmSnap1 = makeVmSnapshot(VMSnapshot.State.Ready, null); + VMSnapshotVO vmSnap2 = makeVmSnapshot(VMSnapshot.State.Ready, null); + VMSnapshotVO vmSnapExp = makeVmSnapshot(VMSnapshot.State.Expunging, null); + when(vmSnapshotDao.searchByVms(anyList())).thenReturn(Arrays.asList(vmSnap1, vmSnap2, vmSnapExp)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":2"), "Expunging must be excluded; desc=" + desc); + } + + @Test + void poolMessage_vmSnapshots_removedSnapshotsExcluded() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + VolumeVO vol = mockVolume(10L, 100L, 1_073_741_824L); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.singletonList(vol)); + when(snapshotDao.searchByVolumes(anyList())).thenReturn(Collections.emptyList()); + + VMSnapshotVO active = makeVmSnapshot(VMSnapshot.State.Ready, null); + VMSnapshotVO deleted = makeVmSnapshot(VMSnapshot.State.Ready, new java.util.Date()); // removed + when(vmSnapshotDao.searchByVms(anyList())).thenReturn(Arrays.asList(active, deleted)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":1"), "removed snapshots must be excluded; desc=" + desc); + } + + @Test + void poolMessage_vmSnapshots_zeroWhenNoVmIdsOnPool() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + // instanceId=null → detached data disk → vmSnapshotDao must NOT be called + VolumeVO vol = mockVolume(10L, null, 1_073_741_824L); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.singletonList(vol)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":0"), "desc=" + desc); + verify(vmSnapshotDao, never()).searchByVms(anyList()); + } + + @Test + void poolMessage_vmSnapshots_countedOnlyOnRootDiskPool() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + // Data disk of a VM whose ROOT lives on another pool — must not count that VM snapshot. + VolumeVO dataDisk = mockVolume(20L, 100L, 1_073_741_824L, Volume.Type.DATADISK); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())) + .thenReturn(Collections.singletonList(dataDisk)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":0"), "desc=" + desc); + verify(vmSnapshotDao, never()).searchByVms(anyList()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Best-effort: DAO failures must never suppress the pool message + // ────────────────────────────────────────────────────────────────────────── + + @Test + void poolMessage_snapshotDaoThrows_poolMessageStillSent() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())) + .thenThrow(new RuntimeException("DB error")); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + // heartbeat + pool both sent even when DAO fails + verify(mockStrategy, times(2)).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Multi-pool: same cluster → single heartbeat + // ────────────────────────────────────────────────────────────────────────── + + @Test + void twoPoolsSameCluster_singleHeartbeat() { + StoragePoolVO pool2 = mock(StoragePoolVO.class); + when(pool2.getId()).thenReturn(2L); + when(pool2.getName()).thenReturn("ontap-pool-2"); + + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(storagePoolDetailsDao.listDetailsKeyPairs(2L)).thenReturn(new HashMap<>(poolDetails)); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(anyLong(), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + + Map clientsByStorageIp = new HashMap<>(); + asupManager.pushAsupForStoragePool(pool, clientsByStorageIp); + asupManager.pushAsupForStoragePool(pool2, clientsByStorageIp); + + u.verify(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any()), times(1)); + } + + verify(mockStrategy, times(1)).getClusterInfo(); + // 1 heartbeat + 2 pool messages = 3 total + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(3)).sendAsupMessage(cap.capture()); + + long heartbeats = cap.getAllValues().stream() + .filter(m -> OntapStorageConstants.ASUP_EVENT_ID_HEARTBEAT.equals(m.getEventId())) + .count(); + assertEquals(1, heartbeats, "exactly 1 heartbeat for two pools sharing a cluster"); + } + + @Test + void twoPoolsDifferentStorageIp_resolvesStrategyPerCluster() { + StoragePoolVO pool2 = mock(StoragePoolVO.class); + when(pool2.getId()).thenReturn(2L); + when(pool2.getName()).thenReturn("ontap-pool-2"); + + Map otherCluster = new HashMap<>(poolDetails); + otherCluster.put(OntapStorageConstants.STORAGE_IP, "192.168.1.20"); + + Cluster cluster2 = mock(Cluster.class); + when(cluster2.getUuid()).thenReturn("cluster-uuid-2"); + + StorageStrategy strategy2 = mock(StorageStrategy.class); + when(strategy2.getClusterInfo()).thenReturn(cluster2); + when(strategy2.getClusterVersion(cluster2)).thenReturn("9.16.1"); + + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(storagePoolDetailsDao.listDetailsKeyPairs(2L)).thenReturn(otherCluster); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(anyLong(), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(poolDetails)).thenReturn(mockStrategy); + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(otherCluster)).thenReturn(strategy2); + + Map clientsByStorageIp = new HashMap<>(); + asupManager.pushAsupForStoragePool(pool, clientsByStorageIp); + asupManager.pushAsupForStoragePool(pool2, clientsByStorageIp); + + u.verify(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any()), times(2)); + } + + verify(mockStrategy, times(1)).getClusterInfo(); + verify(strategy2, times(1)).getClusterInfo(); + verify(mockStrategy, times(2)).sendAsupMessage(any()); + verify(strategy2, times(2)).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Common EMS envelope fields + // ────────────────────────────────────────────────────────────────────────── + + @Test + void allMessages_haveCorrectEnvelopeFields() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.resolveStrategyFromPoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, new HashMap<>()); + } + + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + + for (EmsApplicationLog msg : cap.getAllValues()) { + assertEquals(OntapStorageConstants.ASUP_EVENT_SOURCE, msg.getEventSource()); + assertEquals(OntapStorageConstants.ASUP_CATEGORY, msg.getCategory()); + assertEquals(OntapStorageConstants.ASUP_SEVERITY, msg.getSeverity()); + assertFalse(msg.getAutosupportRequired(), "autosupport_required should be false"); + assertEquals(asupManager.getComputerName(), msg.getComputerName()); + assertEquals(asupManager.getCloudStackVersion(), msg.getAppVersion()); + } + } + + // ────────────────────────────────────────────────────────────────────────── + // Config defaults + // ────────────────────────────────────────────────────────────────────────── + + @Test + void asupIntervalHours_defaultIsProductionValue() { + assertEquals(4, OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS); + assertEquals(24, OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS); + assertEquals(String.valueOf(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS), + OntapConfigurationManager.AsupIntervalHours.defaultValue()); + } + + @Test + void asupIntervalHours_keyIsAutosupportInterval() { + assertEquals(OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY, + OntapConfigurationManager.AsupIntervalHours.key()); + assertEquals("ontap.autosupport.interval", OntapConfigurationManager.AsupIntervalHours.key()); + } + + @Test + void asupIntervalHours_descriptionIncludesAllowedRange() { + String description = OntapConfigurationManager.AsupIntervalHours.description(); + assertTrue(description.contains(String.valueOf(OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS))); + assertTrue(description.contains(String.valueOf(OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS))); + assertTrue(description.contains(String.valueOf(OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS))); + assertTrue(description.contains("1 week"), description); + } + + @Test + void validateAsupInterval_acceptsDisabledMinMaxAndDefault() { + OntapConfigurationManager.AsupIntervalHours.validateValue(String.valueOf(OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS)); + OntapConfigurationManager.AsupIntervalHours.validateValue(String.valueOf(OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS)); + OntapConfigurationManager.AsupIntervalHours.validateValue(String.valueOf(OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS)); + OntapConfigurationManager.AsupIntervalHours.validateValue(String.valueOf(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS)); + } + + @Test + void validateAsupInterval_rejectsOutOfRangeAndNonInteger() { + assertThrows(InvalidParameterValueException.class, () -> OntapConfigurationManager.AsupIntervalHours.validateValue("-1")); + assertThrows(InvalidParameterValueException.class, () -> OntapConfigurationManager.AsupIntervalHours.validateValue("1")); + assertThrows(InvalidParameterValueException.class, () -> OntapConfigurationManager.AsupIntervalHours.validateValue("169")); + assertThrows(InvalidParameterValueException.class, () -> OntapConfigurationManager.AsupIntervalHours.validateValue("abc")); + assertThrows(InvalidParameterValueException.class, () -> OntapConfigurationManager.AsupIntervalHours.validateValue("")); + } + + @Test + void getAsupIntervalHours_fallsBackOutsideRange() { + assertEquals(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(null)); + assertEquals(OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(OntapStorageConstants.ASUP_DISABLED_INTERVAL_HOURS)); + assertEquals(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(1)); + assertEquals(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(2)); + assertEquals(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(3)); + assertEquals(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(-1)); + assertEquals(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(169)); + assertEquals(OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(OntapStorageConstants.ASUP_MIN_INTERVAL_HOURS)); + assertEquals(OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS, + asupManager.getAsupIntervalHours(OntapStorageConstants.ASUP_MAX_INTERVAL_HOURS)); + assertEquals(25, asupManager.getAsupIntervalHours(25)); + } + + // ────────────────────────────────────────────────────────────────────────── + // OntapAsupTask / schedule + // ────────────────────────────────────────────────────────────────────────── + + @Test + void millisUntilNextPush_zeroWhenOverdue() { + asupManager.lastPushTime = Instant.EPOCH; // never pushed + assertEquals(0L, asupManager.millisUntilNextPush()); + } + + @Test + void millisUntilNextPush_remainderOfConfiguredInterval() { + asupManager.lastPushTime = Instant.now(); + long remaining = asupManager.millisUntilNextPush(); + long configured = Duration.ofHours(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_HOURS).toMillis(); + assertTrue(remaining > configured - 5000L && remaining <= configured, + "remaining=" + remaining); + } + + @Test + void asupTask_stampsLastPushTimeAndPushes() { + asupManager.lastPushTime = Instant.EPOCH; // never pushed + when(storagePoolDao.findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME)) + .thenReturn(Collections.emptyList()); + Instant before = Instant.now(); + OntapAsupManager.OntapAsupTask task = asupManager.new OntapAsupTask(); + task.run(); + verify(storagePoolDao).findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME); + assertFalse(asupManager.lastPushTime.equals(Instant.EPOCH)); + assertFalse(asupManager.lastPushTime.isBefore(before)); + } + + // ────────────────────────────────────────────────────────────────────────── + // Utility helpers + // ────────────────────────────────────────────────────────────────────────── + + @Test + void getCloudStackVersion_returnsManagementServiceVersion() { + assertEquals("4.23.0.0-SNAPSHOT", asupManager.getCloudStackVersion()); + } + + @Test + void getCloudStackVersion_blank_returnsUnknown() { + when(managementService.getVersion()).thenReturn(" "); + assertEquals(OntapStorageConstants.ASUP_UNKNOWN, asupManager.getCloudStackVersion()); + } + + @Test + void getManagementServerCount_returnsRegisteredHostCount() { + when(managementServerHostDao.listAll()).thenReturn(Arrays.asList( + mock(ManagementServerHostVO.class), mock(ManagementServerHostVO.class))); + assertEquals(2, asupManager.getManagementServerCount()); + } + + @Test + void getComputerName_returnsNonEmpty() { + String host = asupManager.getComputerName(); + assertNotNull(host); + assertFalse(host.isEmpty()); + } + + @Test + void getOperatingSystem_returnsNonEmpty() { + String os = asupManager.getOperatingSystem(); + assertNotNull(os); + assertFalse(os.isEmpty()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Helpers + // ────────────────────────────────────────────────────────────────────────── + + /** + * Captures and returns the event-id 1 (pool) message description. + * Expects exactly 2 messages to have been sent (heartbeat + pool). + */ + private String capturePoolMessage() { + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + EmsApplicationLog poolMsg = cap.getAllValues().get(1); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL, poolMsg.getEventId()); + String desc = poolMsg.getEventDescription(); + assertNotNull(desc); + return desc; + } + + /** + * Creates a mock VolumeVO with state=Ready so that CS_VOLUME_STATES filter includes it + * and getSize() is exercised (avoiding UnnecessaryStubbingException in strict mode). + */ + private VolumeVO mockVolume(long id, Long instanceId, long size) { + return mockVolume(id, instanceId, size, Volume.Type.ROOT); + } + + private VolumeVO mockVolume(long id, Long instanceId, long size, Volume.Type type) { + VolumeVO vol = mock(VolumeVO.class); + when(vol.getId()).thenReturn(id); + lenient().when(vol.getInstanceId()).thenReturn(instanceId); + when(vol.getSize()).thenReturn(size); + when(vol.getState()).thenReturn(Volume.State.Ready); + when(vol.getVolumeType()).thenReturn(type); + return vol; + } + + /** Creates a mock SnapshotVO with the given id, volumeId and state. */ + private SnapshotVO makeSnapshot(long id, long volumeId, Snapshot.State state) { + SnapshotVO snap = mock(SnapshotVO.class); + when(snap.getState()).thenReturn(state); + return snap; + } + + /** Creates a mock VMSnapshotVO with the given state and removed timestamp. */ + private VMSnapshotVO makeVmSnapshot(VMSnapshot.State state, java.util.Date removed) { + VMSnapshotVO vmSnap = mock(VMSnapshotVO.class); + when(vmSnap.getState()).thenReturn(state); + lenient().when(vmSnap.getRemoved()).thenReturn(removed); + return vmSnap; + } +} diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriverTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriverTest.java index 3c139e23cb88..b675618ae2a8 100644 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriverTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriverTest.java @@ -21,23 +21,45 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ResizeVolumePayload; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotDetailsDao; +import com.cloud.storage.dao.SnapshotDetailsVO; +import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.FileInfo; import org.apache.cloudstack.storage.feign.model.Igroup; import org.apache.cloudstack.storage.feign.model.Lun; +import org.apache.cloudstack.storage.feign.model.VolumeSpace; +import org.apache.cloudstack.storage.feign.model.Svm; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; +import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.service.UnifiedNASStrategy; import org.apache.cloudstack.storage.service.UnifiedSANStrategy; import org.apache.cloudstack.storage.service.model.AccessGroup; import org.apache.cloudstack.storage.service.model.CloudStackVolume; @@ -54,29 +76,49 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static com.cloud.agent.api.to.DataObjectType.SNAPSHOT; +import static com.cloud.agent.api.to.DataObjectType.TEMPLATE; import static com.cloud.agent.api.to.DataObjectType.VOLUME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class OntapPrimaryDatastoreDriverTest { + /** + * A real serial number taken from an ONTAP array, and the WWID NetApp derives from it as its + * OUI plus the serial's ASCII bytes in hex. The serial deliberately contains a '/', which is + * why volume paths carry the WWID: the raw serial would break the '/targetIQN/LUN' parsing that + * every hypervisor resource applies to the path. + */ + private static final String LUN_SERIAL_NUMBER = "x0M-8?/kEsJT"; + private static final String LUN_WWID = "600a098078304d2d383f2f6b45734a54"; + @Mock private StoragePoolDetailsDao storagePoolDetailsDao; @@ -89,12 +131,27 @@ class OntapPrimaryDatastoreDriverTest { @Mock private VolumeDetailsDao volumeDetailsDao; + @Mock + private SnapshotDetailsDao snapshotDetailsDao; + + @Mock + private SnapshotDao snapshotDao; + + @Mock + private VMTemplatePoolDao vmTemplatePoolDao; + + @Mock + private VMTemplateStoragePoolVO templatePoolRef; + @Mock private DataStore dataStore; @Mock private VolumeInfo volumeInfo; + @Mock + private TemplateInfo templateInfo; + @Mock private StoragePoolVO storagePool; @@ -107,6 +164,9 @@ class OntapPrimaryDatastoreDriverTest { @Mock private UnifiedSANStrategy sanStrategy; + @Mock + private UnifiedNASStrategy nasStrategy; + @Mock private AsyncCompletionCallback createCallback; @@ -123,6 +183,11 @@ void setUp() { storagePoolDetails = new HashMap<>(); storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.ISCSI.name()); storagePoolDetails.put(OntapStorageConstants.SVM_NAME, "svm1"); + storagePoolDetails.put(OntapStorageConstants.IS_AFF, "true"); + lenient().when(volumeInfo.getName()).thenReturn("test-volume"); + lenient().when(volumeInfo.getSize()).thenReturn(1073741824L); + lenient().when(storagePool.getName()).thenReturn("vol1"); + lenient().when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); } @Test @@ -134,6 +199,10 @@ void testGetCapabilities() { // so StorageSystemSnapshotStrategy handles snapshot backup to secondary storage assertEquals(Boolean.TRUE.toString(), capabilities.get("STORAGE_SYSTEM_SNAPSHOT")); assertEquals(Boolean.TRUE.toString(), capabilities.get("CAN_CREATE_VOLUME_FROM_SNAPSHOT")); + assertEquals(Boolean.TRUE.toString(), capabilities.get("CAN_REVERT_VOLUME_TO_SNAPSHOT")); + assertEquals(Boolean.TRUE.toString(), capabilities.get("CAN_CREATE_VOLUME_FROM_VOLUME")); + assertEquals(Boolean.TRUE.toString(), capabilities.get("CAN_CREATE_TEMPLATE_FROM_SNAPSHOT")); + assertEquals(Boolean.FALSE.toString(), capabilities.get("CAN_DIRECT_ATTACH_SNAPSHOT")); } @Test @@ -164,8 +233,11 @@ void testCreateAsync_VolumeWithISCSI_Success() { when(volumeInfo.getName()).thenReturn("test-volume"); when(storagePoolDao.findById(1L)).thenReturn(storagePool); - when(storagePool.getId()).thenReturn(1L); - when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + lenient().when(storagePool.getId()).thenReturn(1L); + lenient().when(storagePool.getName()).thenReturn("vol1"); + + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); when(volumeDao.findById(100L)).thenReturn(volumeVO); @@ -174,18 +246,12 @@ void testCreateAsync_VolumeWithISCSI_Success() { Lun mockLun = new Lun(); mockLun.setName("/vol/vol1/lun1"); mockLun.setUuid("lun-uuid-123"); - // Create request volume (returned by Utility.createCloudStackVolumeRequestByProtocol) - CloudStackVolume requestVolume = new CloudStackVolume(); - requestVolume.setLun(mockLun); - // Create response volume (returned by sanStrategy.createCloudStackVolume) CloudStackVolume responseVolume = new CloudStackVolume(); responseVolume.setLun(mockLun); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(sanStrategy); - utilityMock.when(() -> OntapStorageUtils.createCloudStackVolumeRequestByProtocol( - any(), any(), any())).thenReturn(requestVolume); when(sanStrategy.createCloudStackVolume(any())).thenReturn(responseVolume); // Execute @@ -201,6 +267,7 @@ void testCreateAsync_VolumeWithISCSI_Success() { verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_UUID), eq("lun-uuid-123"), eq(false)); verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_NAME), eq("/vol/vol1/lun1"), eq(false)); + verify(volumeVO).setFormat(Storage.ImageFormat.RAW); verify(volumeDao).update(eq(100L), any(VolumeVO.class)); } } @@ -219,19 +286,18 @@ void testCreateAsync_VolumeWithNFS_Success() { when(storagePoolDao.findById(1L)).thenReturn(storagePool); when(storagePool.getId()).thenReturn(1L); when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); when(volumeDao.findById(100L)).thenReturn(volumeVO); when(volumeVO.getId()).thenReturn(100L); CloudStackVolume mockCloudStackVolume = new CloudStackVolume(); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) - .thenReturn(sanStrategy); - utilityMock.when(() -> OntapStorageUtils.createCloudStackVolumeRequestByProtocol( - any(), any(), any())).thenReturn(mockCloudStackVolume); + .thenReturn(nasStrategy); - when(sanStrategy.createCloudStackVolume(any())).thenReturn(mockCloudStackVolume); + when(nasStrategy.createCloudStackVolume(any())).thenReturn(mockCloudStackVolume); // Execute driver.createAsync(dataStore, volumeInfo, createCallback); @@ -243,10 +309,65 @@ void testCreateAsync_VolumeWithNFS_Success() { CreateCmdResult result = resultCaptor.getValue(); assertNotNull(result); assertTrue(result.isSuccess()); + verify(volumeVO).setFormat(Storage.ImageFormat.QCOW2); verify(volumeDao).update(eq(100L), any(VolumeVO.class)); } } + @Test + void testCreateAsync_UnsupportedHypervisor_FailsWithError() { + // Use NFS so LUN OS-type resolution does not fail earlier in getOSTypeFromHypervisor; + // the failure under test is image-format resolution for non-KVM hypervisors. + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.VMware); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + + CloudStackVolume mockCloudStackVolume = new CloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + when(nasStrategy.createCloudStackVolume(any())).thenReturn(mockCloudStackVolume); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Unsupported hypervisor [VMware]")); + } + } + + @Test + void testCreateAsync_KvmUnsupportedProtocol_FailsWithError() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, "FC"); + + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getName()).thenReturn("test-volume"); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Unsupported protocol [FC]")); + } + @Test void testDeleteAsync_NullStore_ThrowsException() { ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); @@ -274,8 +395,9 @@ void testDeleteAsync_ISCSIVolume_Success() { when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_NAME)).thenReturn(lunNameDetail); when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)).thenReturn(lunUuidDetail); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(null); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) .thenReturn(sanStrategy); @@ -292,6 +414,8 @@ void testDeleteAsync_ISCSIVolume_Success() { assertNotNull(result); assertTrue(result.isSuccess()); verify(sanStrategy).deleteCloudStackVolume(any(CloudStackVolume.class)); + verify(volumeDao, never()).remove(anyLong()); + verify(volumeDetailsDao, never()).removeDetails(anyLong()); } } @@ -318,6 +442,93 @@ void testDeleteAsync_NFSVolume_Success() { // NFS deletion doesn't fail, handled by hypervisor } + @Test + void testDeleteAsync_Template_DeletesCacheLun() { + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getLocalDownloadPath()).thenReturn("template-lun-uuid"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + + driver.deleteAsync(dataStore, templateInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).deleteCloudStackVolume(requestCaptor.capture()); + assertEquals("template-lun-uuid", requestCaptor.getValue().getLun().getUuid()); + } + } + + + @Test + void testDeleteAsync_Template_ResolvesLunByNameWhenUuidMissing() { + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getLocalDownloadPath()).thenReturn(null); + + Lun templateLun = new Lun(); + templateLun.setName("/vol/vol1/cs_tmpl_50"); + templateLun.setUuid("resolved-lun-uuid"); + CloudStackVolume existing = new CloudStackVolume(); + existing.setLun(templateLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + when(sanStrategy.getCloudStackVolume(argThat(map -> + map != null && "/vol/vol1/cs_tmpl_50".equals(map.get("name"))))) + .thenReturn(existing); + doNothing().when(sanStrategy).deleteCloudStackVolume(any()); + + driver.deleteAsync(dataStore, templateInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor deleteCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).deleteCloudStackVolume(deleteCaptor.capture()); + assertEquals("resolved-lun-uuid", deleteCaptor.getValue().getLun().getUuid()); + assertEquals("/vol/vol1/cs_tmpl_50", deleteCaptor.getValue().getLun().getName()); + } + } + + @Test + void testDeleteAsync_Template_NoCachedLun_SucceedsWithoutCallingOntap() { + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(null); + + driver.deleteAsync(dataStore, templateInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).deleteCloudStackVolume(any()); + } + @Test void testGrantAccess_NullParameters_ThrowsException() { assertThrows(CloudRuntimeException.class, @@ -359,7 +570,7 @@ void testGrantAccess_ClusterScope_Success() { existingIgroup.setName("igroup1"); existingAccessGroup.setIgroup(existingIgroup); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) .thenReturn(sanStrategy); utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())) @@ -367,6 +578,7 @@ void testGrantAccess_ClusterScope_Success() { when(sanStrategy.getAccessGroup(any())).thenReturn(existingAccessGroup); when(sanStrategy.ensureLunMapped(anyString(), anyString(), anyString())).thenReturn("0"); + when(sanStrategy.getLunSerialNumber(anyString(), anyString())).thenReturn(LUN_SERIAL_NUMBER); // Execute boolean result = driver.grantAccess(volumeInfo, host, dataStore); @@ -377,6 +589,7 @@ void testGrantAccess_ClusterScope_Success() { verify(sanStrategy).getAccessGroup(any()); verify(sanStrategy).ensureLunMapped(anyString(), anyString(), anyString()); verify(sanStrategy, never()).validateInitiatorInAccessGroup(anyString(), anyString(), any(Igroup.class)); + verify(volumeInfo, never()).getVolume(); } } @@ -410,7 +623,7 @@ void testGrantAccess_IgroupNotFound_CreatesNewIgroup() { createdIgroup.setName("igroup1"); createdAccessGroup.setIgroup(createdIgroup); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) .thenReturn(sanStrategy); utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())) @@ -419,6 +632,7 @@ void testGrantAccess_IgroupNotFound_CreatesNewIgroup() { when(sanStrategy.getAccessGroup(any())).thenReturn(null); when(sanStrategy.createAccessGroup(any())).thenReturn(createdAccessGroup); when(sanStrategy.ensureLunMapped(anyString(), anyString(), anyString())).thenReturn("0"); + when(sanStrategy.getLunSerialNumber(anyString(), anyString())).thenReturn(LUN_SERIAL_NUMBER); // Execute boolean result = driver.grantAccess(volumeInfo, hostVO, dataStore); @@ -435,12 +649,12 @@ void testGrantAccess_IgroupNotFound_CreatesNewIgroup() { @Test void testRevokeAccess_NFSVolume_SkipsRevoke() { // Setup - NFS volumes have no LUN mapping, so revokeAccess is a no-op + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); when(dataStore.getId()).thenReturn(1L); when(volumeInfo.getType()).thenReturn(VOLUME); when(volumeInfo.getId()).thenReturn(100L); when(volumeDao.findById(100L)).thenReturn(volumeVO); - when(volumeVO.getId()).thenReturn(100L); when(volumeVO.getName()).thenReturn("test-volume"); when(storagePoolDao.findById(1L)).thenReturn(storagePool); @@ -449,7 +663,7 @@ void testRevokeAccess_NFSVolume_SkipsRevoke() { when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); when(host.getName()).thenReturn("host1"); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) .thenReturn(sanStrategy); @@ -496,7 +710,7 @@ void testRevokeAccess_ISCSIVolume_Success() { AccessGroup mockAccessGroup = new AccessGroup(); mockAccessGroup.setIgroup(mockIgroup); - try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) .thenReturn(sanStrategy); utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())) @@ -539,32 +753,3009 @@ void testRevokeAccess_ISCSIVolume_Success() { } @Test - void testCanHostAccessStoragePool_ReturnsTrue() { - assertTrue(driver.canHostAccessStoragePool(host, storagePool)); + void testGetDataObjectSizeIncludingHypervisorSnapshotReserve_NoReserveAdded() { + when(templateInfo.getSize()).thenReturn(5368709120L); + + assertEquals(5368709120L, driver.getDataObjectSizeIncludingHypervisorSnapshotReserve(templateInfo, storagePool)); } @Test - void testIsVmInfoNeeded_ReturnsTrue() { - assertTrue(driver.isVmInfoNeeded()); + void testGetBytesRequiredForTemplate_AlreadyCached_ReturnsZero() { + when(storagePool.getId()).thenReturn(1L); + when(templateInfo.getId()).thenReturn(50L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); + when(templatePoolRef.getLocalDownloadPath()).thenReturn("template-lun-uuid"); + + assertEquals(0L, driver.getBytesRequiredForTemplate(templateInfo, storagePool)); } @Test - void testIsStorageSupportHA_ReturnsTrue() { - assertTrue(driver.isStorageSupportHA(Storage.StoragePoolType.NetworkFilesystem)); + void testGetBytesRequiredForTemplate_NotCached_ReturnsVirtualSize() { + when(storagePool.getId()).thenReturn(1L); + when(templateInfo.getId()).thenReturn(50L); + when(templateInfo.getSize()).thenReturn(5368709120L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(null); + + assertEquals(5368709120L, driver.getBytesRequiredForTemplate(templateInfo, storagePool)); } @Test - void testGetChapInfo_ReturnsNull() { - assertNull(driver.getChapInfo(volumeInfo)); + void testGetBytesRequiredForTemplate_SpoolRefNotReady_ReturnsVirtualSize() { + when(storagePool.getId()).thenReturn(1L); + when(templateInfo.getId()).thenReturn(50L); + when(templateInfo.getSize()).thenReturn(5368709120L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Allocated); + + assertEquals(5368709120L, driver.getBytesRequiredForTemplate(templateInfo, storagePool)); } @Test - void testCanProvideStorageStats_ReturnsFalse() { - assertFalse(driver.canProvideStorageStats()); + void testGetBytesRequiredForTemplate_ReadyWithoutBackendIdentity_ReturnsVirtualSize() { + when(storagePool.getId()).thenReturn(1L); + when(templateInfo.getId()).thenReturn(50L); + when(templateInfo.getSize()).thenReturn(5368709120L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); + when(templatePoolRef.getLocalDownloadPath()).thenReturn(null); + + assertEquals(5368709120L, driver.getBytesRequiredForTemplate(templateInfo, storagePool)); } @Test - void testCanProvideVolumeStats_ReturnsFalse() { - assertFalse(driver.canProvideVolumeStats()); + void testGetBytesRequiredForTemplate_NfsCached_ReturnsZero() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + when(storagePool.getId()).thenReturn(1L); + when(templateInfo.getId()).thenReturn(50L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); + when(templatePoolRef.getInstallPath()).thenReturn("/mnt/pool/template-uuid"); + + assertEquals(0L, driver.getBytesRequiredForTemplate(templateInfo, storagePool)); + } + + @Test + void testCreateAsync_TemplateWithISCSI_CreatesLunAndRecordsCloneSource() { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + when(templateInfo.getSize()).thenReturn(5368709120L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getId()).thenReturn(7L); + + Lun templateLun = new Lun(); + templateLun.setName("/vol/vol1/cs_tmpl_50"); + templateLun.setUuid("template-lun-uuid"); + CloudStackVolume created = new CloudStackVolume(); + created.setLun(templateLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.createTemplateCache(any(), any(), any(), anyLong())).thenReturn(created); + + driver.createAsync(dataStore, templateInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + // local_download_path carries the clone source; install_path is left for grantAccess + verify(templatePoolRef).setLocalDownloadPath("template-lun-uuid"); + verify(templatePoolRef).setTemplateSize(5368709120L); + verify(templatePoolRef, never()).setInstallPath(any()); + verify(vmTemplatePoolDao).update(eq(7L), any(VMTemplateStoragePoolVO.class)); + } + } + + + @Test + void testCreateAsync_TemplateWithISCSI_RollsBackLunWhenDbUpdateFails() { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + when(templateInfo.getSize()).thenReturn(5368709120L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getId()).thenReturn(7L); + doThrow(new RuntimeException("db update failed")).when(vmTemplatePoolDao) + .update(eq(7L), any(VMTemplateStoragePoolVO.class)); + + Lun templateLun = new Lun(); + templateLun.setName("/vol/vol1/cs_tmpl_50"); + templateLun.setUuid("template-lun-uuid"); + CloudStackVolume created = new CloudStackVolume(); + created.setLun(templateLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.createTemplateCache(any(), any(), any(), anyLong())).thenReturn(created); + doNothing().when(sanStrategy).deleteCloudStackVolume(any()); + + driver.createAsync(dataStore, templateInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor deleteCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).deleteCloudStackVolume(deleteCaptor.capture()); + assertEquals("template-lun-uuid", deleteCaptor.getValue().getLun().getUuid()); + assertEquals("/vol/vol1/cs_tmpl_50", deleteCaptor.getValue().getLun().getName()); + } + } + + @Test + void testCreateAsync_TemplateWithISCSI_UnknownSize_Fails() { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getSize()).thenReturn(0L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.createTemplateCache(any(), any(), any(), anyLong())) + .thenThrow(new CloudRuntimeException("Unknown virtual size for template [50]")); + + driver.createAsync(dataStore, templateInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + } + } + + @Test + void testCreateAsync_TemplateWithNFS_IsMetadataOnly() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getUuid()).thenReturn("template-uuid"); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + when(nasStrategy.createTemplateCache(any(), any(), any(), anyLong())).thenReturn(null); + + driver.createAsync(dataStore, templateInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(vmTemplatePoolDao, never()).update(any(Long.class), any(VMTemplateStoragePoolVO.class)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplate_ClonesWithoutGrowing() { + stubVolumeCloneFromTemplate(5368709120L, 5368709120L); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertEquals("/vol/vol1/cs_tmpl_50", requestCaptor.getValue().getLun().getClone().getSource().getName()); + assertEquals("template-lun-uuid", requestCaptor.getValue().getLun().getClone().getSource().getUuid()); + assertSame(volumeInfo, requestCaptor.getValue().getVolumeInfo()); + verify(sanStrategy, never()).createCloudStackVolume(any()); + verify(sanStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_UUID), eq("cloned-lun-uuid"), eq(false)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplate_GrowsWhenOfferingIsLarger() { + stubVolumeCloneFromTemplate(5368709120L, 21474836480L); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(sanStrategy).resizeCloudStackVolume(eq(cloned), eq(21474836480L)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_IscsiSuccessWithoutGrow() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + Lun lunRequest = requestCaptor.getValue().getLun(); + assertEquals("/vol/vol1/.snapshot/snap_cs200/source_lun", lunRequest.getClone().getSource().getName()); + assertNull(lunRequest.getClone().getSource().getUuid()); + assertEquals("/vol/vol1/test_volume", lunRequest.getName()); + assertEquals("svm1", lunRequest.getSvm().getName()); + assertSame(volumeInfo, requestCaptor.getValue().getVolumeInfo()); + verify(sanStrategy, never()).createCloudStackVolume(any()); + verify(sanStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_UUID), eq("snap-cloned-lun-uuid"), eq(false)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_IscsiWithIops_SetsQosOnLunCreate() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + lenient().when(storagePool.getCapacityIops()).thenReturn(null); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + VolumeQosPolicy qosPolicy = qosPolicy("qos-snap-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloned, qosPolicy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertTrue(isQosPolicyReference(requestCaptor.getValue().getLun().getQosPolicy(), + "qos-snap-uuid", "cs_100_to_200_iops_svm1")); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + eq("qos-snap-uuid"), eq(false)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_NfsWithIops_AttachesQosAfterClone() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flex-uuid"); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(volumeInfo.getUuid()).thenReturn("new-volume-uuid"); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + lenient().when(storagePool.getCapacityIops()).thenReturn(null); + + CloudStackVolume cloned = new CloudStackVolume(); + VolumeQosPolicy qosPolicy = lookedUpQosPolicy("qos-nfs-snap-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, nasStrategy, cloned, qosPolicy); + when(nasStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + when(nasStrategy.updateCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(nasStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + ArgumentCaptor cloneCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(nasStrategy).cloneCloudStackVolume(cloneCaptor.capture()); + assertNull(cloneCaptor.getValue().getFile().getQosPolicy()); + verify(nasStrategy).updateCloudStackVolume(argThat(request -> + "1".equals(request.getDatastoreId()) + && "flex-uuid".equals(request.getFlexVolumeUuid()) + && request.getVolumeInfo() == volumeInfo + && request.getFile() != null + && isQosPolicyReference(request.getFile().getQosPolicy(), + "qos-nfs-snap-uuid", "cs_100_to_200_iops_svm1"))); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + eq("qos-nfs-snap-uuid"), eq(false)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_NoIops_DoesNotCreatePolicy() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(null); + when(volumeInfo.getMaxIops()).thenReturn(null); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(sanStrategy, never()).createVolumeQosPolicy(nullable(String.class), nullable(Long.class), nullable(Long.class)); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertNull(requestCaptor.getValue().getLun().getQosPolicy()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_CloneFails_DeletesNewPolicy() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + lenient().when(storagePool.getCapacityIops()).thenReturn(null); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-snap-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, new CloudStackVolume(), qosPolicy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenThrow(new CloudRuntimeException("clone failed")); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).deleteVolumeQosPolicy("qos-snap-uuid"); + verify(volumeDetailsDao, never()).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + anyString(), eq(false)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_GrowsWhenOfferingIsLarger() { + stubVolumeCloneFromSnapshot(5368709120L, 21474836480L, ProtocolType.ISCSI.name()); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(sanStrategy).resizeCloudStackVolume(eq(cloned), eq(21474836480L)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_ResizeFails_DeletesOrphanedClone() { + stubVolumeCloneFromSnapshot(5368709120L, 21474836480L, ProtocolType.ISCSI.name()); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + doThrow(new CloudRuntimeException("resize failed")) + .when(sanStrategy).resizeCloudStackVolume(eq(cloned), eq(21474836480L)); + doNothing().when(sanStrategy).deleteCloudStackVolume(eq(cloned)); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).deleteCloudStackVolume(eq(cloned)); + verify(volumeDetailsDao, never()).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_UUID), anyString(), anyBoolean()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_NfsSuccess() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(volumeInfo.getUuid()).thenReturn("new-volume-uuid"); + + CloudStackVolume cloned = new CloudStackVolume(); + FileInfo file = new FileInfo(); + file.setPath("new-volume-uuid"); + cloned.setFile(file); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + when(nasStrategy.cloneCloudStackVolume(any())) + .thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(nasStrategy).cloneCloudStackVolume(requestCaptor.capture()); + CloudStackVolume request = requestCaptor.getValue(); + assertEquals("./.snapshot/snap_cs200/source-file-uuid", request.getFile().getPath()); + assertEquals("new-volume-uuid", request.getDestinationPath()); + assertEquals("1", request.getDatastoreId()); + verify(nasStrategy, never()).createCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_PoolMismatch_Fails() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.PRIMARY_POOL_ID)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.PRIMARY_POOL_ID, "999", false)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).cloneCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_MissingDetail_Fails() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.ONTAP_SNAP_NAME)).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).cloneCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_ProtocolMismatch_Fails() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + // Pool is iSCSI (default stub details) but snapshot was taken on NFS. + when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.PROTOCOL)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name(), false)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).cloneCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_NullStrategyResult_Fails() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenReturn(null); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_StrategyThrows_Fails() { + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenThrow(new CloudRuntimeException("ONTAP clone failed")); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_PrefersSnapshotOverTemplate() { + // Corner: when both details are present, the snapshot clone wins over the template clone. + stubVolumeCloneFromSnapshot(5368709120L, 5368709120L, ProtocolType.ISCSI.name()); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.CLONE_OF_TEMPLATE)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.CLONE_OF_TEMPLATE, "50", false)); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertEquals("/vol/vol1/.snapshot/snap_cs200/source_lun", + requestCaptor.getValue().getLun().getClone().getSource().getName()); + verify(vmTemplatePoolDao, never()).findByPoolTemplate(anyLong(), anyLong(), any()); + verify(sanStrategy, never()).createCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_NfsGrowsWhenOfferingIsLarger() { + stubVolumeCloneFromSnapshot(5368709120L, 21474836480L, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(volumeInfo.getUuid()).thenReturn("new-volume-uuid"); + + CloudStackVolume cloned = new CloudStackVolume(); + FileInfo file = new FileInfo(); + file.setPath("new-volume-uuid"); + cloned.setFile(file); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + when(nasStrategy.cloneCloudStackVolume(any())) + .thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(nasStrategy).resizeCloudStackVolume(eq(cloned), eq(21474836480L)); + verify(sanStrategy, never()).cloneCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromSnapshot_SkipsGrowWhenSnapshotSizeUnknown() { + stubVolumeCloneFromSnapshot(0L, 21474836480L, ProtocolType.ISCSI.name()); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("snap-cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(sanStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + } + } + + /** + * Sets up a volume create that the orchestrator has marked as a clone of a CloudStack snapshot. + */ + private void stubVolumeCloneFromSnapshot(long snapshotSize, long volumeSize, String protocol) { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + lenient().when(volumeInfo.getSize()).thenReturn(volumeSize); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + // getId is only used after snapshot_details validation succeeds; lenient for early-fail tests. + lenient().when(storagePool.getId()).thenReturn(1L); + lenient().when(storagePool.getName()).thenReturn("vol1"); + lenient().when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + lenient().when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + when(volumeDao.findById(100L)).thenReturn(volumeVO); + lenient().when(volumeVO.getId()).thenReturn(100L); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.CLONE_OF_SNAPSHOT)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.CLONE_OF_SNAPSHOT, "200", false)); + + String volumePath = ProtocolType.NFS3.name().equalsIgnoreCase(protocol) + ? "source-file-uuid" + : "/vol/vol1/source_lun"; + if (!ProtocolType.NFS3.name().equalsIgnoreCase(protocol)) { + storagePoolDetails.put(OntapStorageConstants.VOLUME_NAME, "vol1"); + } + // Lenient so early-fail tests can override/null individual keys without STRICT_STUBS noise. + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.ONTAP_SNAP_NAME)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.ONTAP_SNAP_NAME, "snap_cs200", false)); + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.VOLUME_PATH)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.VOLUME_PATH, volumePath, false)); + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.PRIMARY_POOL_ID)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.PRIMARY_POOL_ID, "1", false)); + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.PROTOCOL)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.PROTOCOL, protocol, false)); + + SnapshotVO snapshotVO = mock(SnapshotVO.class); + lenient().when(snapshotDao.findById(200L)).thenReturn(snapshotVO); + lenient().when(snapshotVO.getSize()).thenReturn(snapshotSize); + } + + /** + * Sets up a volume create that the orchestrator has marked as a clone of a cached template. + */ + private void stubVolumeCloneFromTemplate(long templateSize, long volumeSize) { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + lenient().when(volumeInfo.getSize()).thenReturn(volumeSize); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + lenient().when(storagePool.getName()).thenReturn("vol1"); + lenient().when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + lenient().when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); + lenient().when(storagePool.getCapacityIops()).thenReturn(null); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + when(volumeDao.findById(100L)).thenReturn(volumeVO); + lenient().when(volumeVO.getId()).thenReturn(100L); + // createAsync checks cloneOfSnapshot before cloneOfTemplate; under STRICT_STUBS an + // unstubbed alternate key on the same method is treated as an argument mismatch. + lenient().when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.CLONE_OF_SNAPSHOT)).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.CLONE_OF_TEMPLATE)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.CLONE_OF_TEMPLATE, "50", false)); + + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + lenient().when(templatePoolRef.getLocalDownloadPath()).thenReturn("template-lun-uuid"); + lenient().when(templatePoolRef.getTemplateSize()).thenReturn(templateSize); + } + + @Test + void testGrantAccess_Template_WritesInstallPathAndRefreshesStoreTarget() { + PrimaryDataStore primaryDataStore = mock(PrimaryDataStore.class); + Map dataStoreDetails = new HashMap<>(); + dataStoreDetails.put(PrimaryDataStore.MANAGED_STORE_TARGET, "stale-value"); + + when(primaryDataStore.getId()).thenReturn(1L); + when(primaryDataStore.getDetails()).thenReturn(dataStoreDetails); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePool.getPath()).thenReturn("iqn.1992-08.com.netapp:sn.123456"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getId()).thenReturn(7L); + + when(host.getName()).thenReturn("host1"); + when(host.getUuid()).thenReturn("host-uuid-1"); + + AccessGroup existingAccessGroup = new AccessGroup(); + Igroup existingIgroup = new Igroup(); + existingIgroup.setName("igroup1"); + existingAccessGroup.setIgroup(existingIgroup); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())).thenReturn("igroup1"); + + when(sanStrategy.getAccessGroup(any())).thenReturn(existingAccessGroup); + when(sanStrategy.ensureLunMapped(eq("svm1"), eq("/vol/vol1/cs_tmpl_50"), eq("igroup1"))).thenReturn("3"); + when(sanStrategy.getLunSerialNumber(eq("svm1"), eq("/vol/vol1/cs_tmpl_50"))).thenReturn(LUN_SERIAL_NUMBER); + + assertTrue(driver.grantAccess(templateInfo, host, primaryDataStore)); + + String expectedPath = "/iqn.1992-08.com.netapp:sn.123456/" + LUN_WWID; + verify(templatePoolRef).setInstallPath(expectedPath); + verify(vmTemplatePoolDao).update(eq(7L), any(VMTemplateStoragePoolVO.class)); + + ArgumentCaptor> detailsCaptor = ArgumentCaptor.forClass(Map.class); + verify(primaryDataStore).setDetails(detailsCaptor.capture()); + assertEquals(expectedPath, detailsCaptor.getValue().get(PrimaryDataStore.MANAGED_STORE_TARGET)); + } + } + + + @Test + void testGrantAccess_Template_UnmapsWhenInstallPathUpdateFails() { + PrimaryDataStore primaryDataStore = mock(PrimaryDataStore.class); + + when(primaryDataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePool.getPath()).thenReturn("iqn.1992-08.com.netapp:sn.123456"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getId()).thenReturn(7L); + doThrow(new RuntimeException("install path update failed")).when(vmTemplatePoolDao) + .update(eq(7L), any(VMTemplateStoragePoolVO.class)); + + when(host.getName()).thenReturn("host1"); + when(host.getUuid()).thenReturn("host-uuid-1"); + when(host.getStorageUrl()).thenReturn("iqn.1993-08.org.debian:01:host1"); + + AccessGroup existingAccessGroup = new AccessGroup(); + Igroup existingIgroup = mock(Igroup.class); + when(existingIgroup.getName()).thenReturn("igroup1"); + when(existingIgroup.getUuid()).thenReturn("igroup-uuid-123"); + existingAccessGroup.setIgroup(existingIgroup); + + Lun templateLun = new Lun(); + templateLun.setName("/vol/vol1/cs_tmpl_50"); + templateLun.setUuid("template-lun-uuid"); + CloudStackVolume cachedTemplate = new CloudStackVolume(); + cachedTemplate.setLun(templateLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())).thenReturn("igroup1"); + + when(sanStrategy.getAccessGroup(any())).thenReturn(existingAccessGroup); + when(sanStrategy.ensureLunMapped(eq("svm1"), eq("/vol/vol1/cs_tmpl_50"), eq("igroup1"))).thenReturn("3"); + when(sanStrategy.getLunSerialNumber(eq("svm1"), eq("/vol/vol1/cs_tmpl_50"))).thenReturn(LUN_SERIAL_NUMBER); + when(sanStrategy.getCloudStackVolume(argThat(map -> + map != null && "/vol/vol1/cs_tmpl_50".equals(map.get("name"))))) + .thenReturn(cachedTemplate); + when(sanStrategy.validateInitiatorInAccessGroup(anyString(), anyString(), any(Igroup.class))).thenReturn(true); + doNothing().when(sanStrategy).disableLogicalAccess(any()); + + assertThrows(CloudRuntimeException.class, + () -> driver.grantAccess(templateInfo, host, primaryDataStore)); + + verify(sanStrategy).disableLogicalAccess(argThat(map -> + map != null && "template-lun-uuid".equals(map.get("lun.uuid")) + && "igroup-uuid-123".equals(map.get("igroup.uuid")))); + } + } + + @Test + void testGrantAccess_TemplateOnNFS_SkipsMapping() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getUuid()).thenReturn("template-uuid"); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + assertTrue(driver.grantAccess(templateInfo, host, dataStore)); + verify(vmTemplatePoolDao, never()).update(any(Long.class), any(VMTemplateStoragePoolVO.class)); + } + + @Test + void testCreateAsync_VolumeClonedFromTemplateNFS_ClonesFile() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + stubVolumeCloneFromTemplate(5368709120L, 5368709120L); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(volumeInfo.getUuid()).thenReturn("volume-uuid"); + when(templatePoolRef.getInstallPath()).thenReturn("template-uuid"); + + CloudStackVolume cloned = new CloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + when(nasStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(nasStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertEquals("template-uuid", requestCaptor.getValue().getFile().getPath()); + assertEquals("volume-uuid", requestCaptor.getValue().getDestinationPath()); + verify(nasStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + verify(nasStrategy, never()).updateCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplate_RootWithIops_SetsQosOnLunCreate() { + stubVolumeCloneFromTemplate(5368709120L, 5368709120L); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/test_volume"); + clonedLun.setUuid("cloned-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + VolumeQosPolicy qosPolicy = qosPolicy("qos-root-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloned, qosPolicy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + verify(sanStrategy).cloneCloudStackVolume(argThat(request -> + request.getLun() != null && request.getLun().getQosPolicy() != null + && "qos-root-uuid".equals(request.getLun().getQosPolicy().getUuid()))); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + eq("qos-root-uuid"), eq(false)); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplateNFS_RootWithIops_AttachesQosToFile() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flex-uuid"); + stubVolumeCloneFromTemplate(5368709120L, 5368709120L); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(volumeInfo.getUuid()).thenReturn("volume-uuid"); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + when(templatePoolRef.getInstallPath()).thenReturn("template-uuid"); + + CloudStackVolume cloned = new CloudStackVolume(); + VolumeQosPolicy qosPolicy = lookedUpQosPolicy("qos-nfs-root-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, nasStrategy, cloned, qosPolicy); + when(nasStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + when(nasStrategy.updateCloudStackVolume(any())).thenReturn(cloned); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(nasStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + verify(nasStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + verify(nasStrategy).updateCloudStackVolume(argThat(request -> + "1".equals(request.getDatastoreId()) + && "flex-uuid".equals(request.getFlexVolumeUuid()) + && request.getVolumeInfo() == volumeInfo + && request.getFile() != null + && isQosPolicyReference(request.getFile().getQosPolicy(), + "qos-nfs-root-uuid", "cs_100_to_200_iops_svm1"))); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + eq("qos-nfs-root-uuid"), eq(false)); + } + } + + @Test + void testDeleteAsync_Template_NFS_DeletesCachedFile() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid"); + + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getInstallPath()).thenReturn("template-uuid"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(nasStrategy); + + driver.deleteAsync(dataStore, templateInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(nasStrategy).deleteFileByPath("flexvol-uuid", "template-uuid"); + } + } + + @Test + void testDeleteAsync_Template_NFS_FailsWhenFlexVolUuidMissing() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.remove(OntapStorageConstants.VOLUME_UUID); + + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getInstallPath()).thenReturn("template-uuid"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(nasStrategy); + + driver.deleteAsync(dataStore, templateInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("volumeUUID")); + verify(nasStrategy, never()).deleteFileByPath(any(), any()); + } + } + + @Test + void testRevokeAccess_Template_UnmapsCacheLun() { + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + when(host.getStorageUrl()).thenReturn("iqn.1993-08.org.debian:01:host1"); + when(host.getName()).thenReturn("host1"); + when(host.getUuid()).thenReturn("host-uuid-1"); + + Lun templateLun = new Lun(); + templateLun.setName("/vol/vol1/cs_tmpl_50"); + templateLun.setUuid("template-lun-uuid"); + CloudStackVolume cachedTemplate = new CloudStackVolume(); + cachedTemplate.setLun(templateLun); + + Igroup igroup = mock(Igroup.class); + when(igroup.getName()).thenReturn("igroup1"); + when(igroup.getUuid()).thenReturn("igroup-uuid-123"); + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setIgroup(igroup); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())).thenReturn("igroup1"); + + when(sanStrategy.getCloudStackVolume(argThat(map -> + map != null && "/vol/vol1/cs_tmpl_50".equals(map.get("name"))))) + .thenReturn(cachedTemplate); + when(sanStrategy.getAccessGroup(any())).thenReturn(accessGroup); + when(sanStrategy.validateInitiatorInAccessGroup(anyString(), anyString(), any(Igroup.class))).thenReturn(true); + + driver.revokeAccess(templateInfo, host, dataStore); + + verify(sanStrategy).disableLogicalAccess(argThat(map -> + map != null && "template-lun-uuid".equals(map.get("lun.uuid")) + && "igroup-uuid-123".equals(map.get("igroup.uuid")))); + } + } + + @Test + void testGetUsedIops_SumsMinIopsIncludingCreatingVolumes() { + VolumeVO readyVolume = mock(VolumeVO.class); + VolumeVO creatingVolume = mock(VolumeVO.class); + VolumeVO volumeWithoutMinIops = mock(VolumeVO.class); + + when(storagePool.getId()).thenReturn(1L); + when(readyVolume.getMinIops()).thenReturn(700L); + when(creatingVolume.getMinIops()).thenReturn(300L); + when(volumeWithoutMinIops.getMinIops()).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(1L, null)) + .thenReturn(List.of(readyVolume, creatingVolume, volumeWithoutMinIops)); + + assertEquals(1000L, driver.getUsedIops(storagePool)); + } + + @Test + void testResize_IopsUpdate_AttachesQosPolicy() { + long currentSize = 4L * 1024 * 1024 * 1024; + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 0L, 5000L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false)); + + VolumeQosPolicy qosPolicy = lookedUpQosPolicy("qos-uuid", "cs_0_to_5000_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, iscsiCloudStackVolume(), qosPolicy); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).resizeCloudStackVolume(argThat(request -> + request.getLun() != null + && "lun-uuid-123".equals(request.getLun().getUuid()) + && isQosPolicyReference(request.getLun().getQosPolicy(), "qos-uuid", "cs_0_to_5000_iops_svm1")), + eq(currentSize)); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + verify(volumeDetailsDao).addDetail(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false); + } + } + + @Test + void testResize_IopsUpdate_NfsAttachesQosPolicy() { + long currentSize = 4L * 1024 * 1024 * 1024; + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flex-uuid"); + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 100L, 200L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(null); + + VolumeQosPolicy qosPolicy = lookedUpQosPolicy("qos-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, nasStrategy, nfsCloudStackVolume(), qosPolicy); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(nasStrategy).resizeCloudStackVolume(argThat(request -> + "flex-uuid".equals(request.getFlexVolumeUuid()) + && request.getFile() != null + && isQosPolicyReference(request.getFile().getQosPolicy(), "qos-uuid", "cs_100_to_200_iops_svm1")), + eq(currentSize)); + verify(nasStrategy, never()).updateCloudStackVolume(any()); + verify(volumeDetailsDao).addDetail(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false); + } + } + + @Test + void testResize_SameQosPolicy_SkipsAttach() { + long currentSize = 4L * 1024 * 1024 * 1024; + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 0L, 3333L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false)); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_0_to_3333_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, iscsiCloudStackVolume(), qosPolicy); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).resizeCloudStackVolume(argThat(request -> request.getLun() == null), eq(currentSize)); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + } + } + + @Test + void testResize_ClearIops_DetachesQosPolicy() { + long currentSize = 4L * 1024 * 1024 * 1024; + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 0L, 0L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false)); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).resizeCloudStackVolume(argThat(request -> + request.getLun() != null + && isQosPolicyReference(request.getLun().getQosPolicy(), null, OntapStorageConstants.QOS_POLICY_NONE)), + eq(currentSize)); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + verify(sanStrategy).deleteVolumeQosPolicy("qos-uuid"); + } + } + + @Test + void testResize_SwitchIops_OldPolicyDeleteFails_StillSucceeds() { + long currentSize = 4L * 1024 * 1024 * 1024; + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 0L, 5000L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "old-qos-uuid", false)); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false)); + + VolumeQosPolicy qosPolicy = lookedUpQosPolicy("new-qos-uuid", "cs_0_to_5000_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, iscsiCloudStackVolume(), qosPolicy); + doThrow(new CloudRuntimeException("ONTAP QoS delete failed")).when(sanStrategy) + .deleteVolumeQosPolicy("old-qos-uuid"); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).resizeCloudStackVolume(argThat(request -> + request.getLun() != null + && isQosPolicyReference(request.getLun().getQosPolicy(), "new-qos-uuid", "cs_0_to_5000_iops_svm1")), + eq(currentSize)); + verify(volumeDetailsDao).addDetail(100L, OntapStorageConstants.QOS_POLICY_UUID, "new-qos-uuid", false); + verify(sanStrategy).deleteVolumeQosPolicy("old-qos-uuid"); + verify(sanStrategy, never()).deleteVolumeQosPolicy("new-qos-uuid"); + } + } + + @Test + void testResize_ClearIops_OldPolicyDeleteFails_StillSucceeds() { + long currentSize = 4L * 1024 * 1024 * 1024; + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 0L, 0L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false)); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + doThrow(new CloudRuntimeException("ONTAP QoS delete failed")).when(sanStrategy) + .deleteVolumeQosPolicy("qos-uuid"); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).deleteVolumeQosPolicy("qos-uuid"); + } + } + + @Test + void testResize_MinIopsBeyondPoolCapacity_Fails() { + long currentSize = 4L * 1024 * 1024 * 1024; + VolumeVO otherVolume = mock(VolumeVO.class); + when(otherVolume.getId()).thenReturn(50L); + when(otherVolume.getMinIops()).thenReturn(800L); + + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getDataStore()).thenReturn(dataStore); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 300L, 1000L, null, false, null, null, true)); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getId()).thenReturn(100L); + when(volumeVO.getSize()).thenReturn(currentSize); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("ontap-pool"); + when(storagePool.getCapacityIops()).thenReturn(1000L); + when(volumeDao.findNonDestroyedVolumesByPoolId(1L, null)).thenReturn(List.of(otherVolume, volumeVO)); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains( + "requested total of 1100 IOPS exceeds the pool IOPS capacity of 1000")); + verify(sanStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + verify(sanStrategy, never()).updateCloudStackVolume(any()); + } + + @Test + void testResize_MinIopsEqualToMaxIops_AttachesQosPolicy() { + long currentSize = 4L * 1024 * 1024 * 1024; + stubResizeVolumeRecord(currentSize); + when(volumeInfo.getpayload()).thenReturn( + new ResizeVolumePayload(currentSize, 100L, 100L, null, false, null, null, true)); + when(storagePool.getCapacityIops()).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false)); + + VolumeQosPolicy qosPolicy = lookedUpQosPolicy("qos-uuid", "cs_100_to_100_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, iscsiCloudStackVolume(), qosPolicy); + + driver.resize(volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_100_iops_svm1"), eq(100L), eq(100L)); + verify(sanStrategy).resizeCloudStackVolume(argThat(request -> + request.getLun() != null + && "lun-uuid-123".equals(request.getLun().getUuid()) + && isQosPolicyReference(request.getLun().getQosPolicy(), "qos-uuid", "cs_100_to_100_iops_svm1")), + eq(currentSize)); + } + } + + @Test + void testCreateAsync_MinIopsBeyondPoolCapacity_FailsWithCapacityDetails() { + VolumeVO allocatedVolume = mock(VolumeVO.class); + when(allocatedVolume.getId()).thenReturn(50L); + when(allocatedVolume.getMinIops()).thenReturn(800L); + + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + when(volumeInfo.getMinIops()).thenReturn(300L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("ontap-pool"); + when(storagePool.getCapacityIops()).thenReturn(1000L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeDao.findNonDestroyedVolumesByPoolId(1L, null)).thenReturn(List.of(allocatedVolume)); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains( + "storage pool ontap-pool: requested total of 1100 IOPS exceeds the pool IOPS capacity of 1000")); + verify(sanStrategy, never()).createCloudStackVolume(any()); + } + + @Test + void testCreateAsync_OtherCreatingVolumeCountsAgainstCapacity() { + VolumeVO readyVolume = mock(VolumeVO.class); + when(readyVolume.getId()).thenReturn(50L); + when(readyVolume.getMinIops()).thenReturn(500L); + + VolumeVO otherCreatingVolume = mock(VolumeVO.class); + when(otherCreatingVolume.getId()).thenReturn(60L); + when(otherCreatingVolume.getMinIops()).thenReturn(300L); + + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + when(volumeInfo.getMinIops()).thenReturn(300L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("ontap-pool"); + when(storagePool.getCapacityIops()).thenReturn(1000L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeDao.findNonDestroyedVolumesByPoolId(1L, null)) + .thenReturn(List.of(readyVolume, otherCreatingVolume)); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains( + "storage pool ontap-pool: requested total of 1100 IOPS exceeds the pool IOPS capacity of 1000")); + verify(sanStrategy, never()).createCloudStackVolume(any()); + } + + @Test + void testCreateAsync_DoesNotDoubleCountSelfWhenAlreadyCreating() { + VolumeVO readyVolume = mock(VolumeVO.class); + when(readyVolume.getId()).thenReturn(50L); + when(readyVolume.getMinIops()).thenReturn(500L); + + VolumeVO selfCreatingVolume = mock(VolumeVO.class); + when(selfCreatingVolume.getId()).thenReturn(100L); + + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + when(volumeInfo.getMinIops()).thenReturn(300L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + lenient().when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); + when(storagePool.getCapacityIops()).thenReturn(1000L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getId()).thenReturn(100L); + when(volumeDao.findNonDestroyedVolumesByPoolId(1L, null)) + .thenReturn(List.of(readyVolume, selfCreatingVolume)); + + Lun mockLun = new Lun(); + mockLun.setName("/vol/vol1/lun1"); + mockLun.setUuid("lun-uuid-123"); + CloudStackVolume cloudStackVolume = new CloudStackVolume(); + cloudStackVolume.setLun(mockLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + when(sanStrategy.createCloudStackVolume(any())).thenReturn(cloudStackVolume); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_DataDiskFixedIops_CreatesAndPersistsQosPolicy() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_100_to_200_iops_svm1"); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + verify(sanStrategy).createCloudStackVolume(argThat(request -> + request.getLun() != null && request.getLun().getQosPolicy() != null + && "qos-uuid".equals(request.getLun().getQosPolicy().getUuid()))); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + eq("qos-uuid"), eq(false)); + verify(sanStrategy, never()).isAff(); + } + } + + @Test + void testCreateAsync_FasPoolWithMinIops_FailsWithoutCallingOntap() { + stubIscsiVolumeCreate(); + storagePoolDetails.put(OntapStorageConstants.IS_AFF, "false"); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains(OntapStorageConstants.MIN_IOPS_NOT_SUPPORTED_ON_FAS)); + verify(sanStrategy, never()).isAff(); + verify(sanStrategy, never()).createVolumeQosPolicy(any(), any(), any()); + } + } + + @Test + void testCreateAsync_FasPoolWithMaxIopsOnly_CreatesQosPolicyWithoutNodeCall() { + stubIscsiVolumeCreate(); + storagePoolDetails.put(OntapStorageConstants.IS_AFF, "false"); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(0L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_0_to_200_iops_svm1"); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).isAff(); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_0_to_200_iops_svm1"), eq(0L), eq(200L)); + } + } + + @Test + void testCreateAsync_MissingAffDetail_FetchesFromNodesAndPersists() { + stubIscsiVolumeCreate(); + storagePoolDetails.remove(OntapStorageConstants.IS_AFF); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + when(sanStrategy.isAff()).thenReturn(true); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_100_to_200_iops_svm1"); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).isAff(); + verify(storagePoolDetailsDao).addDetail(1L, OntapStorageConstants.IS_AFF, "true", false); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + } + } + + @Test + void testCreateAsync_MissingAffDetailOnFas_PersistsFalseAndRejectsMinIops() { + stubIscsiVolumeCreate(); + storagePoolDetails.remove(OntapStorageConstants.IS_AFF); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + when(sanStrategy.isAff()).thenReturn(false); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains(OntapStorageConstants.MIN_IOPS_NOT_SUPPORTED_ON_FAS)); + verify(sanStrategy).isAff(); + verify(storagePoolDetailsDao).addDetail(1L, OntapStorageConstants.IS_AFF, "false", false); + verify(sanStrategy, never()).createVolumeQosPolicy(any(), any(), any()); + } + } + + @Test + void testCreateAsync_MissingAffDetail_IsAffThrows_StillCreatesWithMinIops() { + stubIscsiVolumeCreate(); + storagePoolDetails.remove(OntapStorageConstants.IS_AFF); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + when(sanStrategy.isAff()).thenThrow(new CloudRuntimeException( + "Unable to determine whether the ONTAP cluster is AFF or FAS")); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_100_to_200_iops_svm1"); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).isAff(); + verify(storagePoolDetailsDao, never()).addDetail(eq(1L), eq(OntapStorageConstants.IS_AFF), + anyString(), eq(false)); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + verify(sanStrategy).createCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_RootDiskCustomIops_CreatesAndPersistsQosPolicy() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(volumeInfo.getMinIops()).thenReturn(111L); + when(volumeInfo.getMaxIops()).thenReturn(999L); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-root-uuid", "cs_111_to_999_iops_svm1"); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_111_to_999_iops_svm1"), eq(111L), eq(999L)); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.QOS_POLICY_UUID), + eq("qos-root-uuid"), eq(false)); + } + } + + @Test + void testCreateAsync_NfsDataDiskWithIops_CreatesQosPolicy() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); + lenient().when(storagePool.getCapacityIops()).thenReturn(null); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getId()).thenReturn(100L); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-nfs-uuid", "cs_100_to_200_iops_svm1"); + CloudStackVolume cloudStackVolume = new CloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, nasStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(nasStrategy).createVolumeQosPolicy(eq("cs_100_to_200_iops_svm1"), eq(100L), eq(200L)); + verify(nasStrategy).createCloudStackVolume(argThat(request -> + request.getFile() != null && request.getFile().getQosPolicy() != null + && "qos-nfs-uuid".equals(request.getFile().getQosPolicy().getUuid()))); + } + } + + @Test + void testCreateAsync_MinIopsGreaterThanMaxIops_Fails() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(200L); + when(volumeInfo.getMaxIops()).thenReturn(100L); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains( + "Minimum IOPS cannot be greater than maximum IOPS")); + verify(sanStrategy, never()).createVolumeQosPolicy(any(), any(), any()); + verify(sanStrategy, never()).createCloudStackVolume(any()); + } + } + + @Test + void testCreateAsync_MinIopsEqualToMaxIops_CreatesQosPolicy() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(100L); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_100_to_100_iops_svm1"); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + stubQosCreateMocks(utilityMock, sanStrategy, cloudStackVolume, qosPolicy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).createVolumeQosPolicy(eq("cs_100_to_100_iops_svm1"), eq(100L), eq(100L)); + verify(sanStrategy).createCloudStackVolume(argThat(request -> + request.getLun() != null && request.getLun().getQosPolicy() != null + && "qos-uuid".equals(request.getLun().getQosPolicy().getUuid()))); + } + } + + @Test + void testCreateAsync_NoIops_DoesNotCreateQosPolicy() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + when(sanStrategy.createCloudStackVolume(any())).thenReturn(cloudStackVolume); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(sanStrategy, never()).createVolumeQosPolicy(any(), any(), any()); + verify(sanStrategy).createCloudStackVolume(argThat(request -> + request.getLun() != null && request.getLun().getQosPolicy() == null)); + } + } + + @Test + void testCreateAsync_SwapVolumeWithIops_DoesNotCreateQosPolicy() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.SWAP); + when(volumeInfo.getMinIops()).thenReturn(100L); + CloudStackVolume cloudStackVolume = iscsiCloudStackVolume(); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + when(sanStrategy.createCloudStackVolume(any())).thenReturn(cloudStackVolume); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + verify(sanStrategy, never()).createVolumeQosPolicy(any(), any(), any()); + } + } + + @Test + void testCreateAsync_QosCreateThenLunCreateFails_DeletesUnusedPolicy() { + stubIscsiVolumeCreate(); + when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo.getMinIops()).thenReturn(100L); + when(volumeInfo.getMaxIops()).thenReturn(200L); + + VolumeQosPolicy qosPolicy = qosPolicy("qos-uuid", "cs_100_to_200_iops_svm1"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(sanStrategy); + when(sanStrategy.createVolumeQosPolicy(nullable(String.class), nullable(Long.class), nullable(Long.class))) + .thenReturn(qosPolicy); + when(sanStrategy.createCloudStackVolume(any())).thenThrow(new CloudRuntimeException( + "Failed to create Lun: {\"error\":{\"code\":\"8454269\"}}")); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy).deleteVolumeQosPolicy("qos-uuid"); + } + } + + @Test + void testDeleteAsync_DeletesUnusedQosPolicy() { + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + VolumeDetailVO lunNameDetail = new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_NAME, "/vol/vol1/lun1", false); + VolumeDetailVO lunUuidDetail = new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false); + VolumeDetailVO qosDetail = new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false); + + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_NAME)).thenReturn(lunNameDetail); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)).thenReturn(lunUuidDetail); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(qosDetail); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + doNothing().when(sanStrategy).deleteCloudStackVolume(any()); + + driver.deleteAsync(dataStore, volumeInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(volumeDetailsDao, never()).removeDetail(100L, OntapStorageConstants.QOS_POLICY_UUID); + verify(sanStrategy).deleteVolumeQosPolicy("qos-uuid"); + } + } + + @Test + void testDeleteAsync_NfsClearsQosBeforeFileDelete() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flex-uuid"); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(nasStrategy); + when(nasStrategy.updateCloudStackVolume(any())).thenAnswer(invocation -> invocation.getArgument(0)); + doNothing().when(nasStrategy).deleteCloudStackVolume(any()); + + driver.deleteAsync(dataStore, volumeInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(nasStrategy).updateCloudStackVolume(argThat(request -> + "flex-uuid".equals(request.getFlexVolumeUuid()) + && request.getFile() != null + && isQosPolicyReference(request.getFile().getQosPolicy(), null, OntapStorageConstants.QOS_POLICY_NONE))); + verify(nasStrategy).deleteCloudStackVolume(any()); + verify(nasStrategy).deleteVolumeQosPolicy("qos-uuid"); + } + } + + @Test + void testDeleteAsync_NfsRestoresQosWhenFileDeleteFails() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flex-uuid"); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(nasStrategy); + when(nasStrategy.updateCloudStackVolume(any())).thenAnswer(invocation -> invocation.getArgument(0)); + doThrow(new CloudRuntimeException("file delete failed")).when(nasStrategy).deleteCloudStackVolume(any()); + + driver.deleteAsync(dataStore, volumeInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + ArgumentCaptor qosCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(nasStrategy, times(2)).updateCloudStackVolume(qosCaptor.capture()); + assertTrue(isQosPolicyReference(qosCaptor.getAllValues().get(0).getFile().getQosPolicy(), + null, OntapStorageConstants.QOS_POLICY_NONE)); + VolumeQosPolicy restored = qosCaptor.getAllValues().get(1).getFile().getQosPolicy(); + assertEquals("qos-uuid", restored.getUuid()); + assertNull(restored.getName()); + assertNull(restored.getSvm()); + assertNull(restored.getFixed()); + verify(nasStrategy, never()).deleteVolumeQosPolicy(any()); + } + } + + @Test + void testDeleteAsync_QosPolicyDeleteFails_VolumeDeleteStillSucceeds() { + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + VolumeDetailVO lunNameDetail = new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_NAME, "/vol/vol1/lun1", false); + VolumeDetailVO lunUuidDetail = new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false); + VolumeDetailVO qosDetail = new VolumeDetailVO(100L, OntapStorageConstants.QOS_POLICY_UUID, "qos-uuid", false); + + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_NAME)).thenReturn(lunNameDetail); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)).thenReturn(lunUuidDetail); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(qosDetail); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + doNothing().when(sanStrategy).deleteCloudStackVolume(any()); + doThrow(new CloudRuntimeException("ONTAP QoS delete failed")).when(sanStrategy) + .deleteVolumeQosPolicy("qos-uuid"); + + driver.deleteAsync(dataStore, volumeInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(volumeDetailsDao, never()).removeDetail(100L, OntapStorageConstants.QOS_POLICY_UUID); + verify(sanStrategy).deleteVolumeQosPolicy("qos-uuid"); + } + } + + private void stubIscsiVolumeCreate() { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + lenient().when(storagePool.getId()).thenReturn(1L); + lenient().when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + lenient().when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); + lenient().when(storagePool.getCapacityIops()).thenReturn(null); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + lenient().when(volumeVO.getId()).thenReturn(100L); + } + + private CloudStackVolume iscsiCloudStackVolume() { + Lun mockLun = new Lun(); + mockLun.setName("/vol/vol1/lun1"); + mockLun.setUuid("lun-uuid-123"); + CloudStackVolume volume = new CloudStackVolume(); + volume.setLun(mockLun); + return volume; + } + + private CloudStackVolume nfsCloudStackVolume() { + FileInfo fileInfo = new FileInfo(); + CloudStackVolume volume = new CloudStackVolume(); + volume.setFlexVolumeUuid("flex-uuid"); + volume.setFile(fileInfo); + volume.setVolumeInfo(volumeInfo); + return volume; + } + + private VolumeQosPolicy qosPolicy(String uuid, String name) { + VolumeQosPolicy policy = new VolumeQosPolicy(); + policy.setUuid(uuid); + policy.setName(name); + return policy; + } + + /** + * Policy as returned by an ONTAP lookup: name and uuid plus fields that must not be sent on a resize. + */ + private VolumeQosPolicy lookedUpQosPolicy(String uuid, String name) { + VolumeQosPolicy policy = qosPolicy(uuid, name); + Svm svm = new Svm(); + svm.setName("svm1"); + policy.setSvm(svm); + VolumeQosPolicy.Fixed fixed = new VolumeQosPolicy.Fixed(); + fixed.setMinThroughputIops(100L); + fixed.setMaxThroughputIops(5000L); + policy.setFixed(fixed); + policy.setObjectCount(2); + return policy; + } + + private boolean isQosPolicyReference(VolumeQosPolicy sent, String uuid, String name) { + return sent != null + && name.equals(sent.getName()) + && (uuid == null ? sent.getUuid() == null : uuid.equals(sent.getUuid())) + && sent.getSvm() == null + && sent.getFixed() == null + && sent.getObjectCount() == null; + } + + private void stubQosCreateMocks(MockedStatic utilityMock, + StorageStrategy strategy, + CloudStackVolume cloudStackVolume, VolumeQosPolicy qosPolicy) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(strategy); + when(strategy.createVolumeQosPolicy(nullable(String.class), nullable(Long.class), nullable(Long.class))) + .thenReturn(qosPolicy); + lenient().when(strategy.createCloudStackVolume(any())).thenReturn(cloudStackVolume); + } + + @Test + void testCanHostAccessStoragePool_ReturnsTrue() { + assertTrue(driver.canHostAccessStoragePool(host, storagePool)); + } + + @Test + void testIsVmInfoNeeded_ReturnsTrue() { + assertTrue(driver.isVmInfoNeeded()); + } + + @Test + void testIsStorageSupportHA_ReturnsTrue() { + assertTrue(driver.isStorageSupportHA(Storage.StoragePoolType.NetworkFilesystem)); + } + + @Test + void testGetChapInfo_ReturnsNull() { + assertNull(driver.getChapInfo(volumeInfo)); + } + + @Test + void testCanProvideStorageStats_ReturnsFalse() { + assertFalse(driver.canProvideStorageStats()); + } + + @Test + void testCanProvideVolumeStats_ReturnsFalse() { + assertFalse(driver.canProvideVolumeStats()); + } + + @Test + void testGetBytesRequiredForTemplate_NullArgs_ReturnsZero() { + assertEquals(0L, driver.getBytesRequiredForTemplate(null, storagePool)); + assertEquals(0L, driver.getBytesRequiredForTemplate(templateInfo, null)); + } + + @Test + void testCreateAsync_Template_StrategyThrows_Fails() { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + when(templateInfo.getSize()).thenReturn(5368709120L); + when(templateInfo.getName()).thenReturn("tmpl"); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.createTemplateCache(any(), any(), any(), anyLong())) + .thenThrow(new RuntimeException("unexpected")); + + driver.createAsync(dataStore, templateInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplate_NullCloneResult_Fails() { + stubVolumeCloneFromTemplate(5368709120L, 5368709120L); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(null); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplateNFS_MissingInstallPath_Fails() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + stubVolumeCloneFromTemplate(5368709120L, 5368709120L); + when(templatePoolRef.getInstallPath()).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(nasStrategy, never()).cloneCloudStackVolume(any()); + } + } + + @Test + void testDeleteAsync_Template_NoSpoolRef_Succeeds() { + when(dataStore.getId()).thenReturn(1L); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + + driver.deleteAsync(dataStore, templateInfo, commandCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CommandResult.class); + verify(commandCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).deleteCloudStackVolume(any()); + } + } + + @Test + void testGrantAccess_Template_CreatesIgroupWhenMissing() { + PrimaryDataStore primaryDataStore = mock(PrimaryDataStore.class); + Map dataStoreDetails = new HashMap<>(); + dataStoreDetails.put(PrimaryDataStore.MANAGED_STORE_TARGET, "stale-value"); + + when(primaryDataStore.getId()).thenReturn(1L); + when(primaryDataStore.getDetails()).thenReturn(dataStoreDetails); + when(templateInfo.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePool.getPath()).thenReturn("iqn.1992-08.com.netapp:sn.123456"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(templatePoolRef); + when(templatePoolRef.getId()).thenReturn(7L); + + HostVO hostVo = mock(HostVO.class); + when(hostVo.getName()).thenReturn("host1"); + when(hostVo.getUuid()).thenReturn("host-uuid-1"); + + AccessGroup createdAccessGroup = new AccessGroup(); + Igroup createdIgroup = new Igroup(); + createdIgroup.setName("igroup1"); + createdAccessGroup.setIgroup(createdIgroup); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())).thenReturn("igroup1"); + + when(sanStrategy.getAccessGroup(any())).thenReturn(null); + when(sanStrategy.createAccessGroup(any())).thenReturn(createdAccessGroup); + when(sanStrategy.ensureLunMapped(eq("svm1"), eq("/vol/vol1/cs_tmpl_50"), eq("igroup1"))).thenReturn("3"); + when(sanStrategy.getLunSerialNumber(eq("svm1"), eq("/vol/vol1/cs_tmpl_50"))).thenReturn(LUN_SERIAL_NUMBER); + + assertTrue(driver.grantAccess(templateInfo, hostVo, primaryDataStore)); + + verify(sanStrategy).createAccessGroup(any()); + verify(templatePoolRef).setInstallPath("/iqn.1992-08.com.netapp:sn.123456/" + LUN_WWID); + } + } + + @Test + void testCreateAsync_VolumeClonedFromTemplate_MissingSpoolRef_Fails() { + when(dataStore.getId()).thenReturn(1L); + when(dataStore.getName()).thenReturn("ontap-pool"); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getName()).thenReturn("test-volume"); + + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + lenient().when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.CLONE_OF_SNAPSHOT)).thenReturn(null); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.CLONE_OF_TEMPLATE)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.CLONE_OF_TEMPLATE, "50", false)); + when(vmTemplatePoolDao.findByPoolTemplate(1L, 50L, null)).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + + driver.createAsync(dataStore, volumeInfo, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + verify(sanStrategy, never()).cloneCloudStackVolume(any()); + } + } + + // ========================================================================= + // resize() tests + // ========================================================================= + + private void stubResizeVolumeRecord(long currentSize) { + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getDataStore()).thenReturn(dataStore); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getId()).thenReturn(100L); + when(volumeVO.getSize()).thenReturn(currentSize); + lenient().when(volumeVO.getPath()).thenReturn("/vol/vol1/lun1"); + lenient().when(volumeDao.update(100L, volumeVO)).thenReturn(true); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + } + + private DataObject stubResizeCommon(long currentSize, long newSize) { + ResizeVolumePayload payload = new ResizeVolumePayload(newSize, null, null, null, false, "i-2-VM", null, false); + DataObject data = volumeInfo; + when(data.getType()).thenReturn(VOLUME); + when(volumeInfo.getDataStore()).thenReturn(dataStore); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn(payload); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + // storagePool.getName() is only needed for the shrink-error message path + lenient().when(storagePool.getName()).thenReturn("test-pool"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getSize()).thenReturn(currentSize); + // volumeVO.getId() is only needed in the success path (volumeDao.update call) + lenient().when(volumeVO.getId()).thenReturn(100L); + lenient().when(volumeVO.getPath()).thenReturn("/vol/vol1/lun1"); + lenient().when(volumeDao.update(100L, volumeVO)).thenReturn(true); + return data; + } + + @Test + void testResize_iSCSI_Success() { + long currentSize = 10737418240L; // 10 GB + long newSize = 21474836480L; // 20 GB + DataObject data = stubResizeCommon(currentSize, newSize); + assertEquals(VOLUME, data.getType()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + doNothing().when(sanStrategy).resizeCloudStackVolume(any(), eq(newSize)); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + // Driver only wires VolumeInfo; LUN UUID lookup is inside UnifiedSANStrategy + ArgumentCaptor volumeCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).resizeCloudStackVolume(volumeCaptor.capture(), eq(newSize)); + assertEquals(volumeInfo, volumeCaptor.getValue().getVolumeInfo()); + assertNull(volumeCaptor.getValue().getLun()); + verify(volumeDetailsDao, never()).findDetail(anyLong(), eq(OntapStorageConstants.LUN_DOT_UUID)); + + verify(volumeVO).setSize(newSize); + verify(volumeDao).update(eq(100L), any(VolumeVO.class)); + } + } + + @Test + void testResize_NFS_Success() { + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, ProtocolType.NFS3.name()); + long currentSize = 10737418240L; + long newSize = 21474836480L; + DataObject data = stubResizeCommon(currentSize, newSize); + assertEquals(VOLUME, data.getType()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(nasStrategy); + doNothing().when(nasStrategy).resizeCloudStackVolume(any(), eq(newSize)); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + + ArgumentCaptor volumeCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(nasStrategy).resizeCloudStackVolume(volumeCaptor.capture(), eq(newSize)); + assertEquals(volumeInfo, volumeCaptor.getValue().getVolumeInfo()); + // NFS has no LUN – driver must not look up a LUN UUID + verify(volumeDetailsDao, never()).findDetail(anyLong(), eq(OntapStorageConstants.LUN_DOT_UUID)); + verify(volumeVO).setSize(newSize); + verify(volumeDao).update(eq(100L), any(VolumeVO.class)); + } + } + + @Test + void testResize_NullPayload_Fails() { + DataObject data = volumeInfo; + when(data.getType()).thenReturn(VOLUME); + assertEquals(VOLUME, data.getType()); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn(null); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Invalid resize payload")); + } + + @Test + void testResize_NullData_FailsAndCompletesCallbackOnce() { + driver.resize(null, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback, times(1)).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Expected a VOLUME DataObject")); + } + + @Test + void testResize_NonVolumeDataType_FailsAndCompletesCallbackOnce() { + DataObject data = templateInfo; + when(data.getType()).thenReturn(TEMPLATE); + when(templateInfo.getId()).thenReturn(50L); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback, times(1)).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Expected a VOLUME DataObject but received TEMPLATE")); + } + + @Test + void testResize_SnapshotDataType_FailsAndCompletesCallbackOnce() { + DataObject data = mock(DataObject.class); + when(data.getType()).thenReturn(SNAPSHOT); + when(data.getId()).thenReturn(75L); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback, times(1)).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Expected a VOLUME DataObject but received SNAPSHOT")); + } + + @Test + void testResize_VolumeTypeWithoutVolumeDetails_FailsWithoutInternalTypeName() { + DataObject data = mock(DataObject.class); + when(data.getType()).thenReturn(VOLUME); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback, times(1)).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + String message = resultCaptor.getValue().getResult(); + assertTrue(message.contains("Invalid DataObjectType (VOLUME) passed to resize")); + assertFalse(message.contains("VolumeInfo")); + } + + @Test + void testResize_NullNewSize_Fails() { + // payload.newSize is null + ResizeVolumePayload payload = new ResizeVolumePayload(null, null, null, null, false, "i-2-VM", null, false); + DataObject data = volumeInfo; + when(data.getType()).thenReturn(VOLUME); + assertEquals(VOLUME, data.getType()); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn(payload); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Invalid resize payload")); + } + + @Test + void testResize_StoragePoolNotFound_Fails() { + ResizeVolumePayload payload = new ResizeVolumePayload(21474836480L, null, null, null, false, "i-2-VM", null, false); + DataObject data = volumeInfo; + when(data.getType()).thenReturn(VOLUME); + assertEquals(VOLUME, data.getType()); + when(volumeInfo.getDataStore()).thenReturn(dataStore); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn(payload); + when(storagePoolDao.findById(1L)).thenReturn(null); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Storage pool not found")); + } + + @Test + void testResize_MissingDataStore_Fails() { + ResizeVolumePayload payload = new ResizeVolumePayload(21474836480L, null, null, null, false, "none", null, false); + DataObject data = volumeInfo; + when(data.getType()).thenReturn(VOLUME); + assertEquals(VOLUME, data.getType()); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn(payload); + when(volumeInfo.getDataStore()).thenReturn(null); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Data store not found")); + } + + @Test + void testResize_VolumeVONotFound_Fails() { + long newSize = 21474836480L; + ResizeVolumePayload payload = new ResizeVolumePayload(newSize, null, null, null, false, "i-2-VM", null, false); + DataObject data = volumeInfo; + when(data.getType()).thenReturn(VOLUME); + assertEquals(VOLUME, data.getType()); + when(volumeInfo.getDataStore()).thenReturn(dataStore); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getpayload()).thenReturn(payload); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Volume not found")); + } + } + + @Test + void testResize_MissingStoragePoolDetails_Fails() { + long currentSize = 10737418240L; + long newSize = 21474836480L; + DataObject data = stubResizeCommon(currentSize, newSize); + assertEquals(VOLUME, data.getType()); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(null)) + .thenThrow(new CloudRuntimeException("Storage pool details are missing")); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback, times(1)).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Storage pool details are missing")); + verify(volumeVO, never()).setSize(anyLong()); + } + } + + @Test + void testResize_ShrinkAttempt_Fails() { + long currentSize = 21474836480L; // 20 GB + long newSize = 10737418240L; // 10 GB – smaller than current + DataObject data = stubResizeCommon(currentSize, newSize); + assertEquals(VOLUME, data.getType()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Unable to shrink volume")); + } + } + + private static VolumeVO temporarySnapshotCopyVolume() { + return new VolumeVO(Volume.Type.DATADISK, "ROOT-5_20260924.TMP", 1L, 1L, 2L, 0L, + Storage.ProvisioningType.THIN, 5368709120L, 0L, 0L, ""); + } + + private SnapshotInfo stubSnapshotToVolumeCopy(VolumeVO destVolume) { + SnapshotInfo snapshotInfo = mock(SnapshotInfo.class); + lenient().when(snapshotInfo.getType()).thenReturn(SNAPSHOT); + lenient().when(snapshotInfo.getDataStore()).thenReturn(dataStore); + lenient().when(snapshotInfo.getId()).thenReturn(200L); + lenient().when(volumeInfo.getType()).thenReturn(VOLUME); + lenient().when(volumeInfo.getDataStore()).thenReturn(dataStore); + lenient().when(volumeInfo.getId()).thenReturn(100L); + // VolumeObject.getName() reads the VolumeVO, which copyAsync renames before cloning. + lenient().when(volumeInfo.getName()).thenReturn("cs_tmp_snap_200_100"); + lenient().when(dataStore.getId()).thenReturn(1L); + lenient().when(dataStore.getRole()).thenReturn(DataStoreRole.Primary); + lenient().when(volumeDao.findById(100L)).thenReturn(destVolume); + return snapshotInfo; + } + + private void stubSnapshotCloneSource(String protocol) { + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + lenient().when(storagePool.getId()).thenReturn(1L); + lenient().when(storagePool.getName()).thenReturn("vol1"); + lenient().when(storagePool.getHypervisor()).thenReturn(Hypervisor.HypervisorType.KVM); + storagePoolDetails.put(OntapStorageConstants.PROTOCOL, protocol); + storagePoolDetails.put(OntapStorageConstants.VOLUME_NAME, "vol1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + lenient().when(volumeInfo.getSize()).thenReturn(5368709120L); + + String volumePath = ProtocolType.NFS3.name().equals(protocol) ? "source-file-uuid" : "/vol/vol1/source_lun"; + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.ONTAP_SNAP_NAME)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.ONTAP_SNAP_NAME, "snap_cs200", false)); + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.VOLUME_PATH)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.VOLUME_PATH, volumePath, false)); + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.PRIMARY_POOL_ID)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.PRIMARY_POOL_ID, "1", false)); + lenient().when(snapshotDetailsDao.findDetail(200L, OntapStorageConstants.PROTOCOL)) + .thenReturn(new SnapshotDetailsVO(200L, OntapStorageConstants.PROTOCOL, protocol, false)); + SnapshotVO snapshotVO = mock(SnapshotVO.class); + lenient().when(snapshotDao.findById(200L)).thenReturn(snapshotVO); + lenient().when(snapshotVO.getSize()).thenReturn(5368709120L); + } + + @Test + void testCanCopy_TemporarySnapshotCopyOnSamePool_ReturnsTrue() { + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(temporarySnapshotCopyVolume()); + + assertTrue(driver.canCopy(snapshotInfo, volumeInfo)); + } + + @Test + void testCanCopy_CreateVolumeFromSnapshot_ReturnsFalse() { + VolumeVO userVolume = temporarySnapshotCopyVolume(); + userVolume.setDiskOfferingId(5L); + userVolume.setState(Volume.State.Creating); + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(userVolume); + + assertFalse(driver.canCopy(snapshotInfo, volumeInfo)); + } + + @Test + void testCanCopy_AllocatedVolumeWithDiskOffering_ReturnsFalse() { + VolumeVO userVolume = temporarySnapshotCopyVolume(); + userVolume.setDiskOfferingId(5L); + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(userVolume); + + assertFalse(driver.canCopy(snapshotInfo, volumeInfo)); + } + + @Test + void testCanCopy_DifferentPool_ReturnsFalse() { + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(temporarySnapshotCopyVolume()); + DataStore otherStore = mock(DataStore.class); + when(otherStore.getId()).thenReturn(2L); + when(volumeInfo.getDataStore()).thenReturn(otherStore); + + assertFalse(driver.canCopy(snapshotInfo, volumeInfo)); + verify(volumeDao, never()).findById(anyLong()); + } + + @Test + void testCanCopy_TemplateCacheCopies_ReturnFalse() { + TemplateInfo templateOnPrimary = mock(TemplateInfo.class); + lenient().when(templateInfo.getType()).thenReturn(TEMPLATE); + lenient().when(templateInfo.getDataStore()).thenReturn(dataStore); + lenient().when(templateOnPrimary.getType()).thenReturn(TEMPLATE); + lenient().when(templateOnPrimary.getDataStore()).thenReturn(dataStore); + lenient().when(volumeInfo.getType()).thenReturn(VOLUME); + lenient().when(volumeInfo.getDataStore()).thenReturn(dataStore); + + assertFalse(driver.canCopy(templateInfo, templateOnPrimary)); + assertFalse(driver.canCopy(templateOnPrimary, volumeInfo)); + verify(volumeDao, never()).findById(anyLong()); + } + + @Test + void testCanCopy_VolumeMigration_ReturnsFalse() { + VolumeInfo destVolumeInfo = mock(VolumeInfo.class); + DataStore destStore = mock(DataStore.class); + lenient().when(volumeInfo.getType()).thenReturn(VOLUME); + lenient().when(volumeInfo.getDataStore()).thenReturn(dataStore); + lenient().when(destVolumeInfo.getType()).thenReturn(VOLUME); + lenient().when(destVolumeInfo.getDataStore()).thenReturn(destStore); + + assertFalse(driver.canCopy(volumeInfo, destVolumeInfo)); + verify(volumeDao, never()).findById(anyLong()); + } + + @Test + void testCopyAsync_VolumeMigration_Throws() { + VolumeInfo destVolumeInfo = mock(VolumeInfo.class); + DataStore destStore = mock(DataStore.class); + lenient().when(volumeInfo.getType()).thenReturn(VOLUME); + lenient().when(volumeInfo.getDataStore()).thenReturn(dataStore); + lenient().when(destVolumeInfo.getType()).thenReturn(VOLUME); + lenient().when(destVolumeInfo.getDataStore()).thenReturn(destStore); + + assertThrows(UnsupportedOperationException.class, + () -> driver.copyAsync(volumeInfo, destVolumeInfo, null, null)); + verify(sanStrategy, never()).cloneCloudStackVolume(any()); + verify(nasStrategy, never()).cloneCloudStackVolume(any()); + } + + @Test + void testCopyAsync_UnsupportedPair_Throws() { + lenient().when(templateInfo.getType()).thenReturn(TEMPLATE); + lenient().when(templateInfo.getDataStore()).thenReturn(dataStore); + lenient().when(volumeInfo.getType()).thenReturn(VOLUME); + lenient().when(volumeInfo.getDataStore()).thenReturn(dataStore); + + assertThrows(UnsupportedOperationException.class, () -> driver.copyAsync(templateInfo, volumeInfo, null, null)); + } + + @Test + void testCopyAsync_IscsiTemporarySnapshotCopy_ClonesLunAndRecordsIdentity() { + VolumeVO dbVolume = temporarySnapshotCopyVolume(); + VolumeVO inMemoryVolume = temporarySnapshotCopyVolume(); + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(dbVolume); + when(volumeInfo.getVolume()).thenReturn(inMemoryVolume); + stubSnapshotCloneSource(ProtocolType.ISCSI.name()); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/cs_tmp_snap_200_100"); + clonedLun.setUuid("tmp-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.copyAsync(snapshotInfo, volumeInfo, null, null); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertEquals("/vol/vol1/.snapshot/snap_cs200/source_lun", + requestCaptor.getValue().getLun().getClone().getSource().getName()); + assertEquals("/vol/vol1/cs_tmp_snap_200_100", requestCaptor.getValue().getLun().getName()); + assertSame(volumeInfo, requestCaptor.getValue().getVolumeInfo()); + + assertEquals("cs_tmp_snap_200_100", dbVolume.getName()); + assertEquals("cs_tmp_snap_200_100", inMemoryVolume.getName()); + assertEquals(Storage.ImageFormat.RAW, dbVolume.getFormat()); + assertEquals("tmp-lun-uuid", dbVolume.getFolder()); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_UUID), eq("tmp-lun-uuid"), eq(false)); + verify(volumeDetailsDao).addDetail(eq(100L), eq(OntapStorageConstants.LUN_DOT_NAME), eq("/vol/vol1/cs_tmp_snap_200_100"), eq(false)); + verify(volumeDao).update(anyLong(), eq(dbVolume)); + verify(sanStrategy, never()).resizeCloudStackVolume(any(), anyLong()); + verify(sanStrategy, never()).createVolumeQosPolicy(anyString(), any(), any()); + } + } + + @Test + void testCopyAsync_IscsiTemporarySnapshotCopy_IgnoresVolumeIops() { + VolumeVO dbVolume = temporarySnapshotCopyVolume(); + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(dbVolume); + when(volumeInfo.getVolume()).thenReturn(temporarySnapshotCopyVolume()); + stubSnapshotCloneSource(ProtocolType.ISCSI.name()); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + lenient().when(volumeInfo.getVolumeType()).thenReturn(Volume.Type.DATADISK); + lenient().when(volumeInfo.getMinIops()).thenReturn(100L); + lenient().when(volumeInfo.getMaxIops()).thenReturn(200L); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/cs_tmp_snap_200_100"); + clonedLun.setUuid("tmp-lun-uuid"); + CloudStackVolume cloned = new CloudStackVolume(); + cloned.setLun(clonedLun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.copyAsync(snapshotInfo, volumeInfo, null, null); + + verify(sanStrategy, never()).createVolumeQosPolicy(nullable(String.class), nullable(Long.class), nullable(Long.class)); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(sanStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertNull(requestCaptor.getValue().getLun().getQosPolicy()); + } + } + + @Test + void testResize_StrategyThrows_Fails() { + long currentSize = 10737418240L; + long newSize = 21474836480L; + DataObject data = stubResizeCommon(currentSize, newSize); + assertEquals(VOLUME, data.getType()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + doThrow(new com.cloud.utils.exception.CloudRuntimeException("ONTAP resize failed")) + .when(sanStrategy).resizeCloudStackVolume(any(), eq(newSize)); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("ONTAP resize failed")); + // volumeVO size must NOT be updated on failure + verify(volumeVO, never()).setSize(anyLong()); + } + } + + @Test + void testCopyAsync_NfsTemporarySnapshotCopy_SetsFilePath() { + VolumeVO dbVolume = temporarySnapshotCopyVolume(); + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(dbVolume); + when(volumeInfo.getVolume()).thenReturn(temporarySnapshotCopyVolume()); + when(volumeInfo.getUuid()).thenReturn("tmp-volume-uuid"); + stubSnapshotCloneSource(ProtocolType.NFS3.name()); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + + CloudStackVolume cloned = new CloudStackVolume(); + FileInfo file = new FileInfo(); + file.setPath("tmp-volume-uuid"); + cloned.setFile(file); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(nasStrategy); + when(nasStrategy.cloneCloudStackVolume(any())).thenReturn(cloned); + + driver.copyAsync(snapshotInfo, volumeInfo, null, null); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CloudStackVolume.class); + verify(nasStrategy).cloneCloudStackVolume(requestCaptor.capture()); + assertEquals("./.snapshot/snap_cs200/source-file-uuid", + requestCaptor.getValue().getFile().getPath()); + assertEquals("tmp-volume-uuid", requestCaptor.getValue().getDestinationPath()); + + assertEquals("tmp-volume-uuid", dbVolume.getPath()); + assertEquals(Storage.ImageFormat.QCOW2, dbVolume.getFormat()); + verify(volumeDao).update(anyLong(), eq(dbVolume)); + } + } + + @Test + void testResize_EqualSize_InvokesStrategyAndSucceeds() { + long size = 10737418240L; + DataObject data = stubResizeCommon(size, size); + assertEquals(VOLUME, data.getType()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + + driver.resize(data, createCallback); + + verify(sanStrategy).resizeCloudStackVolume(any(), eq(size)); + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback).complete(resultCaptor.capture()); + assertTrue(resultCaptor.getValue().isSuccess()); + } + } + + @Test + void testCopyAsync_CloneFailsWithoutCallback_Throws() { + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(temporarySnapshotCopyVolume()); + when(volumeInfo.getVolume()).thenReturn(temporarySnapshotCopyVolume()); + stubSnapshotCloneSource(ProtocolType.ISCSI.name()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenThrow(new CloudRuntimeException("clone failed")); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> driver.copyAsync(snapshotInfo, volumeInfo, null, null)); + assertTrue(ex.getMessage().contains("clone failed")); + verify(volumeDetailsDao, never()).addDetail(anyLong(), anyString(), anyString(), anyBoolean()); + } + } + + @Test + void testResize_DatabaseUpdateFailure_ReturnsFailure() { + long currentSize = 10737418240L; + long newSize = 21474836480L; + DataObject data = stubResizeCommon(currentSize, newSize); + assertEquals(VOLUME, data.getType()); + when(volumeDao.update(100L, volumeVO)).thenReturn(false); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + + driver.resize(data, createCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CreateCmdResult.class); + verify(createCallback, times(1)).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + assertTrue(resultCaptor.getValue().getResult().contains("Failed to update volume")); + } + } + + // ========================================================================= + // getUsedBytes() tests + // ========================================================================= + + @Test + void testGetUsedBytes_NullPool_ThrowsException() { + InvalidParameterValueException ex = assertThrows(InvalidParameterValueException.class, + () -> driver.getUsedBytes(null)); + assertTrue(ex.getMessage().contains("storagePool is null, ensure the pool exists and is fully initialised before querying used bytes")); + } + + @Test + void testGetUsedBytes_NoFlexVolUuid_ThrowsException() { + // VOLUME_UUID key is absent from pool details + storagePoolDetails.remove(OntapStorageConstants.VOLUME_UUID); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, () -> driver.getUsedBytes(storagePool)); + assertTrue(ex.getMessage().contains("FlexVolume UUID not found in pool details for pool 1")); + } + + @Test + void testGetUsedBytes_BlankFlexVolUuid_ThrowsException() { + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, " "); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, () -> driver.getUsedBytes(storagePool)); + assertTrue(ex.getMessage().contains("FlexVolume UUID not found in pool details for pool 1")); + } + + @Test + void testGetUsedBytes_NullPoolDetails_ThrowsException() { + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(null); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, () -> driver.getUsedBytes(storagePool)); + assertTrue(ex.getMessage().contains("FlexVolume UUID not found in pool details for pool 1")); + } + + @Test + void testGetUsedBytes_Success_ReturnsUsedBytes() { + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-123"); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + var flexVol = new org.apache.cloudstack.storage.feign.model.Volume(); + VolumeSpace space = new VolumeSpace(); + space.setUsed(10737418240L); // 10 GB + flexVol.setSpace(space); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + Map queryParams = Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.SPACE_USED); + when(sanStrategy.getStorageVolume(eq("flexvol-uuid-123"), eq(queryParams))).thenReturn(flexVol); + + assertEquals(10737418240L, driver.getUsedBytes(storagePool)); + verify(sanStrategy).getStorageVolume(eq("flexvol-uuid-123"), eq(queryParams)); + } + } + + @Test + void testGetUsedBytes_FlexVolNotFound_ThrowsException() { + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-123"); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + when(sanStrategy.getStorageVolume(eq("flexvol-uuid-123"), any())).thenReturn(null); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> driver.getUsedBytes(storagePool)); + assertTrue(ex.getMessage().contains("was not found on ONTAP")); + } + } + + @Test + void testCopyAsync_CloneFailsWithCallback_CompletesWithFailure() { + SnapshotInfo snapshotInfo = stubSnapshotToVolumeCopy(temporarySnapshotCopyVolume()); + when(volumeInfo.getVolume()).thenReturn(temporarySnapshotCopyVolume()); + stubSnapshotCloneSource(ProtocolType.ISCSI.name()); + @SuppressWarnings("unchecked") + AsyncCompletionCallback copyCallback = mock(AsyncCompletionCallback.class); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(sanStrategy); + when(sanStrategy.cloneCloudStackVolume(any())) + .thenThrow(new CloudRuntimeException("clone failed")); + + driver.copyAsync(snapshotInfo, volumeInfo, null, copyCallback); + + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(CopyCommandResult.class); + verify(copyCallback).complete(resultCaptor.capture()); + assertFalse(resultCaptor.getValue().isSuccess()); + } + } + + @Test + void testDeleteAsync_TemporarySnapshotCopy_NullCallbackRemovesRecord() { + VolumeVO tempVolume = temporarySnapshotCopyVolume(); + tempVolume.setName("cs_tmp_snap_200_100"); + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(tempVolume); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_NAME)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_NAME, "/vol/vol1/cs_tmp_snap_200_100", false)); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "tmp-lun-uuid", false)); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.QOS_POLICY_UUID)).thenReturn(null); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + + driver.deleteAsync(dataStore, volumeInfo, null); + + verify(sanStrategy).deleteCloudStackVolume(any(CloudStackVolume.class)); + verify(volumeDetailsDao).removeDetails(100L); + verify(volumeDao).remove(100L); + } + } + + @Test + void testGetUsedBytes_FlexVolNullSpace_ThrowsException() { + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-123"); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + var flexVol = new org.apache.cloudstack.storage.feign.model.Volume(); + // space is intentionally left null + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + when(sanStrategy.getStorageVolume(eq("flexvol-uuid-123"), any())).thenReturn(flexVol); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> driver.getUsedBytes(storagePool)); + assertTrue(ex.getMessage().contains("no space information")); + } + } + + @Test + void testGetUsedBytes_OntapException_PropagatesException() { + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-123"); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + when(sanStrategy.getStorageVolume(eq("flexvol-uuid-123"), any())) + .thenThrow(new com.cloud.utils.exception.CloudRuntimeException("ONTAP unreachable")); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> driver.getUsedBytes(storagePool)); + assertEquals("ONTAP unreachable", ex.getMessage()); + } + } + + @Test + void testGetUsedBytes_UnexpectedException_IsWrapped() { + storagePoolDetails.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-123"); + when(storagePool.getId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)) + .thenReturn(sanStrategy); + when(sanStrategy.getStorageVolume(eq("flexvol-uuid-123"), any())) + .thenThrow(new IllegalStateException("invalid ONTAP response")); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> driver.getUsedBytes(storagePool)); + assertTrue(ex.getMessage().contains("Could not read used space")); + assertTrue(ex.getMessage().contains("invalid ONTAP response")); + assertTrue(ex.getCause() instanceof IllegalStateException); + } + } + + @Test + void testGrantAccess_TemporarySnapshotCopy_SyncsIscsiPathOnInMemoryVolume() { + String iscsiPath = "/iqn.1992-08.com.netapp:sn.123456/0"; + VolumeVO dbVolume = mock(VolumeVO.class); + when(dbVolume.getId()).thenReturn(100L); + when(dbVolume.getState()).thenReturn(Volume.State.Allocated); + when(dbVolume.getDiskOfferingId()).thenReturn(0L); + when(dbVolume.getPath()).thenReturn(iscsiPath); + when(dbVolume.get_iScsiName()).thenReturn(iscsiPath); + VolumeVO inMemoryVolume = temporarySnapshotCopyVolume(); + + when(dataStore.getId()).thenReturn(1L); + when(volumeInfo.getType()).thenReturn(VOLUME); + when(volumeInfo.getId()).thenReturn(100L); + when(volumeInfo.getVolume()).thenReturn(inMemoryVolume); + when(storagePoolDao.findById(1L)).thenReturn(storagePool); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePool.getPath()).thenReturn("iqn.1992-08.com.netapp:sn.123456"); + when(storagePool.getPoolType()).thenReturn(Storage.StoragePoolType.OntapiSCSI); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(storagePoolDetails); + when(volumeDao.findById(100L)).thenReturn(dbVolume); + when(host.getUuid()).thenReturn("host-uuid-1"); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_NAME)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_NAME, "/vol/vol1/cs_tmp_snap_200_100", false)); + + AccessGroup existingAccessGroup = new AccessGroup(); + Igroup existingIgroup = new Igroup(); + existingIgroup.setName("igroup1"); + existingAccessGroup.setIgroup(existingIgroup); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(storagePoolDetails)).thenReturn(sanStrategy); + utilityMock.when(() -> OntapStorageUtils.getIgroupName(anyString(), anyString())).thenReturn("igroup1"); + when(sanStrategy.getAccessGroup(any())).thenReturn(existingAccessGroup); + when(sanStrategy.ensureLunMapped(anyString(), anyString(), anyString())).thenReturn("0"); + when(sanStrategy.getLunSerialNumber(anyString(), anyString())).thenReturn(LUN_SERIAL_NUMBER); + + assertTrue(driver.grantAccess(volumeInfo, host, dataStore)); + + assertEquals(iscsiPath, inMemoryVolume.getPath()); + assertEquals(iscsiPath, inMemoryVolume.get_iScsiName()); + } } } diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycleTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycleTest.java index 751b864ecfcc..de3e6bd9e234 100644 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycleTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/lifecycle/OntapPrimaryDatastoreLifecycleTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; @@ -30,34 +31,49 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import org.apache.cloudstack.storage.feign.model.Aggregate; import org.apache.cloudstack.storage.feign.model.Volume; import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.dc.ClusterVO; import com.cloud.host.HostVO; import com.cloud.resource.ResourceManager; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.service.model.AccessGroup; import com.cloud.hypervisor.Hypervisor; +import com.cloud.alert.AlertManager; +import com.cloud.capacity.CapacityManager; import java.util.Map; import java.util.List; import java.util.ArrayList; +import java.util.HashMap; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.withSettings; +import static org.mockito.Mockito.mock; +import org.mockito.InOrder; +import static org.mockito.ArgumentMatchers.contains; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertFalse; -import java.util.HashMap; +import com.cloud.storage.StoragePool; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; import org.apache.cloudstack.storage.provider.StorageProviderFactory; import org.apache.cloudstack.storage.service.StorageStrategy; import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; @@ -92,6 +108,12 @@ public class OntapPrimaryDatastoreLifecycleTest { @Mock private PrimaryDataStoreDao storagePoolDao; + @Mock + private AlertManager _alertMgr; + + @Mock + private CapacityManager _capacityMgr; + // Mock object that implements both DataStore and PrimaryDataStoreInfo // This is needed because attachCluster(DataStore) casts DataStore to PrimaryDataStoreInfo internally private DataStore dataStore; @@ -116,12 +138,21 @@ void setUp() { when(_clusterDao.findById(1L)).thenReturn(clusterVO); when(storageStrategy.connect()).thenReturn(true); - when(storageStrategy.getNetworkInterface()).thenReturn("testNetworkInterface"); + Aggregate aggregate = new Aggregate(); + aggregate.setName("aggr1"); + aggregate.setUuid("aggr-uuid-1"); + Aggregate.Node node = new Aggregate.Node(); + node.setName("node-a"); + aggregate.setNode(node); + when(storageStrategy.isAff()).thenReturn(true); + when(storageStrategy.chooseAggregate(any(), any())).thenReturn(aggregate); + when(storageStrategy.getNetworkInterface(any())).thenReturn( + Map.of(OntapStorageConstants.DATA_LIF, "testNetworkInterface")); Volume volume = new Volume(); volume.setUuid("test-volume-uuid"); volume.setName("testVolume"); - when(storageStrategy.createStorageVolume(any(), any())).thenReturn(volume); + when(storageStrategy.createStorageVolume(any(), any(), any())).thenReturn(volume); // Setup for attachCluster tests // Configure dataStore mock with necessary methods (works for both DataStore and PrimaryDataStoreInfo) @@ -153,6 +184,7 @@ void setUp() { poolDetails.put("svmName", "svm1"); poolDetails.put("protocol", "NFS3"); poolDetails.put("storageIP", "192.168.1.100"); + when(zoneScope.getScopeId()).thenReturn(1L); } @Test @@ -171,7 +203,7 @@ public void testInitialize_positive() { dsInfos.put("clusterId", 1L); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -183,6 +215,112 @@ public void testInitialize_positive() { } } + private Map buildDsInfosForProtocol(String protocol) { + HashMap detailsMap = new HashMap(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, protocol); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + return dsInfos; + } + + private Storage.StoragePoolType initializeAndCapturePoolType(String protocol) { + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + ontapPrimaryDatastoreLifecycle.initialize(buildDsInfosForProtocol(protocol)); + } + ArgumentCaptor captor = ArgumentCaptor.forClass(PrimaryDataStoreParameters.class); + verify(_dataStoreHelper).createPrimaryDataStore(captor.capture()); + return captor.getValue().getType(); + } + + @Test + public void testInitialize_iscsiPoolUsesOntapIscsiType() { + when(storageStrategy.getStoragePath()).thenReturn("iqn.1992-08.com.netapp:sn.abc123"); + + assertEquals(Storage.StoragePoolType.OntapiSCSI, initializeAndCapturePoolType("ISCSI")); + } + + @Test + public void testInitialize_nfsPoolKeepsNetworkFilesystemType() { + assertEquals(Storage.StoragePoolType.NetworkFilesystem, initializeAndCapturePoolType("NFS3")); + } + + private Long initializeAndCaptureCapacityIops(Map dsInfos) { + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + ontapPrimaryDatastoreLifecycle.initialize(dsInfos); + } + ArgumentCaptor captor = ArgumentCaptor.forClass(PrimaryDataStoreParameters.class); + verify(_dataStoreHelper).createPrimaryDataStore(captor.capture()); + return captor.getValue().getCapacityIops(); + } + + @Test + public void testInitialize_blankCapacityIopsLeavesPoolWithoutIopsCeiling() { + assertNull(initializeAndCaptureCapacityIops(buildDsInfosForProtocol("NFS3"))); + } + + @Test + public void testInitialize_capacityIopsIsStoredOnPool() { + Map dsInfos = buildDsInfosForProtocol("NFS3"); + dsInfos.put("capacityIops", 5000L); + + assertEquals(Long.valueOf(5000L), initializeAndCaptureCapacityIops(dsInfos)); + } + + @Test + public void testInitialize_persistsAffPlatformDetail() { + Map details = initializeAndCaptureDetails(buildDsInfosForProtocol("NFS3")); + + assertEquals("true", details.get(OntapStorageConstants.IS_AFF)); + verify(storageStrategy).isAff(); + } + + @Test + public void testInitialize_persistsFasPlatformDetail() { + when(storageStrategy.isAff()).thenReturn(false); + + Map details = initializeAndCaptureDetails(buildDsInfosForProtocol("NFS3")); + + assertEquals("false", details.get(OntapStorageConstants.IS_AFF)); + } + + private Map initializeAndCaptureDetails(Map dsInfos) { + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + ontapPrimaryDatastoreLifecycle.initialize(dsInfos); + } + ArgumentCaptor captor = ArgumentCaptor.forClass(PrimaryDataStoreParameters.class); + verify(_dataStoreHelper).createPrimaryDataStore(captor.capture()); + return captor.getValue().getDetails(); + } + + @Test + public void testInitialize_nonPositiveCapacityIopsIsRejected() { + Map dsInfos = buildDsInfosForProtocol("NFS3"); + dsInfos.put("capacityIops", 0L); + + Exception ex = assertThrows(InvalidParameterValueException.class, + () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + + assertTrue(ex.getMessage().contains("IOPS capacity must be greater than 0")); + verify(_dataStoreHelper, never()).createPrimaryDataStore(any()); + } + @Test public void testInitialize_null_Arg() { Exception ex = assertThrows(CloudRuntimeException.class,() -> @@ -205,7 +343,7 @@ public void testInitialize_missingRequiredDetailKey() { dsInfos.put("clusterId", 1L); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -242,7 +380,37 @@ public void testInitialize_invalidCapacityBytes() { try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); - ontapPrimaryDatastoreLifecycle.initialize(dsInfos); + Exception ex = assertThrows(InvalidParameterValueException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("must be at least")); + } + } + + @Test + public void testInitialize_capacityBelowOntapMinimum() { + + HashMap detailsMap = new HashMap(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId",1L); + dsInfos.put("podId",1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 20971519L); + dsInfos.put("managed",true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + Exception ex = assertThrows(InvalidParameterValueException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("below the ONTAP minimum volume size")); } } @@ -254,7 +422,7 @@ public void testInitialize_unmanagedStorage() { dsInfos.put("clusterId", 1L); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",false); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -277,7 +445,7 @@ public void testInitialize_nullStoragePoolName() { dsInfos.put("clusterId", 1L); dsInfos.put("name", null); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -300,7 +468,7 @@ public void testInitialize_nullProviderName() { dsInfos.put("clusterId", 1L); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", null); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -323,7 +491,7 @@ public void testInitialize_nullPodAndClusterAndZone() { dsInfos.put("clusterId", null); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -357,7 +525,7 @@ public void testInitialize_clusterNotKVM() { dsInfos.put("clusterId", 2L); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes", 200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed", true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -389,7 +557,7 @@ public void testInitialize_unexpectedDetailKey() { dsInfos.put("clusterId", 1L); dsInfos.put("name", "testStoragePool"); dsInfos.put("providerName", "testProvider"); - dsInfos.put("capacityBytes",200000L); + dsInfos.put("capacityBytes", 1073741824L); dsInfos.put("managed",true); dsInfos.put("tags", "testTag"); dsInfos.put("isTagARule", false); @@ -404,6 +572,248 @@ public void testInitialize_unexpectedDetailKey() { assertTrue(ex.getMessage().contains("Unexpected ONTAP detail key in URL")); } + @Test + public void testInitialize_dataLifWithWarning() { + // Test when getNetworkInterface returns a warning in LIF_WARNING + // This exercises the processDataLifSelection path for non-null warning + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + String warningMessage = "LIF on node-b; expected on node-a;Details about LIF failover"; + when(storageStrategy.getNetworkInterface(any())).thenReturn(Map.of( + OntapStorageConstants.DATA_LIF, "10.0.0.1", + OntapStorageConstants.LIF_WARNING, warningMessage)); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class); + MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + ontapPrimaryDatastoreLifecycle.initialize(dsInfos); + + // Verify alert was sent with warning message + utilityMock.verify(() -> OntapStorageUtils.sendStorageAlert(eq(_alertMgr), eq(1L), eq(1L), + contains("LIF on node-b"), eq(warningMessage)), times(1)); + } + } + + @Test + public void testInitialize_nullDataLif() { + // Test when DATA_LIF is missing from the result map + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + when(storageStrategy.getNetworkInterface(any())).thenReturn(new HashMap<>()); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + Exception ex = assertThrows(CloudRuntimeException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("Failed to retrieve Data LIF from ONTAP, cannot create primary storage")); + verify(storageStrategy, never()).createStorageVolume(any(), any(), any()); + } + } + + @Test + public void testInitialize_emptyDataLif() { + // Test when DATA_LIF is an empty string + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + when(storageStrategy.getNetworkInterface(any())).thenReturn( + Map.of(OntapStorageConstants.DATA_LIF, "")); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + Exception ex = assertThrows(CloudRuntimeException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("Failed to retrieve Data LIF from ONTAP, cannot create primary storage")); + verify(storageStrategy, never()).createStorageVolume(any(), any(), any()); + } + } + + @Test + public void testInitialize_getNetworkInterfaceException() { + // Test when getNetworkInterface throws an exception + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + when(storageStrategy.getNetworkInterface(any())).thenThrow(new RuntimeException("ONTAP API error")); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + Exception ex = assertThrows(CloudRuntimeException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("Failed to retrieve Data LIF from ONTAP")); + assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("ONTAP API error")); + verify(storageStrategy, never()).createStorageVolume(any(), any(), any()); + } + } + + @Test + public void testInitialize_volumeCreationFailure_nullVolume() { + // Test when createStorageVolume returns null + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + when(storageStrategy.createStorageVolume(any(), any(), any())).thenReturn(null); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + Exception ex = assertThrows(CloudRuntimeException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("Failed to create ONTAP volume")); + } + } + + @Test + public void testInitialize_volumeCreationException() { + // Test when createStorageVolume throws an exception + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + when(storageStrategy.createStorageVolume(any(), any(), any())).thenThrow(new RuntimeException("Volume creation failed")); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + Exception ex = assertThrows(CloudRuntimeException.class, () -> ontapPrimaryDatastoreLifecycle.initialize(dsInfos)); + assertTrue(ex.getMessage().contains("Failed to create ONTAP volume")); + assertTrue(ex.getCause() != null && ex.getCause().getMessage().contains("Volume creation failed")); + } + } + + @Test + public void testInitialize_positiveWithDetailAssertions() { + // Enhanced positive test that verifies DATA_LIF detail is persisted and host is set correctly + HashMap detailsMap = new HashMap<>(); + detailsMap.put(OntapStorageConstants.USERNAME, "testUser"); + detailsMap.put(OntapStorageConstants.PASSWORD, "testPassword"); + detailsMap.put(OntapStorageConstants.STORAGE_IP, "10.10.10.10"); + detailsMap.put(OntapStorageConstants.SVM_NAME, "vs0"); + detailsMap.put(OntapStorageConstants.PROTOCOL, "NFS3"); + + Map dsInfos = new HashMap<>(); + dsInfos.put("zoneId", 1L); + dsInfos.put("podId", 1L); + dsInfos.put("clusterId", 1L); + dsInfos.put("name", "testStoragePool"); + dsInfos.put("providerName", "testProvider"); + dsInfos.put("capacityBytes", 1073741824L); + dsInfos.put("managed", true); + dsInfos.put("tags", "testTag"); + dsInfos.put("isTagARule", false); + dsInfos.put("details", detailsMap); + + String expectedDataLif = "192.168.1.100"; + when(storageStrategy.getNetworkInterface(any())).thenReturn( + Map.of(OntapStorageConstants.DATA_LIF, expectedDataLif)); + when(storageStrategy.getStoragePath()).thenReturn("/vol/testVolume"); + + try (MockedStatic storageProviderFactory = Mockito.mockStatic(StorageProviderFactory.class)) { + storageProviderFactory.when(() -> StorageProviderFactory.getStrategy(any())).thenReturn(storageStrategy); + ontapPrimaryDatastoreLifecycle.initialize(dsInfos); + + // Verify LIF selection completes before FlexVol creation + InOrder inOrder = inOrder(storageStrategy); + inOrder.verify(storageStrategy).chooseAggregate(any(), any()); + inOrder.verify(storageStrategy).getNetworkInterface(any()); + inOrder.verify(storageStrategy).createStorageVolume(any(), any(), any()); + + // Verify that createPrimaryDataStore was called and host parameter contains the DATA_LIF + verify(_dataStoreHelper, times(1)).createPrimaryDataStore(any()); + } + } + // ========== attachCluster Tests ========== @Test @@ -412,12 +822,12 @@ public void testAttachCluster_positive() throws Exception { when(_resourceMgr.getEligibleUpAndEnabledHostsInClusterForStorageConnection(any())) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); - when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); // Mock successful host connections when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); @@ -446,12 +856,12 @@ public void testAttachCluster_withSingleHost() throws Exception { when(_resourceMgr.getEligibleUpAndEnabledHostsInClusterForStorageConnection(any())) .thenReturn(singleHost); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); - when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); // Execute @@ -477,12 +887,12 @@ public void testAttachCluster_withMultipleHosts() throws Exception { when(_resourceMgr.getEligibleUpAndEnabledHostsInClusterForStorageConnection(any())) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); - when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); // Execute @@ -507,6 +917,7 @@ public void testAttachCluster_hostConnectionFailure() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); // Mock host connection failure for first host when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())) @@ -533,12 +944,12 @@ public void testAttachCluster_emptyHostList() throws Exception { when(_resourceMgr.getEligibleUpAndEnabledHostsInClusterForStorageConnection(any())) .thenReturn(emptyHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); - when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); // Execute boolean result = ontapPrimaryDatastoreLifecycle.attachCluster( @@ -562,6 +973,7 @@ public void testAttachCluster_secondHostConnectionFails() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); // Mock: first host succeeds, second host fails when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())) @@ -585,12 +997,12 @@ public void testAttachCluster_createAccessGroupCalled() throws Exception { when(_resourceMgr.getEligibleUpAndEnabledHostsInClusterForStorageConnection(any())) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); - when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachCluster(any(DataStore.class))).thenReturn(dataStore); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); // Execute @@ -608,7 +1020,6 @@ public void testAttachCluster_createAccessGroupCalled() throws Exception { @Test public void testAttachZone_positive() throws Exception { // Setup - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -618,6 +1029,7 @@ public void testAttachZone_positive() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); // Mock successful host connections when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); @@ -643,7 +1055,6 @@ public void testAttachZone_withSingleHost() throws Exception { List singleHost = new ArrayList<>(); singleHost.add(mockHosts.get(0)); - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(singleHost); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -653,6 +1064,7 @@ public void testAttachZone_withSingleHost() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); // Execute @@ -675,7 +1087,6 @@ public void testAttachZone_withMultipleHosts() throws Exception { host3.setClusterId(1L); mockHosts.add(host3); - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -685,6 +1096,7 @@ public void testAttachZone_withMultipleHosts() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); // Execute @@ -701,7 +1113,6 @@ public void testAttachZone_withMultipleHosts() throws Exception { @Test public void testAttachZone_hostConnectionFailure() throws Exception { // Setup - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -710,6 +1121,7 @@ public void testAttachZone_hostConnectionFailure() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); // Mock host connection failure for first host when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())) @@ -733,7 +1145,6 @@ public void testAttachZone_emptyHostList() throws Exception { // Setup - no hosts in zone List emptyHosts = new ArrayList<>(); - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(emptyHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -743,6 +1154,7 @@ public void testAttachZone_emptyHostList() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); // Execute boolean result = ontapPrimaryDatastoreLifecycle.attachZone( @@ -758,7 +1170,6 @@ public void testAttachZone_emptyHostList() throws Exception { @Test public void testAttachZone_secondHostConnectionFails() throws Exception { // Setup - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -767,6 +1178,7 @@ public void testAttachZone_secondHostConnectionFails() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); // Mock: first host succeeds, second host fails when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())) @@ -787,7 +1199,6 @@ public void testAttachZone_secondHostConnectionFails() throws Exception { @Test public void testAttachZone_createAccessGroupCalled() throws Exception { // Setup - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -797,6 +1208,7 @@ public void testAttachZone_createAccessGroupCalled() throws Exception { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); + when(_dataStoreHelper.attachZone(any(DataStore.class))).thenReturn(dataStore); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); // Execute @@ -834,7 +1246,6 @@ public void testAttachZone_nullHypervisorThrowsException() { @Test public void testAttachZone_kvmHypervisorSetsAndUpdatesPool() throws Exception { // KVM hypervisorType should be set on the pool and persisted via storagePoolDao.update - when(zoneScope.getScopeId()).thenReturn(1L); when(_resourceMgr.getEligibleUpAndEnabledHostsInZoneForStorageConnection(any(), eq(1L), eq(Hypervisor.HypervisorType.KVM))) .thenReturn(mockHosts); when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); @@ -843,7 +1254,6 @@ public void testAttachZone_kvmHypervisorSetsAndUpdatesPool() throws Exception { try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) .thenReturn(storageStrategy); - when(storageStrategy.createAccessGroup(any(AccessGroup.class))).thenReturn(null); when(_storageMgr.connectHostToSharedPool(any(HostVO.class), anyLong())).thenReturn(true); boolean result = ontapPrimaryDatastoreLifecycle.attachZone( @@ -854,4 +1264,142 @@ public void testAttachZone_kvmHypervisorSetsAndUpdatesPool() throws Exception { } } + // ========== updateStoragePool() Tests ========== + + @Test + public void testUpdateStoragePool_positive_resizesFlexVolume() { + // Setup + StoragePool storagePool = mock(StoragePool.class); + when(storagePool.getName()).thenReturn("test-pool"); + when(storagePool.getCapacityBytes()).thenReturn(2147483648L); // 2 GB current + + Map details = new HashMap<>(); + details.put(PrimaryDataStoreLifeCycle.CAPACITY_BYTES, String.valueOf(5368709120L)); // 5 GB new + details.put(OntapStorageConstants.VOLUME_UUID, "flex-vol-uuid-123"); + details.put(OntapStorageConstants.VOLUME_NAME, "flexvol-name"); + details.put("protocol", "NFS3"); + + Volume updatedVolume = new Volume(); + updatedVolume.setUuid("flex-vol-uuid-123"); + updatedVolume.setSize(5368709120L); + when(storageStrategy.updateStorageVolume(any(Volume.class))).thenReturn(updatedVolume); + + try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(storageStrategy); + + // Execute + ontapPrimaryDatastoreLifecycle.updateStoragePool(storagePool, details); + + // Verify + verify(storageStrategy, times(1)).updateStorageVolume(any(Volume.class)); + } + } + + @Test + public void testUpdateStoragePool_noCapacityBytesInDetails_skipsResize() { + // Setup + StoragePool storagePool = mock(StoragePool.class); + when(storagePool.getName()).thenReturn("test-pool"); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.VOLUME_UUID, "flex-vol-uuid-123"); + details.put("protocol", "NFS3"); + // No CAPACITY_BYTES key — resize should be skipped + + // Execute + ontapPrimaryDatastoreLifecycle.updateStoragePool(storagePool, details); + + // Verify — storageStrategy should never be called + verify(storageStrategy, never()).updateStorageVolume(any()); + } + + @Test + public void testUpdateStoragePool_missingVolumeUuid_throwsCloudRuntimeException() { + // Setup + StoragePool storagePool = mock(StoragePool.class); + when(storagePool.getName()).thenReturn("test-pool"); + when(storagePool.getCapacityBytes()).thenReturn(1073741824L); + + Map details = new HashMap<>(); + details.put(PrimaryDataStoreLifeCycle.CAPACITY_BYTES, String.valueOf(3221225472L)); + details.put("protocol", "NFS3"); + // No VOLUME_UUID — cannot resize without it + + try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(storageStrategy); + + // Execute & Verify + assertThrows(CloudRuntimeException.class, + () -> ontapPrimaryDatastoreLifecycle.updateStoragePool(storagePool, details)); + verify(storageStrategy, never()).updateStorageVolume(any()); + } + } + + @Test + public void testUpdateStoragePool_capacityIopsBelowAllocated_throwsInvalidParameterValueException() { + StoragePool storagePool = mock(StoragePool.class); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("test-pool"); + + Map details = new HashMap<>(); + details.put(PrimaryDataStoreLifeCycle.CAPACITY_IOPS, "400"); + details.put(OntapStorageConstants.VOLUME_UUID, "flex-vol-uuid-123"); + details.put("protocol", "NFS3"); + + when(_capacityMgr.getUsedIops(any(StoragePoolVO.class))).thenReturn(900L); + + Exception ex = assertThrows(InvalidParameterValueException.class, + () -> ontapPrimaryDatastoreLifecycle.updateStoragePool(storagePool, details)); + + assertTrue(ex.getMessage().contains("900 IOPS are already allocated")); + verify(storageStrategy, never()).updateStorageVolume(any(Volume.class)); + } + + @Test + public void testUpdateStoragePool_capacityIopsAboveAllocated_isAccepted() { + StoragePool storagePool = mock(StoragePool.class); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("test-pool"); + + Map details = new HashMap<>(); + details.put(PrimaryDataStoreLifeCycle.CAPACITY_IOPS, "2000"); + details.put("protocol", "NFS3"); + // No CAPACITY_BYTES key — only the IOPS ceiling is being raised. + + when(_capacityMgr.getUsedIops(any(StoragePoolVO.class))).thenReturn(900L); + + ontapPrimaryDatastoreLifecycle.updateStoragePool(storagePool, details); + + verify(storageStrategy, never()).updateStorageVolume(any(Volume.class)); + } + + @Test + public void testUpdateStoragePool_updateStorageVolumeThrows_propagatesCloudRuntimeException() { + // Setup + StoragePool storagePool = mock(StoragePool.class); + when(storagePool.getName()).thenReturn("test-pool"); + when(storagePool.getCapacityBytes()).thenReturn(1073741824L); + + Map details = new HashMap<>(); + details.put(PrimaryDataStoreLifeCycle.CAPACITY_BYTES, String.valueOf(3221225472L)); + details.put(OntapStorageConstants.VOLUME_UUID, "flex-vol-uuid-err"); + details.put(OntapStorageConstants.VOLUME_NAME, "flexvol-err"); + details.put("protocol", "NFS3"); + + when(storageStrategy.updateStorageVolume(any(Volume.class))) + .thenThrow(new CloudRuntimeException("ONTAP resize failed")); + + try (MockedStatic utilityMock = Mockito.mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenReturn(storageStrategy); + + // Execute & Verify + assertThrows(CloudRuntimeException.class, + () -> ontapPrimaryDatastoreLifecycle.updateStoragePool(storagePool, details)); + verify(storageStrategy, times(1)).updateStorageVolume(any(Volume.class)); + } + } + } diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/StorageStrategyTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/StorageStrategyTest.java index df9afe2542f9..17d553951637 100644 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/StorageStrategyTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/StorageStrategyTest.java @@ -18,15 +18,25 @@ */ package org.apache.cloudstack.storage.service; -import com.cloud.utils.exception.CloudRuntimeException; -import feign.FeignException; +import java.lang.reflect.Field; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + import org.apache.cloudstack.storage.feign.client.AggregateFeignClient; +import org.apache.cloudstack.storage.feign.client.ClusterFeignClient; import org.apache.cloudstack.storage.feign.client.JobFeignClient; import org.apache.cloudstack.storage.feign.client.NetworkFeignClient; import org.apache.cloudstack.storage.feign.client.SANFeignClient; +import org.apache.cloudstack.storage.feign.client.SnapshotFeignClient; import org.apache.cloudstack.storage.feign.client.SvmFeignClient; import org.apache.cloudstack.storage.feign.client.VolumeFeignClient; import org.apache.cloudstack.storage.feign.model.Aggregate; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.ClusterNode; import org.apache.cloudstack.storage.feign.model.IpInterface; import org.apache.cloudstack.storage.feign.model.IscsiService; import org.apache.cloudstack.storage.feign.model.Job; @@ -39,32 +49,36 @@ import org.apache.cloudstack.storage.service.model.CloudStackVolume; import org.apache.cloudstack.storage.service.model.ProtocolType; import org.apache.cloudstack.storage.utils.OntapStorageConstants; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import org.mockito.Mock; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import com.cloud.utils.exception.CloudRuntimeException; + +import feign.FeignException; +import feign.Request; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) @@ -88,6 +102,12 @@ public class StorageStrategyTest { @Mock private SANFeignClient sanFeignClient; + @Mock + private SnapshotFeignClient snapshotFeignClient; + + @Mock + private ClusterFeignClient clusterFeignClient; + private TestableStorageStrategy storageStrategy; // Concrete implementation for testing abstract class @@ -98,7 +118,9 @@ public TestableStorageStrategy(OntapStorage ontapStorage, SvmFeignClient svmFeignClient, JobFeignClient jobFeignClient, NetworkFeignClient networkFeignClient, - SANFeignClient sanFeignClient) { + SANFeignClient sanFeignClient, + SnapshotFeignClient snapshotFeignClient, + ClusterFeignClient clusterFeignClient) { super(ontapStorage); // Use reflection to replace the private Feign client fields with mocked ones injectMockedClient("aggregateFeignClient", aggregateFeignClient); @@ -107,6 +129,8 @@ public TestableStorageStrategy(OntapStorage ontapStorage, injectMockedClient("jobFeignClient", jobFeignClient); injectMockedClient("networkFeignClient", networkFeignClient); injectMockedClient("sanFeignClient", sanFeignClient); + injectMockedClient("snapshotFeignClient", snapshotFeignClient); + injectMockedClient("clusterFeignClient", clusterFeignClient); } private void injectMockedClient(String fieldName, Object mockedClient) { @@ -125,7 +149,14 @@ public CloudStackVolume createCloudStackVolume(CloudStackVolume cloudstackVolume } @Override - CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume) { + public CloudStackVolume createTemplateCache(org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool, + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo, + Map details, long sizeInBytes) { + return null; + } + + @Override + public CloudStackVolume updateCloudStackVolume(CloudStackVolume cloudstackVolume) { return null; } @@ -134,8 +165,12 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) { } @Override - public void copyCloudStackVolume(CloudStackVolume cloudstackVolume) { + public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume) { + return null; + } + @Override + public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes) { } @Override @@ -158,7 +193,7 @@ public void deleteAccessGroup(AccessGroup accessGroup) { } @Override - AccessGroup updateAccessGroup(AccessGroup accessGroup) { + public AccessGroup updateAccessGroup(AccessGroup accessGroup) { return null; } @@ -192,7 +227,8 @@ void setUp() { // For testing, we'll need to mock the FeignClientFactory behavior storageStrategy = new TestableStorageStrategy(ontapStorage, aggregateFeignClient, volumeFeignClient, svmFeignClient, - jobFeignClient, networkFeignClient, sanFeignClient); + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); } // ========== connect() Tests ========== @@ -214,13 +250,8 @@ public void testConnect_positive() { svmResponse.setRecords(List.of(svm)); when(svmFeignClient.getSvmResponse(anyMap(), anyString())).thenReturn(svmResponse); - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(Aggregate.StateEnum.ONLINE); - when(aggregateDetail.getSpace()).thenReturn(mock(Aggregate.AggregateSpace.class)); - when(aggregateDetail.getAvailableBlockStorageSpace()).thenReturn(10000000000.0); - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))).thenReturn(aggregateDetail); + Aggregate aggregateDetail = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0); + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())).thenReturn(aggregateDetail); // Execute boolean result = storageStrategy.connect(); @@ -231,12 +262,40 @@ public void testConnect_positive() { } @Test - public void testConnect_succeedsWhenAggregateSpaceBelowPoolCapacity() { - // Regression: connect() must validate connectivity/SVM/aggregate-state ONLY. - // Capacity is validated per-volume in createStorageVolume(name, size). Previously - // connect() compared aggregate free space against the whole storage pool size - // (storage.getSize()), which incorrectly failed data-path operations (volume/LUN - // create, grant/revoke access, delete) once the pool FlexVolume already existed. + public void testConnect_collectsAllEligibleAggregates() { + Svm svm = new Svm(); + svm.setName("svm1"); + svm.setState(OntapStorageConstants.RUNNING); + svm.setNfsEnabled(true); + + Aggregate aggregate1 = new Aggregate(); + aggregate1.setName("aggr1"); + aggregate1.setUuid("aggr-uuid-1"); + Aggregate aggregate2 = new Aggregate(); + aggregate2.setName("aggr2"); + aggregate2.setUuid("aggr-uuid-2"); + svm.setAggregates(List.of(aggregate1, aggregate2)); + + OntapResponse svmResponse = new OntapResponse<>(); + svmResponse.setRecords(List.of(svm)); + + when(svmFeignClient.getSvmResponse(anyMap(), anyString())).thenReturn(svmResponse); + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())) + .thenReturn(buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0)); + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-2"), anyMap())) + .thenReturn(buildAggregate("aggr2", "aggr-uuid-2", 20000000000.0)); + + assertTrue(storageStrategy.connect()); + + List aggregates = storageStrategy.getAggregates(); + assertNotNull(aggregates); + assertEquals(2, aggregates.size()); + assertEquals("aggr-uuid-1", aggregates.get(0).getUuid()); + assertEquals("aggr-uuid-2", aggregates.get(1).getUuid()); + } + + @Test + public void testConnect_operationsOnly_skipsAggregateValidation() { Svm svm = new Svm(); svm.setName("svm1"); svm.setState(OntapStorageConstants.RUNNING); @@ -253,17 +312,187 @@ public void testConnect_succeedsWhenAggregateSpaceBelowPoolCapacity() { when(svmFeignClient.getSvmResponse(anyMap(), anyString())).thenReturn(svmResponse); // Aggregate is ONLINE but has far less free space than the configured pool size (5GB). - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(Aggregate.StateEnum.ONLINE); - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))).thenReturn(aggregateDetail); + Aggregate aggregateDetail = buildAggregate("aggr1", "aggr-uuid-1", 1000000.0); // only 1MB free + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())).thenReturn(aggregateDetail); - // Execute & Verify - connect() should succeed regardless of available space. - boolean result = storageStrategy.connect(); + // Execute & Verify - connect(false) should succeed regardless of available space. + boolean result = storageStrategy.connect(false); assertTrue(result, "connect() should succeed for an online aggregate even when its free space is below the pool capacity"); } + @Test + public void testGetClusterInfo_dedupesIdenticalNodeModels() { + Cluster cluster = stubClusterGet(); + ClusterNode node1 = new ClusterNode(); + node1.setModel("AFF-A400"); + ClusterNode node2 = new ClusterNode(); + node2.setModel("AFF-A400"); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of(node1, node2))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals(cluster, result); + assertEquals("AFF-A400", result.getModel()); + } + + @Test + public void testGetClusterInfo_joinsDistinctModels() { + stubClusterGet(); + ClusterNode node1 = new ClusterNode(); + node1.setModel("AFF-A400"); + ClusterNode node2 = new ClusterNode(); + node2.setModel("FAS8300"); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of(node1, node2))); + + assertEquals("AFF-A400,FAS8300", storageStrategy.getClusterInfo().getModel()); + } + + @Test + public void testGetClusterInfo_allFlashPerformance() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", true, true, false), + clusterNode("AFF-A400", true, true, false)))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals("AFF-A400", result.getModel()); + assertEquals(OntapStorageConstants.ASUP_PLATFORM_TYPE_PERFORMANCE, result.getPlatformType()); + } + + @Test + public void testGetClusterInfo_allFlashCapacity() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-C800", true, false, true), + clusterNode("AFF-C800", true, false, true)))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals("AFF-C800", result.getModel()); + assertEquals(OntapStorageConstants.ASUP_PLATFORM_TYPE_CAPACITY, result.getPlatformType()); + } + + @Test + public void testGetClusterInfo_notAllFlashIsFas() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("FAS8300", false, false, false), + clusterNode("FAS8300", false, false, false)))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals("FAS8300", result.getModel()); + assertEquals(OntapStorageConstants.ASUP_PLATFORM_TYPE_FAS, result.getPlatformType()); + } + + @Test + public void testGetClusterInfo_mixedPersonalitiesIsComposite() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", true, true, false), + clusterNode("AFF-A400", true, true, false), + clusterNode("FAS8300", false, false, false), + clusterNode("FAS8300", false, false, false)))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals("AFF-A400,FAS8300", result.getModel()); + assertEquals(OntapStorageConstants.ASUP_PLATFORM_TYPE_COMPOSITE, result.getPlatformType()); + } + + @Test + public void testGetClusterInfo_missingAllFlashFlagLeavesPlatformUnset() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", null, null, null)))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals("AFF-A400", result.getModel()); + assertNull(result.getPlatformType()); + } + + @Test + public void testGetClusterInfo_missingFlagDoesNotChangeKnownPlatform() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", true, true, false), + clusterNode("AFF-A400", null, null, null)))); + + Cluster result = storageStrategy.getClusterInfo(); + assertEquals("AFF-A400", result.getModel()); + assertEquals(OntapStorageConstants.ASUP_PLATFORM_TYPE_PERFORMANCE, result.getPlatformType()); + } + + @Test + public void testGetClusterInfo_nodesGetFailureLeavesModelUnset() { + stubClusterGet(); + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenThrow(new RuntimeException("connection refused")); + + Cluster result = storageStrategy.getClusterInfo(); + assertNotNull(result); + assertNull(result.getModel()); + assertNull(result.getPlatformType()); + } + + @Test + public void testIsAff_allNodesAllFlash_returnsTrue() { + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", true, true, false), + clusterNode("AFF-A400", true, true, false)))); + + assertTrue(storageStrategy.isAff()); + } + + @Test + public void testIsAff_anyNodeNotAllFlash_returnsFalse() { + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", true, true, false), + clusterNode("FAS8300", false, false, false)))); + + assertFalse(storageStrategy.isAff()); + } + + @Test + public void testIsAff_nullAllFlashOptimized_returnsFalse() { + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of( + clusterNode("AFF-A400", true, true, false), + clusterNode("unknown", null, null, null)))); + + assertFalse(storageStrategy.isAff()); + } + + @Test + public void testIsAff_noNodes_throws() { + when(clusterFeignClient.getClusterNodes(anyString(), anyMap())) + .thenReturn(new OntapResponse<>(List.of())); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, () -> storageStrategy.isAff()); + assertTrue(ex.getMessage().contains("Unable to determine whether the ONTAP cluster is AFF or FAS")); + } + + private Cluster stubClusterGet() { + Cluster cluster = new Cluster(); + when(clusterFeignClient.getCluster(anyString(), eq(true))).thenReturn(cluster); + return cluster; + } + + private static ClusterNode clusterNode(String model, Boolean allFlash, Boolean performance, Boolean capacity) { + ClusterNode node = new ClusterNode(); + node.setModel(model); + node.setAllFlashOptimized(allFlash); + node.setPerformanceOptimized(performance); + node.setCapacityOptimized(capacity); + return node; + } + @Test public void testConnect_noOnlineAggregates() { // Setup - aggregate assigned to the SVM exists but is not ONLINE @@ -282,15 +511,20 @@ public void testConnect_noOnlineAggregates() { when(svmFeignClient.getSvmResponse(anyMap(), anyString())).thenReturn(svmResponse); - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(null); // not online - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))).thenReturn(aggregateDetail); + Aggregate aggregateDetail = new Aggregate(); + aggregateDetail.setName("aggr1"); + aggregateDetail.setUuid("aggr-uuid-1"); + aggregateDetail.setState(null); // not online + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())).thenReturn(aggregateDetail); // Execute & Verify CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, () -> storageStrategy.connect()); assertTrue(ex.getMessage().contains("No suitable aggregates found")); + boolean result = storageStrategy.connect(false); + + assertTrue(result); + // connect(true) called getAggregateByUUID once; connect(false) must not add more calls + verify(aggregateFeignClient, times(1)).getAggregateByUUID(anyString(), anyString(), anyMap()); } @Test @@ -354,7 +588,8 @@ public void testConnect_iscsiNotEnabled() { "svm1", 5000000000L, ProtocolType.ISCSI); storageStrategy = new TestableStorageStrategy(iscsiStorage, aggregateFeignClient, volumeFeignClient, svmFeignClient, - jobFeignClient, networkFeignClient, sanFeignClient); + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); Svm svm = new Svm(); svm.setName("svm1"); @@ -408,8 +643,10 @@ public void testConnect_nullSvmResponse() { @Test public void testConnect_invalidCredentials() { // Setup - ONTAP rejects the supplied username/password with HTTP 401 Unauthorized. + Map> emptyHeaders = Collections.emptyMap(); + Request dummyReq = Request.create(Request.HttpMethod.GET, "http://test", emptyHeaders, (byte[]) null, (Charset) null); when(svmFeignClient.getSvmResponse(anyMap(), anyString())) - .thenThrow(mock(FeignException.Unauthorized.class)); + .thenThrow(new FeignException.Unauthorized("Unauthorized", dummyReq, null)); // Execute & Verify - connect() must surface a clear "invalid credentials" error. CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, () -> storageStrategy.connect()); @@ -419,24 +656,117 @@ public void testConnect_invalidCredentials() { "Expected the message to prompt verifying username/password but got: " + ex.getMessage()); } + // ========== chooseAggregate() Tests ========== + + @Test + public void testChooseAggregate_positive() { + Aggregate aggregateDetail = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())) + .thenReturn(aggregateDetail); + + Aggregate result = storageStrategy.chooseAggregate(candidateAggregates(), 5000000000L); + + assertNotNull(result); + assertEquals("aggr1", result.getName()); + assertEquals("aggr-uuid-1", result.getUuid()); + assertEquals("node-a", result.getNode().getName()); + } + + @Test + public void testChooseAggregate_picksLargestAvailable() { + Aggregate candidate1 = new Aggregate(); + candidate1.setName("aggr1"); + candidate1.setUuid("aggr-uuid-1"); + Aggregate candidate2 = new Aggregate(); + candidate2.setName("aggr2"); + candidate2.setUuid("aggr-uuid-2"); + + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())) + .thenReturn(buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a")); + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-2"), anyMap())) + .thenReturn(buildAggregate("aggr2", "aggr-uuid-2", 20000000000.0, "node-b")); + + Aggregate result = storageStrategy.chooseAggregate(List.of(candidate1, candidate2), 5000000000L); + + assertEquals("aggr-uuid-2", result.getUuid()); + assertEquals("node-b", result.getNode().getName()); + } + + @Test + public void testChooseAggregate_invalidSize() { + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(candidateAggregates(), -1L)); + assertTrue(ex.getMessage().contains("Invalid volume size")); + } + + @Test + public void testChooseAggregate_nullSize() { + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(candidateAggregates(), null)); + assertTrue(ex.getMessage().contains("Invalid volume size")); + } + + @Test + public void testChooseAggregate_noAggregates() { + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(null, 5000000000L)); + assertTrue(ex.getMessage().contains("No aggregates available")); + + ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(List.of(), 5000000000L)); + assertTrue(ex.getMessage().contains("No aggregates available")); + } + + @Test + public void testChooseAggregate_aggregateNotOnline() { + Aggregate aggregateDetail = new Aggregate(); + aggregateDetail.setName("aggr1"); + aggregateDetail.setUuid("aggr-uuid-1"); + aggregateDetail.setState(null); + + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())) + .thenReturn(aggregateDetail); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(candidateAggregates(), 5000000000L)); + assertTrue(ex.getMessage().contains("No suitable aggregates found")); + } + + @Test + public void testChooseAggregate_insufficientSpace() { + Aggregate aggregateDetail = buildAggregate("aggr1", "aggr-uuid-1", 1000000.0, "node-a"); + + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())) + .thenReturn(aggregateDetail); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(candidateAggregates(), 5000000000L)); + assertTrue(ex.getMessage().contains("No suitable aggregates found")); + } + + @Test + public void testChooseAggregate_missingNode() { + Aggregate aggregateDetail = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0); + when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"), anyMap())) + .thenReturn(aggregateDetail); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.chooseAggregate(candidateAggregates(), 5000000000L)); + assertTrue(ex.getMessage().contains("does not have a node name")); + } + + private List candidateAggregates() { + Aggregate candidate = new Aggregate(); + candidate.setName("aggr1"); + candidate.setUuid("aggr-uuid-1"); + return List.of(candidate); + } + // ========== createStorageVolume() Tests ========== @Test public void testCreateStorageVolume_positive() { - // Setup - First connect to populate aggregates - setupSuccessfulConnect(); - storageStrategy.connect(); - - // Setup aggregate details - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(Aggregate.StateEnum.ONLINE); - when(aggregateDetail.getSpace()).thenReturn(mock(Aggregate.AggregateSpace.class)); // Mock non-null space - when(aggregateDetail.getAvailableBlockStorageSpace()).thenReturn(10000000000.0); - - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))) - .thenReturn(aggregateDetail); + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); // Setup job response Job job = new Job(); @@ -467,7 +797,7 @@ public void testCreateStorageVolume_positive() { .thenReturn(volumeResponse); // Execute - Volume result = storageStrategy.createStorageVolume("test-volume", 5000000000L); + Volume result = storageStrategy.createStorageVolume("test-volume", 5000000000L, aggregate); // Verify assertNotNull(result); @@ -479,84 +809,32 @@ public void testCreateStorageVolume_positive() { @Test public void testCreateStorageVolume_invalidSize() { - // Setup - setupSuccessfulConnect(); - storageStrategy.connect(); + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); - // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", -1L)); + () -> storageStrategy.createStorageVolume("test-volume", -1L, aggregate)); assertTrue(ex.getMessage().contains("Invalid volume size")); } @Test public void testCreateStorageVolume_nullSize() { - // Setup - setupSuccessfulConnect(); - storageStrategy.connect(); + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); - // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", null)); + () -> storageStrategy.createStorageVolume("test-volume", null, aggregate)); assertTrue(ex.getMessage().contains("Invalid volume size")); } @Test - public void testCreateStorageVolume_noAggregates() { - // Execute & Verify - without calling connect first + public void testCreateStorageVolume_nullAggregate() { Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", 5000000000L)); - assertTrue(ex.getMessage().contains("No aggregates available")); - } - - @Test - public void testCreateStorageVolume_aggregateNotOnline() { - // Setup - setupSuccessfulConnect(); - storageStrategy.connect(); - - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(null); // null state to simulate offline - - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))) - .thenReturn(aggregateDetail); - - // Execute & Verify - Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", 5000000000L)); - assertTrue(ex.getMessage().contains("No suitable aggregates found")); - } - - @Test - public void testCreateStorageVolume_insufficientSpace() { - // Setup - setupSuccessfulConnect(); - storageStrategy.connect(); - - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(Aggregate.StateEnum.ONLINE); - when(aggregateDetail.getAvailableBlockStorageSpace()).thenReturn(1000000.0); // Only 1MB available - - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))) - .thenReturn(aggregateDetail); - - // Execute & Verify - Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", 5000000000L)); // Request 5GB - assertTrue(ex.getMessage().contains("No suitable aggregates found")); + () -> storageStrategy.createStorageVolume("test-volume", 5000000000L, null)); + assertTrue(ex.getMessage().contains("Aggregate is required")); } @Test public void testCreateStorageVolume_jobFailed() { - // Setup - setupSuccessfulConnect(); - storageStrategy.connect(); - - setupAggregateForVolumeCreation(); + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); Job job = new Job(); job.setUuid("job-uuid-1"); @@ -574,18 +852,14 @@ public void testCreateStorageVolume_jobFailed() { when(jobFeignClient.getJobByUUID(anyString(), eq("job-uuid-1"))) .thenReturn(failedJob); - // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", 5000000000L)); + () -> storageStrategy.createStorageVolume("test-volume", 5000000000L, aggregate)); assertTrue(ex.getMessage().contains("failed") || ex.getMessage().contains("Job failed")); } @Test public void testCreateStorageVolume_volumeNotFoundAfterCreation() { - // Setup - setupSuccessfulConnect(); - storageStrategy.connect(); - setupAggregateForVolumeCreation(); + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); setupSuccessfulJobCreation(); // Setup empty volume response @@ -595,9 +869,8 @@ public void testCreateStorageVolume_volumeNotFoundAfterCreation() { when(volumeFeignClient.getAllVolumes(anyString(), anyMap())) .thenReturn(emptyResponse); - // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.createStorageVolume("test-volume", 5000000000L)); + () -> storageStrategy.createStorageVolume("test-volume", 5000000000L, aggregate)); assertTrue(ex.getMessage() != null && ex.getMessage().contains("not found after creation")); } @@ -667,8 +940,10 @@ public void testDeleteStorageVolume_feignException() { volume.setName("test-volume"); volume.setUuid("vol-uuid-1"); + Map> emptyHeaders = Collections.emptyMap(); + Request dummyReq = Request.create(Request.HttpMethod.DELETE, "http://test", emptyHeaders, (byte[]) null, (Charset) null); when(volumeFeignClient.deleteVolume(anyString(), eq("vol-uuid-1"))) - .thenThrow(mock(FeignException.FeignClientException.class)); + .thenThrow(new FeignException.FeignClientException(500, "error", dummyReq, null)); // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, @@ -676,6 +951,25 @@ public void testDeleteStorageVolume_feignException() { assertTrue(ex.getMessage().contains("Failed to delete volume")); } + @Test + public void testDeleteStorageVolume_notFound_404_returnsWithoutThrowing() { + // Setup + Volume volume = new Volume(); + volume.setName("test-volume"); + volume.setUuid("vol-uuid-1"); + + FeignException feignEx = mock(FeignException.class); + when(feignEx.status()).thenReturn(404); + when(volumeFeignClient.deleteVolume(anyString(), eq("vol-uuid-1"))) + .thenThrow(feignEx); + + // Execute - 404 means volume already gone on ONTAP, treated as no-op + storageStrategy.deleteStorageVolume(volume); + + // Verify the delete was attempted + verify(volumeFeignClient).deleteVolume(anyString(), eq("vol-uuid-1")); + } + // ========== getStoragePath() Tests ========== @Test @@ -685,7 +979,8 @@ public void testGetStoragePath_iscsi() { "svm1", null, ProtocolType.ISCSI); storageStrategy = new TestableStorageStrategy(iscsiStorage, aggregateFeignClient, volumeFeignClient, svmFeignClient, - jobFeignClient, networkFeignClient, sanFeignClient); + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); IscsiService.IscsiServiceTarget target = new IscsiService.IscsiServiceTarget(); target.setName("iqn.1992-08.com.netapp:sn.123456:vs.1"); @@ -715,7 +1010,8 @@ public void testGetStoragePath_iscsi_noService() { "svm1", null, ProtocolType.ISCSI); storageStrategy = new TestableStorageStrategy(iscsiStorage, aggregateFeignClient, volumeFeignClient, svmFeignClient, - jobFeignClient, networkFeignClient, sanFeignClient); + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); OntapResponse emptyResponse = new OntapResponse<>(); emptyResponse.setRecords(new ArrayList<>()); @@ -736,7 +1032,8 @@ public void testGetStoragePath_iscsi_noTargetIqn() { "svm1", null, ProtocolType.ISCSI); storageStrategy = new TestableStorageStrategy(iscsiStorage, aggregateFeignClient, volumeFeignClient, svmFeignClient, - jobFeignClient, networkFeignClient, sanFeignClient); + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); IscsiService iscsiService = new IscsiService(); iscsiService.setTarget(null); @@ -757,12 +1054,22 @@ public void testGetStoragePath_iscsi_noTargetIqn() { @Test public void testGetNetworkInterface_nfs() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + // Setup IpInterface.IpInfo ipInfo = new IpInterface.IpInfo(); ipInfo.setAddress("192.168.1.50"); IpInterface ipInterface = new IpInterface(); ipInterface.setIp(ipInfo); + ipInterface.setState(OntapStorageConstants.LIF_STATE_UP); + ipInterface.setEnabled(true); + IpInterface.Node homeNode = new IpInterface.Node(); + homeNode.setName("node-a"); + IpInterface.Location location = new IpInterface.Location(); + location.setHomeNode(homeNode); + location.setNode(homeNode); + ipInterface.setLocation(location); OntapResponse interfaceResponse = new OntapResponse<>(); interfaceResponse.setRecords(List.of(ipInterface)); @@ -771,11 +1078,13 @@ public void testGetNetworkInterface_nfs() { .thenReturn(interfaceResponse); // Execute - String result = storageStrategy.getNetworkInterface(); + Map result = storageStrategy.getNetworkInterface(aggregate); // Verify assertNotNull(result); - assertEquals("192.168.1.50", result); + assertEquals("192.168.1.50", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) == null, + "Expect no warning when a suitable LIF is found"); verify(networkFeignClient, times(1)).getNetworkIpInterfaces(anyString(), anyMap()); } @@ -786,13 +1095,24 @@ public void testGetNetworkInterface_iscsi() { "svm1", null, ProtocolType.ISCSI); storageStrategy = new TestableStorageStrategy(iscsiStorage, aggregateFeignClient, volumeFeignClient, svmFeignClient, - jobFeignClient, networkFeignClient, sanFeignClient); + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); + + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); IpInterface.IpInfo ipInfo = new IpInterface.IpInfo(); ipInfo.setAddress("192.168.1.51"); IpInterface ipInterface = new IpInterface(); ipInterface.setIp(ipInfo); + ipInterface.setState(OntapStorageConstants.LIF_STATE_UP); + ipInterface.setEnabled(true); + IpInterface.Node homeNode = new IpInterface.Node(); + homeNode.setName("node-a"); + IpInterface.Location location = new IpInterface.Location(); + location.setHomeNode(homeNode); + location.setNode(homeNode); + ipInterface.setLocation(location); OntapResponse interfaceResponse = new OntapResponse<>(); interfaceResponse.setRecords(List.of(ipInterface)); @@ -801,15 +1121,98 @@ public void testGetNetworkInterface_iscsi() { .thenReturn(interfaceResponse); // Execute - String result = storageStrategy.getNetworkInterface(); + Map result = storageStrategy.getNetworkInterface(aggregate); // Verify assertNotNull(result); - assertEquals("192.168.1.51", result); + assertEquals("192.168.1.51", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) == null, + "Expect no warning when a suitable LIF is found"); + } + + @Test + public void testGetNetworkInterface_nfs_lifDown() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + + // LIF exists but is operationally down — should fail + IpInterface.IpInfo ipInfo = new IpInterface.IpInfo(); + ipInfo.setAddress("192.168.1.50"); + + IpInterface ipInterface = new IpInterface(); + ipInterface.setIp(ipInfo); + ipInterface.setState("down"); + ipInterface.setEnabled(true); + + OntapResponse interfaceResponse = new OntapResponse<>(); + interfaceResponse.setRecords(List.of(ipInterface)); + + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(interfaceResponse); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.getNetworkInterface(aggregate)); + assertTrue(ex.getMessage().contains("operationally UP and enabled")); + } + + @Test + public void testGetNetworkInterface_nfs_lifDisabled() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + + // LIF exists but is administratively disabled — should fail + IpInterface.IpInfo ipInfo = new IpInterface.IpInfo(); + ipInfo.setAddress("192.168.1.50"); + + IpInterface ipInterface = new IpInterface(); + ipInterface.setIp(ipInfo); + ipInterface.setState(OntapStorageConstants.LIF_STATE_UP); + ipInterface.setEnabled(false); + + OntapResponse interfaceResponse = new OntapResponse<>(); + interfaceResponse.setRecords(List.of(ipInterface)); + + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(interfaceResponse); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.getNetworkInterface(aggregate)); + assertTrue(ex.getMessage().contains("operationally UP and enabled")); + } + + @Test + public void testGetNetworkInterface_iscsi_lifDown() { + // iSCSI LIF exists but is operationally down — should fail + OntapStorage iscsiStorage = new OntapStorage("admin", "password", "192.168.1.100", + "svm1", null, ProtocolType.ISCSI); + storageStrategy = new TestableStorageStrategy(iscsiStorage, + aggregateFeignClient, volumeFeignClient, svmFeignClient, + jobFeignClient, networkFeignClient, sanFeignClient, snapshotFeignClient, + clusterFeignClient); + + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + + IpInterface.IpInfo ipInfo = new IpInterface.IpInfo(); + ipInfo.setAddress("192.168.1.51"); + + IpInterface ipInterface = new IpInterface(); + ipInterface.setIp(ipInfo); + ipInterface.setState("down"); + ipInterface.setEnabled(true); + + OntapResponse interfaceResponse = new OntapResponse<>(); + interfaceResponse.setRecords(List.of(ipInterface)); + + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(interfaceResponse); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.getNetworkInterface(aggregate)); + assertTrue(ex.getMessage().contains("operationally UP and enabled")); } @Test public void testGetNetworkInterface_noInterfaces() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + // Setup OntapResponse emptyResponse = new OntapResponse<>(); emptyResponse.setRecords(new ArrayList<>()); @@ -819,61 +1222,153 @@ public void testGetNetworkInterface_noInterfaces() { // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.getNetworkInterface()); + () -> storageStrategy.getNetworkInterface(aggregate)); assertTrue(ex.getMessage().contains("No network interfaces found")); } @Test public void testGetNetworkInterface_feignException() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + // Setup + Map> emptyHeaders = Collections.emptyMap(); + Request dummyReq = Request.create(Request.HttpMethod.GET, "http://test", emptyHeaders, (byte[]) null, (Charset) null); when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) - .thenThrow(mock(FeignException.FeignClientException.class)); + .thenThrow(new FeignException.FeignClientException(500, "error", dummyReq, null)); // Execute & Verify Exception ex = assertThrows(CloudRuntimeException.class, - () -> storageStrategy.getNetworkInterface()); + () -> storageStrategy.getNetworkInterface(aggregate)); assertTrue(ex.getMessage().contains("Failed to retrieve network interfaces")); } - // ========== Helper Methods ========== + // ========== getNetworkInterface() Node-Affinity Tests ========== - private void setupSuccessfulConnect() { - Svm svm = new Svm(); - svm.setName("svm1"); - svm.setState(OntapStorageConstants.RUNNING); - svm.setNfsEnabled(true); + /** + * Tier 1: LIF homed on the same node as the chosen aggregate — selected without warning. + */ + @Test + public void testGetNetworkInterface_nfs_tier1_homeNodeMatch() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); - Aggregate aggregate = new Aggregate(); - aggregate.setName("aggr1"); - aggregate.setUuid("aggr-uuid-1"); - svm.setAggregates(List.of(aggregate)); + IpInterface lif = buildLif("10.0.0.1", OntapStorageConstants.LIF_STATE_UP, true, "node-a", "node-a"); + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(wrapLifs(List.of(lif))); - OntapResponse svmResponse = new OntapResponse<>(); - svmResponse.setRecords(List.of(svm)); + Map result = storageStrategy.getNetworkInterface(aggregate); - when(svmFeignClient.getSvmResponse(anyMap(), anyString())).thenReturn(svmResponse); + assertEquals("10.0.0.1", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) == null, "Tier 1 should produce no warning"); + } - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(Aggregate.StateEnum.ONLINE); - when(aggregateDetail.getSpace()).thenReturn(mock(Aggregate.AggregateSpace.class)); - when(aggregateDetail.getAvailableBlockStorageSpace()).thenReturn(10000000000.0); - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))).thenReturn(aggregateDetail); + /** + * Tier 2: No home-node match, but another UP LIF is currently running on the target node (failover). + * The result should carry a warning. + */ + @Test + public void testGetNetworkInterface_nfs_tier2_currentNodeMatch() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + + // home node = node-b, currently running on node-a after failover + IpInterface lif = buildLif("10.0.0.2", OntapStorageConstants.LIF_STATE_UP, true, "node-b", "node-a"); + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(wrapLifs(List.of(lif))); + + Map result = storageStrategy.getNetworkInterface(aggregate); + + assertEquals("10.0.0.2", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) != null, "Tier 2 should produce a warning"); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING).contains("node-a")); } - private void setupAggregateForVolumeCreation() { - Aggregate aggregateDetail = mock(Aggregate.class); - when(aggregateDetail.getName()).thenReturn("aggr1"); - when(aggregateDetail.getUuid()).thenReturn("aggr-uuid-1"); - when(aggregateDetail.getState()).thenReturn(Aggregate.StateEnum.ONLINE); - when(aggregateDetail.getSpace()).thenReturn(mock(Aggregate.AggregateSpace.class)); // Mock non-null space - when(aggregateDetail.getAvailableBlockStorageSpace()).thenReturn(10000000000.0); + /** + * Tier 3 fallback: No LIF matches the target node in either home_node or current node. + * First UP/enabled LIF used; result carries a warning directing the user to create a + * LIF on the correct node. + */ + @Test + public void testGetNetworkInterface_nfs_tier3_crossNodeFallback() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); - when(aggregateFeignClient.getAggregateByUUID(anyString(), eq("aggr-uuid-1"))) - .thenReturn(aggregateDetail); + // Both home_node and current node are node-b — no affinity to node-a + IpInterface lif = buildLif("10.0.0.3", OntapStorageConstants.LIF_STATE_UP, true, "node-b", "node-b"); + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(wrapLifs(List.of(lif))); + + Map result = storageStrategy.getNetworkInterface(aggregate); + + assertEquals("10.0.0.3", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) != null, "Tier 3 fallback should produce a warning"); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING).contains("node-a"), + "Warning should mention the expected node"); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING).contains("10.0.0.3"), + "Warning should mention the fallback LIF IP"); + } + + /** + * Null aggregate or missing node fails clearly — no silent unaffined LIF selection. + */ + @Test + public void testGetNetworkInterface_nullAggregate_fails() { + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.getNetworkInterface(null)); + assertTrue(ex.getMessage().contains("Aggregate with a node name is required")); + } + + @Test + public void testGetNetworkInterface_missingNode_fails() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0); + + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.getNetworkInterface(aggregate)); + assertTrue(ex.getMessage().contains("Aggregate with a node name is required")); + } + + /** + * Tier-1 LIF is down; Tier-2 LIF matches the current node and should be selected with a warning. + */ + @Test + public void testGetNetworkInterface_nfs_tier1Down_tier2Used() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + + // Tier 1 candidate: home_node = node-a but operationally DOWN + IpInterface lifDown = buildLif("10.0.0.5", "down", true, "node-a", "node-a"); + // Tier 2 candidate: home_node = node-b, currently on node-a + IpInterface lifFailover = buildLif("10.0.0.6", OntapStorageConstants.LIF_STATE_UP, true, "node-b", "node-a"); + + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(wrapLifs(List.of(lifDown, lifFailover))); + + Map result = storageStrategy.getNetworkInterface(aggregate); + + assertEquals("10.0.0.6", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) != null, + "Should warn that the home-node LIF is not in use"); } + /** + * A LIF with no ip object is skipped. A later valid LIF on the same node is still selected. + */ + @Test + public void testGetNetworkInterface_nfs_nullIpSkipped() { + Aggregate aggregate = buildAggregate("aggr1", "aggr-uuid-1", 10000000000.0, "node-a"); + + IpInterface lifWithoutIp = buildLif("10.0.0.1", OntapStorageConstants.LIF_STATE_UP, true, "node-a", "node-a"); + lifWithoutIp.setIp(null); + IpInterface lifWithIp = buildLif("10.0.0.2", OntapStorageConstants.LIF_STATE_UP, true, "node-a", "node-a"); + + when(networkFeignClient.getNetworkIpInterfaces(anyString(), anyMap())) + .thenReturn(wrapLifs(List.of(lifWithoutIp, lifWithIp))); + + Map result = storageStrategy.getNetworkInterface(aggregate); + + assertEquals("10.0.0.2", result.get(OntapStorageConstants.DATA_LIF)); + assertTrue(result.get(OntapStorageConstants.LIF_WARNING) == null, + "The valid same-node LIF should be selected with no warning"); + } + + // ========== Helper Methods ========== + private void setupSuccessfulJobCreation() { Job job = new Job(); job.setUuid("job-uuid-1"); @@ -900,4 +1395,318 @@ private void setupSuccessfulJobCreation() { when(volumeFeignClient.getVolume(anyString(), anyMap())) .thenReturn(volumeResponse); } + + /** + * Builds an {@link IpInterface} with all node-affinity fields populated. + * + * @param ip the LIF's IP address (IPv4 for NFS3 selection to work) + * @param state operational state (e.g. "up" or "down") + * @param enabled administrative state + * @param homeNode name of the node the LIF is homed to + * @param currentNode name of the node the LIF is currently running on + */ + private static IpInterface buildLif(String ip, String state, boolean enabled, + String homeNode, String currentNode) { + IpInterface.IpInfo ipInfo = new IpInterface.IpInfo(); + ipInfo.setAddress(ip); + + IpInterface.Node homeNodeObj = new IpInterface.Node(); + homeNodeObj.setName(homeNode); + + IpInterface.Node currentNodeObj = new IpInterface.Node(); + currentNodeObj.setName(currentNode); + + IpInterface.Location location = new IpInterface.Location(); + location.setHomeNode(homeNodeObj); + location.setNode(currentNodeObj); + + IpInterface lif = new IpInterface(); + lif.setIp(ipInfo); + lif.setState(state); + lif.setEnabled(enabled); + lif.setLocation(location); + return lif; + } + + private static OntapResponse wrapLifs(List lifs) { + OntapResponse response = new OntapResponse<>(); + response.setRecords(lifs); + return response; + } + + /** + * Creates a real {@link Aggregate} with nested space information so tests can avoid + * {@code mock(Aggregate.class)} which fails on JDK 26+ due to Byte Buddy limitations. + */ + private static Aggregate buildAggregate(String name, String uuid, double availableBytes) { + return buildAggregate(name, uuid, availableBytes, null); + } + + private static Aggregate buildAggregate(String name, String uuid, double availableBytes, String nodeName) { + Aggregate.AggregateSpaceBlockStorage blockStorage = new Aggregate.AggregateSpaceBlockStorage(); + blockStorage.setAvailable(availableBytes); + + Aggregate.AggregateSpace space = new Aggregate.AggregateSpace(); + space.setBlockStorage(blockStorage); + + Aggregate agg = new Aggregate(); + agg.setName(name); + agg.setUuid(uuid); + agg.setState(Aggregate.StateEnum.ONLINE); + agg.setSpace(space); + if (nodeName != null) { + Aggregate.Node node = new Aggregate.Node(); + node.setName(nodeName); + agg.setNode(node); + } + return agg; + } + + // ========== pollJobIfPresent / executeCliSfsrRestore Tests ========== + + @Test + void testPollJobIfPresent_NoJob_DoesNotPoll() { + storageStrategy.pollJobIfPresent(null, "test operation"); + storageStrategy.pollJobIfPresent(new JobResponse(), "test operation"); + verify(jobFeignClient, times(0)).getJobByUUID(anyString(), anyString()); + } + + @Test + void testPollJobIfPresent_WithJob_PollsUntilSuccess() { + Job job = new Job(); + job.setUuid("sfsr-job-1"); + JobResponse response = new JobResponse(); + response.setJob(job); + + Job completedJob = new Job(); + completedJob.setUuid("sfsr-job-1"); + completedJob.setState(OntapStorageConstants.JOB_SUCCESS); + when(jobFeignClient.getJobByUUID(anyString(), eq("sfsr-job-1"))).thenReturn(completedJob); + + storageStrategy.executeCliSfsrRestore(response, "CLI SFSR restore"); + + verify(jobFeignClient, atLeastOnce()).getJobByUUID(anyString(), eq("sfsr-job-1")); + } + + @Test + void testPollJobIfPresent_JobFailure_ThrowsCloudRuntimeException() { + Job job = new Job(); + job.setUuid("sfsr-job-fail"); + JobResponse response = new JobResponse(); + response.setJob(job); + + Job failedJob = new Job(); + failedJob.setUuid("sfsr-job-fail"); + failedJob.setState(OntapStorageConstants.JOB_FAILURE); + failedJob.setMessage("restore failed"); + when(jobFeignClient.getJobByUUID(anyString(), eq("sfsr-job-fail"))).thenReturn(failedJob); + + assertThrows(CloudRuntimeException.class, + () -> storageStrategy.executeCliSfsrRestore(response, "CLI SFSR restore")); + } + + @Test + void testDeleteFlexVolSnapshotForCloudStackVolume_PollsJobAndSucceeds() { + Job job = new Job(); + job.setUuid("delete-job-1"); + JobResponse response = new JobResponse(); + response.setJob(job); + when(snapshotFeignClient.deleteSnapshot(anyString(), eq("fv-uuid-1"), eq("snap-uuid-1"))) + .thenReturn(response); + + Job completedJob = new Job(); + completedJob.setUuid("delete-job-1"); + completedJob.setState(OntapStorageConstants.JOB_SUCCESS); + when(jobFeignClient.getJobByUUID(anyString(), eq("delete-job-1"))).thenReturn(completedJob); + + storageStrategy.deleteFlexVolSnapshotForCloudStackVolume("fv-uuid-1", "snap-uuid-1", "snap-name-1"); + + verify(snapshotFeignClient).deleteSnapshot(anyString(), eq("fv-uuid-1"), eq("snap-uuid-1")); + } + + @Test + void testDeleteFlexVolSnapshotForCloudStackVolume_AlreadyAbsentOnOntap() { + Job job = new Job(); + job.setUuid("delete-job-missing"); + JobResponse response = new JobResponse(); + response.setJob(job); + when(snapshotFeignClient.deleteSnapshot(anyString(), eq("fv-uuid-1"), eq("snap-uuid-1"))) + .thenReturn(response); + + Job failedJob = new Job(); + failedJob.setUuid("delete-job-missing"); + failedJob.setState(OntapStorageConstants.JOB_FAILURE); + failedJob.setMessage("entry doesn't exist"); + when(jobFeignClient.getJobByUUID(anyString(), eq("delete-job-missing"))).thenReturn(failedJob); + + storageStrategy.deleteFlexVolSnapshotForCloudStackVolume("fv-uuid-1", "snap-uuid-1", "snap-name-1"); + + verify(snapshotFeignClient).deleteSnapshot(anyString(), eq("fv-uuid-1"), eq("snap-uuid-1")); + } + + @Test + void testDeleteFlexVolSnapshotForCloudStackVolume_Feign404_TreatedAsSuccess() { + FeignException notFoundException = mock(FeignException.class); + when(notFoundException.status()).thenReturn(404); + when(snapshotFeignClient.deleteSnapshot(anyString(), eq("fv-uuid-1"), eq("snap-uuid-1"))) + .thenThrow(notFoundException); + + storageStrategy.deleteFlexVolSnapshotForCloudStackVolume("fv-uuid-1", "snap-uuid-1", "snap-name-1"); + + verify(snapshotFeignClient).deleteSnapshot(anyString(), eq("fv-uuid-1"), eq("snap-uuid-1")); + verify(jobFeignClient, never()).getJobByUUID(anyString(), anyString()); + } + + @Test + void testGetStorageVolume_ByUuid_Success() { + Volume expected = new Volume(); + expected.setUuid("flexvol-uuid-123"); + Map queryParams = Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.SPACE_USED); + when(volumeFeignClient.getVolumeByUUID(anyString(), eq("flexvol-uuid-123"), eq(queryParams))).thenReturn(expected); + + Volume result = storageStrategy.getStorageVolume("flexvol-uuid-123", queryParams); + + assertNotNull(result); + assertEquals("flexvol-uuid-123", result.getUuid()); + verify(volumeFeignClient).getVolumeByUUID(anyString(), eq("flexvol-uuid-123"), eq(queryParams)); + } + + @Test + void testGetStorageVolume_ByUuid_NotFound_ReturnsNull() { + FeignException notFound = mock(FeignException.class); + when(notFound.status()).thenReturn(404); + Map queryParams = Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.SPACE_USED); + doThrow(notFound).when(volumeFeignClient).getVolumeByUUID(anyString(), eq("missing-uuid"), eq(queryParams)); + + Volume result = storageStrategy.getStorageVolume("missing-uuid", queryParams); + + assertNull(result); + } + + @Test + void testGetStorageVolume_ByUuid_ServerError_Throws() { + FeignException serverError = mock(FeignException.class); + when(serverError.status()).thenReturn(500); + when(serverError.getMessage()).thenReturn("Internal Server Error"); + Map queryParams = Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.SPACE_USED); + doThrow(serverError).when(volumeFeignClient).getVolumeByUUID(anyString(), eq("flexvol-uuid-999"), eq(queryParams)); + + assertThrows(CloudRuntimeException.class, () -> storageStrategy.getStorageVolume("flexvol-uuid-999", queryParams)); + } + + @Test + void testGetStorageVolume_ByUuid_ForwardsQueryParams() { + Volume expected = new Volume(); + expected.setUuid("flexvol-uuid-123"); + Map queryParams = Map.of(OntapStorageConstants.FIELDS, OntapStorageConstants.SPACE_USED); + when(volumeFeignClient.getVolumeByUUID(anyString(), eq("flexvol-uuid-123"), eq(queryParams))).thenReturn(expected); + + Volume result = storageStrategy.getStorageVolume("flexvol-uuid-123", queryParams); + + assertNotNull(result); + assertEquals("flexvol-uuid-123", result.getUuid()); + verify(volumeFeignClient).getVolumeByUUID(anyString(), eq("flexvol-uuid-123"), eq(queryParams)); + } + + // ========== updateStorageVolume() Tests ========== + + @Test + public void testUpdateStorageVolume_positive() { + // Setup + Volume volume = new Volume(); + volume.setUuid("vol-uuid-resize"); + volume.setName("flexvol-resize"); + volume.setSize(5368709120L); // 5 GB + + Job job = new Job(); + job.setUuid("resize-job-uuid"); + JobResponse jobResponse = new JobResponse(); + jobResponse.setJob(job); + + when(volumeFeignClient.updateVolume(anyString(), eq("vol-uuid-resize"), any())) + .thenReturn(jobResponse); + + Job completedJob = new Job(); + completedJob.setUuid("resize-job-uuid"); + completedJob.setState(OntapStorageConstants.JOB_SUCCESS); + when(jobFeignClient.getJobByUUID(anyString(), eq("resize-job-uuid"))) + .thenReturn(completedJob); + + // Execute + Volume result = storageStrategy.updateStorageVolume(volume); + + // Verify + assertSame(volume, result); + assertEquals(5368709120L, result.getSize()); + verify(volumeFeignClient, times(1)).updateVolume(anyString(), eq("vol-uuid-resize"), any()); + verify(volumeFeignClient, never()).getVolumeByUUID(anyString(), anyString(), any()); + verify(jobFeignClient, atLeastOnce()).getJobByUUID(anyString(), eq("resize-job-uuid")); + } + + @Test + public void testUpdateStorageVolume_jobFailed() { + // Setup + Volume volume = new Volume(); + volume.setUuid("vol-uuid-resize"); + volume.setName("flexvol-resize"); + volume.setSize(5368709120L); + + Job job = new Job(); + job.setUuid("resize-job-uuid"); + JobResponse jobResponse = new JobResponse(); + jobResponse.setJob(job); + + when(volumeFeignClient.updateVolume(anyString(), eq("vol-uuid-resize"), any())) + .thenReturn(jobResponse); + + Job failedJob = new Job(); + failedJob.setUuid("resize-job-uuid"); + failedJob.setState(OntapStorageConstants.JOB_FAILURE); + failedJob.setMessage("Resize failed"); + when(jobFeignClient.getJobByUUID(anyString(), eq("resize-job-uuid"))) + .thenReturn(failedJob); + + // Execute & Verify + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.updateStorageVolume(volume)); + assertTrue(ex.getMessage().contains("Job failed")); + } + + @Test + public void testUpdateStorageVolume_feignException() { + // Setup + Volume volume = new Volume(); + volume.setUuid("vol-uuid-fail"); + volume.setName("flexvol-fail"); + volume.setSize(3221225472L); + + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(500); + when(volumeFeignClient.updateVolume(anyString(), eq("vol-uuid-fail"), any())) + .thenThrow(feignException); + + // Execute & Verify + Exception ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.updateStorageVolume(volume)); + assertTrue(ex.getMessage().contains("Failed to resize ONTAP FlexVolume")); + } + + @Test + public void testUpdateStorageVolume_notFound_404_throwsCloudRuntimeException() { + // Setup + Volume volume = new Volume(); + volume.setUuid("vol-uuid-notfound"); + volume.setName("flexvol-notfound"); + volume.setSize(1073741824L); + + FeignException feignEx = mock(FeignException.class); + when(feignEx.status()).thenReturn(404); + when(volumeFeignClient.updateVolume(anyString(), eq("vol-uuid-notfound"), any())) + .thenThrow(feignEx); + + // Execute & Verify — 404 means volume not found on ONTAP, should throw + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> storageStrategy.updateStorageVolume(volume)); + assertTrue(ex.getMessage().contains("not found on ONTAP")); + } } diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedNASStrategyTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedNASStrategyTest.java index c4d5ddf6878c..60bdf64855e3 100755 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedNASStrategyTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedNASStrategyTest.java @@ -20,7 +20,9 @@ package org.apache.cloudstack.storage.service; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.host.HostVO; +import com.cloud.storage.ResizeVolumePayload; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.exception.CloudRuntimeException; @@ -28,7 +30,10 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.command.CreateObjectCommand; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.feign.client.JobFeignClient; import org.apache.cloudstack.storage.feign.client.NASFeignClient; import org.apache.cloudstack.storage.feign.client.VolumeFeignClient; @@ -37,8 +42,12 @@ import org.apache.cloudstack.storage.feign.client.NetworkFeignClient; import org.apache.cloudstack.storage.feign.client.SANFeignClient; import org.apache.cloudstack.storage.feign.model.ExportPolicy; +import org.apache.cloudstack.storage.feign.model.ExportRule; +import org.apache.cloudstack.storage.feign.model.FileCloneRequest; +import org.apache.cloudstack.storage.feign.model.FileInfo; import org.apache.cloudstack.storage.feign.model.Job; import org.apache.cloudstack.storage.feign.model.OntapStorage; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import org.apache.cloudstack.storage.service.model.AccessGroup; @@ -63,19 +72,26 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import feign.FeignException; + @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class UnifiedNASStrategyTest { @@ -110,6 +126,9 @@ public class UnifiedNASStrategyTest { @Mock private StoragePoolDetailsDao storagePoolDetailsDao; + @Mock + private PrimaryDataStoreDao primaryDataStoreDao; + private TestableUnifiedNASStrategy strategy; private OntapStorage ontapStorage; @@ -128,6 +147,7 @@ public void setUp() throws Exception { injectField("volumeDao", volumeDao); injectField("epSelector", epSelector); injectField("storagePoolDetailsDao", storagePoolDetailsDao); + injectField("primaryDataStoreDao", primaryDataStoreDao); } private void injectField(String fieldName, Object mockedField) throws Exception { @@ -196,6 +216,177 @@ public void testCreateCloudStackVolume_Success() throws Exception { verify(endPoint).sendMessage(any(CreateObjectCommand.class)); } + @Test + public void testCreateTemplateCache_IsNoOp() { + org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool = + mock(org.apache.cloudstack.storage.datastore.db.StoragePoolVO.class); + when(storagePool.getId()).thenReturn(1L); + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo = + mock(org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo.class); + when(templateInfo.getId()).thenReturn(50L); + + CloudStackVolume result = strategy.createTemplateCache(storagePool, templateInfo, Map.of(), 0L); + + // NFS seeds via host CopyCommand; strategy intentionally returns null (no LUN/file yet). + assertNull(result); + } + + + @Test + public void testCreateCloudStackVolume_AppliesQosPolicyToNfsFile() throws Exception { + CloudStackVolume cloudStackVolume = mock(CloudStackVolume.class); + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + EndPoint endPoint = mock(EndPoint.class); + Answer answer = new Answer(null, true, "Success"); + + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_100_to200_iops_svm1"); + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicy); + + when(cloudStackVolume.getDatastoreId()).thenReturn("1"); + when(cloudStackVolume.getVolumeInfo()).thenReturn(volumeObject); + when(cloudStackVolume.getFlexVolumeUuid()).thenReturn("flex-uuid"); + when(cloudStackVolume.getFile()).thenReturn(fileInfo); + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid-123"); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeDao.update(anyLong(), any(VolumeVO.class))).thenReturn(true); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(CreateObjectCommand.class))).thenReturn(answer); + + CloudStackVolume result = strategy.createCloudStackVolume(cloudStackVolume); + + assertNotNull(result); + verify(nasFeignClient).updateFile(anyString(), eq("flex-uuid"), eq("volume-uuid-123"), + argThat(file -> file.getQosPolicy() != null + && "cs_100_to200_iops_svm1".equals(file.getQosPolicy().getName()))); + } + + @Test + public void testCreateCloudStackVolume_QosAttachFails_DeletesLeftoverNfsFile() { + CloudStackVolume cloudStackVolume = mock(CloudStackVolume.class); + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + EndPoint endPoint = mock(EndPoint.class); + Answer createAnswer = new Answer(null, true, "Success"); + Answer deleteAnswer = new Answer(null, true, "Deleted"); + + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_100_to200_iops_svm1"); + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicy); + + when(cloudStackVolume.getDatastoreId()).thenReturn("1"); + when(cloudStackVolume.getVolumeInfo()).thenReturn(volumeObject); + when(cloudStackVolume.getFlexVolumeUuid()).thenReturn("flex-uuid"); + when(cloudStackVolume.getFile()).thenReturn(fileInfo); + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid-123"); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeDao.update(anyLong(), any(VolumeVO.class))).thenReturn(true); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(CreateObjectCommand.class))).thenReturn(createAnswer); + when(endPoint.sendMessage(any(DeleteCommand.class))).thenReturn(deleteAnswer); + + FeignException feignException = mock(FeignException.class); + when(feignException.contentUTF8()).thenReturn( + "{\"error\":{\"code\":\"8454269\",\"message\":\"Invalid QoS policy group specified\"}}"); + when(feignException.getMessage()).thenReturn("Bad Request"); + doThrow(feignException).when(nasFeignClient).updateFile(anyString(), eq("flex-uuid"), + eq("volume-uuid-123"), any(FileInfo.class)); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> strategy.createCloudStackVolume(cloudStackVolume)); + assertTrue(ex.getMessage().contains("Failed to update NFS volume file")); + verify(endPoint).sendMessage(any(DeleteCommand.class)); + } + + @Test + public void testUpdateCloudStackVolume_AppliesQosPolicy() { + VolumeInfo volumeInfo = mock(VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn("volume-uuid-123"); + + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_100_to_200_iops_svm1"); + qosPolicy.setUuid("qos-uuid"); + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicy); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeInfo); + request.setFlexVolumeUuid("flex-uuid"); + request.setFile(fileInfo); + + CloudStackVolume result = strategy.updateCloudStackVolume(request); + + assertSame(request, result); + verify(nasFeignClient).updateFile(anyString(), eq("flex-uuid"), eq("volume-uuid-123"), + argThat(file -> file.getQosPolicy() != null + && "cs_100_to_200_iops_svm1".equals(file.getQosPolicy().getName()) + && "qos-uuid".equals(file.getQosPolicy().getUuid()))); + } + + @Test + public void testUpdateCloudStackVolume_UnexpectedException_IsWrapped() { + VolumeInfo volumeInfo = mock(VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn("volume-uuid-123"); + + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_100_to_200_iops_svm1"); + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicy); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeInfo); + request.setFlexVolumeUuid("flex-uuid"); + request.setFile(fileInfo); + + doThrow(new IllegalStateException("decoder failed")).when(nasFeignClient) + .updateFile(anyString(), eq("flex-uuid"), eq("volume-uuid-123"), any(FileInfo.class)); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> strategy.updateCloudStackVolume(request)); + assertTrue(ex.getMessage().contains("Failed to update NFS volume file")); + assertTrue(ex.getCause() instanceof IllegalStateException); + } + + @Test + public void testUpdateCloudStackVolume_NoQosPolicy_SkipsPatch() { + VolumeInfo volumeInfo = mock(VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn("volume-uuid-123"); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeInfo); + request.setFlexVolumeUuid("flex-uuid"); + request.setFile(new FileInfo()); + + CloudStackVolume result = strategy.updateCloudStackVolume(request); + + assertSame(request, result); + verify(nasFeignClient, never()).updateFile(anyString(), anyString(), anyString(), any(FileInfo.class)); + } + + @Test + public void testUpdateCloudStackVolume_InvalidRequest_ThrowsException() { + CloudStackVolume missingFile = new CloudStackVolume(); + missingFile.setVolumeInfo(mock(VolumeInfo.class)); + missingFile.setFlexVolumeUuid("flex-uuid"); + + assertInvalidNfsVolumeUpdate(null); + assertInvalidNfsVolumeUpdate(new CloudStackVolume()); + assertInvalidNfsVolumeUpdate(missingFile); + + verify(nasFeignClient, never()).updateFile(anyString(), anyString(), anyString(), any(FileInfo.class)); + } + + private void assertInvalidNfsVolumeUpdate(CloudStackVolume request) { + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> strategy.updateCloudStackVolume(request)); + assertEquals("Invalid NFS volume request", ex.getMessage()); + } + // Test createCloudStackVolume - Volume Not Found @Test public void testCreateCloudStackVolume_VolumeNotFound() { @@ -291,7 +482,7 @@ public void testCreateAccessGroup_Success() throws Exception { when(accessGroup.getHostsToConnect()).thenReturn(hosts); doNothing().when(nasFeignClient).createExportPolicy(anyString(), any(ExportPolicy.class)); when(nasFeignClient.getExportPolicyResponse(anyString(), anyMap())).thenReturn(policyResponse); - when(volumeFeignClient.updateVolumeRebalancing(anyString(), anyString(), any())).thenReturn(jobResponse); + when(volumeFeignClient.updateVolume(anyString(), anyString(), any())).thenReturn(jobResponse); when(jobFeignClient.getJobByUUID(anyString(), anyString())).thenReturn(job); doNothing().when(storagePoolDetailsDao).addDetail(anyLong(), anyString(), anyString(), eq(true)); @@ -302,7 +493,7 @@ public void testCreateAccessGroup_Success() throws Exception { assertNotNull(result); verify(nasFeignClient).createExportPolicy(anyString(), any(ExportPolicy.class)); verify(nasFeignClient).getExportPolicyResponse(anyString(), anyMap()); - verify(volumeFeignClient).updateVolumeRebalancing(anyString(), eq("vol-uuid-123"), any()); + verify(volumeFeignClient).updateVolume(anyString(), eq("vol-uuid-123"), any()); verify(storagePoolDetailsDao, times(2)).addDetail(anyLong(), anyString(), anyString(), eq(true)); } @@ -397,7 +588,7 @@ public void testCreateAccessGroup_JobFailure() throws Exception { when(accessGroup.getHostsToConnect()).thenReturn(hosts); doNothing().when(nasFeignClient).createExportPolicy(anyString(), any(ExportPolicy.class)); when(nasFeignClient.getExportPolicyResponse(anyString(), anyMap())).thenReturn(policyResponse); - when(volumeFeignClient.updateVolumeRebalancing(anyString(), anyString(), any())).thenReturn(jobResponse); + when(volumeFeignClient.updateVolume(anyString(), anyString(), any())).thenReturn(jobResponse); when(jobFeignClient.getJobByUUID(anyString(), anyString())).thenReturn(job); assertThrows(CloudRuntimeException.class, () -> { @@ -441,7 +632,7 @@ public void testCreateAccessGroup_HostWithPrivateIP() throws Exception { when(accessGroup.getHostsToConnect()).thenReturn(hosts); doNothing().when(nasFeignClient).createExportPolicy(anyString(), any(ExportPolicy.class)); when(nasFeignClient.getExportPolicyResponse(anyString(), anyMap())).thenReturn(policyResponse); - when(volumeFeignClient.updateVolumeRebalancing(anyString(), anyString(), any())).thenReturn(jobResponse); + when(volumeFeignClient.updateVolume(anyString(), anyString(), any())).thenReturn(jobResponse); when(jobFeignClient.getJobByUUID(anyString(), anyString())).thenReturn(job); doNothing().when(storagePoolDetailsDao).addDetail(anyLong(), anyString(), anyString(), eq(true)); @@ -513,6 +704,26 @@ public void testDeleteAccessGroup_Failed() { }); } + // Test deleteAccessGroup - Export policy not found should be treated as no-op + @Test + public void testDeleteAccessGroup_NotFound404_NoThrow() { + AccessGroup accessGroup = mock(AccessGroup.class); + Map details = new HashMap<>(); + details.put(OntapStorageConstants.EXPORT_POLICY_NAME, "export-policy-1"); + details.put(OntapStorageConstants.EXPORT_POLICY_ID, "1"); + + when(accessGroup.getStoragePoolId()).thenReturn(1L); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(details); + + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(404); + doThrow(feignException).when(nasFeignClient).deleteExportPolicyById(anyString(), eq("1")); + + strategy.deleteAccessGroup(accessGroup); + + verify(nasFeignClient).deleteExportPolicyById(anyString(), eq("1")); + } + // Test deleteCloudStackVolume - Success @Test public void testDeleteCloudStackVolume_Success() throws Exception { @@ -582,4 +793,801 @@ public void testDeleteCloudStackVolume_AnswerNull() throws Exception { strategy.deleteCloudStackVolume(cloudStackVolume); }); } + + // ------------------------------------------------------------------------- + // updateAccessGroup tests + // ------------------------------------------------------------------------- + + private Map detailsWithExportPolicyId() { + Map details = new HashMap<>(); + details.put(OntapStorageConstants.EXPORT_POLICY_ID, "policy-42"); + return details; + } + + private ExportPolicy existingPolicyWithClients(String... matchIps) { + ExportRule rule = new ExportRule(); + List clients = new ArrayList<>(); + for (String ip : matchIps) { + ExportRule.ExportClient client = new ExportRule.ExportClient(); + client.setMatch(ip); + clients.add(client); + } + rule.setClients(clients); + ExportPolicy policy = new ExportPolicy(); + policy.setName("test-policy"); + policy.setRules(new ArrayList<>(List.of(rule))); + return policy; + } + + // updateAccessGroup - null accessGroup + @Test + public void testUpdateAccessGroup_NullAccessGroup() { + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(null)); + } + + // updateAccessGroup - null storagePoolId + @Test + public void testUpdateAccessGroup_NullStoragePoolId() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setHostsToConnect(List.of(mock(HostVO.class))); + // storagePoolId is null by default + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - null hostsToConnect + @Test + public void testUpdateAccessGroup_NullHostsToConnect() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + // hostsToConnect is null by default + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - empty hostsToConnect + @Test + public void testUpdateAccessGroup_EmptyHostsToConnect() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(new ArrayList<>()); + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - storagePoolDetailsDao returns null + @Test + public void testUpdateAccessGroup_NoStoragePoolDetails() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(mock(HostVO.class))); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(null); + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - details missing EXPORT_POLICY_ID key + @Test + public void testUpdateAccessGroup_MissingExportPolicyId() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(mock(HostVO.class))); + Map details = new HashMap<>(); + details.put("someOtherKey", "someValue"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(details); + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - getExportPolicyById returns null + @Test + public void testUpdateAccessGroup_ExportPolicyNotFound() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(mock(HostVO.class))); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(null); + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - existing policy has null rules + @Test + public void testUpdateAccessGroup_NullRules() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(mock(HostVO.class))); + ExportPolicy policy = new ExportPolicy(); + policy.setName("test-policy"); + policy.setRules(null); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(policy); + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - existing policy has empty rules list + @Test + public void testUpdateAccessGroup_EmptyRules() { + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(mock(HostVO.class))); + ExportPolicy policy = new ExportPolicy(); + policy.setName("test-policy"); + policy.setRules(new ArrayList<>()); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(policy); + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + + // updateAccessGroup - all hosts have no IP: returns early without ONTAP patch + @Test + public void testUpdateAccessGroup_AllHostsHaveNoIp_ReturnsEarly() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn(null); + when(host.getPrivateIpAddress()).thenReturn(null); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + ExportPolicy existingPolicy = existingPolicyWithClients("10.0.0.1/32"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + assertSame(existingPolicy, result.getPolicy()); + verify(nasFeignClient, never()).updateExportPolicy(anyString(), anyString(), any()); + } + + // updateAccessGroup - ADD: new host IP added to policy + @Test + public void testUpdateAccessGroup_Add_NewHost_Success() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn("10.0.0.2"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + // default action is ADD + + ExportPolicy existingPolicy = existingPolicyWithClients("10.0.0.1/32"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + assertSame(existingPolicy, result.getPolicy()); + // Existing client + new client = 2 + assertEquals(2, existingPolicy.getRules().get(0).getClients().size()); + verify(nasFeignClient).updateExportPolicy(anyString(), eq("policy-42"), any(ExportPolicy.class)); + } + + // updateAccessGroup - ADD: host uses private IP when storage IP is absent + @Test + public void testUpdateAccessGroup_Add_UsesPrivateIpWhenStorageIpAbsent() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn(null); + when(host.getPrivateIpAddress()).thenReturn("192.168.1.50"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + ExportPolicy existingPolicy = existingPolicyWithClients(); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + List clients = existingPolicy.getRules().get(0).getClients(); + assertEquals(1, clients.size()); + assertEquals("192.168.1.50/32", clients.get(0).getMatch()); + verify(nasFeignClient).updateExportPolicy(anyString(), eq("policy-42"), any(ExportPolicy.class)); + } + + // updateAccessGroup - ADD: host IP already present in policy (no-op) + @Test + public void testUpdateAccessGroup_Add_DuplicateHost_NoUpdate() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn("10.0.0.1"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + ExportPolicy existingPolicy = existingPolicyWithClients("10.0.0.1/32"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + assertSame(existingPolicy, result.getPolicy()); + // Client count must remain 1 (no duplicate inserted) + assertEquals(1, existingPolicy.getRules().get(0).getClients().size()); + verify(nasFeignClient, never()).updateExportPolicy(anyString(), anyString(), any()); + } + + // updateAccessGroup - ADD: existing rule has null clients list + @Test + public void testUpdateAccessGroup_Add_NullClientsInRule() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn("10.0.0.5"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + ExportRule rule = new ExportRule(); + rule.setClients(null); // null clients list + ExportPolicy policy = new ExportPolicy(); + policy.setName("test-policy"); + policy.setRules(new ArrayList<>(List.of(rule))); + + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(policy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + assertEquals(1, rule.getClients().size()); + assertEquals("10.0.0.5/32", rule.getClients().get(0).getMatch()); + verify(nasFeignClient).updateExportPolicy(anyString(), eq("policy-42"), any(ExportPolicy.class)); + } + + // updateAccessGroup - REMOVE: matching host IP removed from policy + @Test + public void testUpdateAccessGroup_Remove_MatchingHost_Success() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn("10.0.0.1"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + accessGroup.setHostRuleAction(AccessGroup.HostRuleAction.REMOVE); + + ExportPolicy existingPolicy = existingPolicyWithClients("10.0.0.1/32", "10.0.0.2/32"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + assertSame(existingPolicy, result.getPolicy()); + // Only 10.0.0.2/32 should remain + List clients = existingPolicy.getRules().get(0).getClients(); + assertEquals(1, clients.size()); + assertEquals("10.0.0.2/32", clients.get(0).getMatch()); + verify(nasFeignClient).updateExportPolicy(anyString(), eq("policy-42"), any(ExportPolicy.class)); + } + + // updateAccessGroup - REMOVE: IP not in policy (no-op) + @Test + public void testUpdateAccessGroup_Remove_IpNotPresent_NoUpdate() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn("10.0.0.99"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + accessGroup.setHostRuleAction(AccessGroup.HostRuleAction.REMOVE); + + ExportPolicy existingPolicy = existingPolicyWithClients("10.0.0.1/32"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + AccessGroup result = strategy.updateAccessGroup(accessGroup); + + assertNotNull(result); + assertSame(existingPolicy, result.getPolicy()); + assertEquals(1, existingPolicy.getRules().get(0).getClients().size()); + verify(nasFeignClient, never()).updateExportPolicy(anyString(), anyString(), any()); + } + + // updateAccessGroup - FeignException from ONTAP wrapped in CloudRuntimeException + @Test + public void testUpdateAccessGroup_FeignExceptionWrapped() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn("10.0.0.1"); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))) + .thenThrow(new RuntimeException("ONTAP unreachable")); + + assertThrows(CloudRuntimeException.class, () -> strategy.updateAccessGroup(accessGroup)); + } + // updateAccessGroup - whitespace in storage IP is trimmed before building match + @Test + public void testUpdateAccessGroup_TrimsWhitespaceFromStorageIp() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn(" 10.0.0.2 "); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + ExportPolicy existingPolicy = existingPolicyWithClients(); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + strategy.updateAccessGroup(accessGroup); + + List clients = existingPolicy.getRules().get(0).getClients(); + assertEquals(1, clients.size()); + assertEquals("10.0.0.2/32", clients.get(0).getMatch()); + } + + // updateAccessGroup - whitespace in private IP is trimmed when storage IP absent + @Test + public void testUpdateAccessGroup_TrimsWhitespaceFromPrivateIp() { + HostVO host = mock(HostVO.class); + when(host.getStorageIpAddress()).thenReturn(null); + when(host.getPrivateIpAddress()).thenReturn(" 192.168.1.10 "); + + AccessGroup accessGroup = new AccessGroup(); + accessGroup.setStoragePoolId(1L); + accessGroup.setHostsToConnect(List.of(host)); + + ExportPolicy existingPolicy = existingPolicyWithClients(); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(detailsWithExportPolicyId()); + when(nasFeignClient.getExportPolicyById(anyString(), eq("policy-42"))).thenReturn(existingPolicy); + + strategy.updateAccessGroup(accessGroup); + + List clients = existingPolicy.getRules().get(0).getClients(); + assertEquals(1, clients.size()); + assertEquals("192.168.1.10/32", clients.get(0).getMatch()); + } + + @Test + public void testCloneCloudStackVolume_Success() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid"); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeDao.update(anyLong(), any(VolumeVO.class))).thenReturn(true); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.SVM_NAME, "svm1"); + details.put(OntapStorageConstants.VOLUME_NAME, "flexvol1"); + details.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(details); + + FileInfo source = new FileInfo(); + source.setPath("template-uuid"); + CloudStackVolume request = new CloudStackVolume(); + request.setDatastoreId("1"); + request.setVolumeInfo(volumeObject); + request.setFile(source); + request.setDestinationPath("volume-uuid"); + + when(nasFeignClient.cloneFile(anyString(), any(FileCloneRequest.class))).thenReturn(new JobResponse()); + + CloudStackVolume result = strategy.cloneCloudStackVolume(request); + + assertNotNull(result); + assertEquals("volume-uuid", result.getFile().getPath()); + ArgumentCaptor captor = ArgumentCaptor.forClass(FileCloneRequest.class); + verify(nasFeignClient).cloneFile(anyString(), captor.capture()); + assertEquals("template-uuid", captor.getValue().getSourcePath()); + assertEquals("volume-uuid", captor.getValue().getDestinationPath()); + assertEquals("flexvol1", captor.getValue().getVolume().getName()); + assertEquals("flexvol-uuid-1", captor.getValue().getVolume().getUuid()); + } + + @Test + public void testCloneCloudStackVolume_SnapshotSource_UsesSnapshotSourcePath() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeDao.update(anyLong(), any(VolumeVO.class))).thenReturn(true); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.VOLUME_NAME, "flexvol1"); + details.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(details); + + FileInfo source = new FileInfo(); + source.setPath("./.snapshot/snap_cs200/source-file-uuid"); + CloudStackVolume request = new CloudStackVolume(); + request.setDatastoreId("1"); + request.setVolumeInfo(volumeObject); + request.setFile(source); + request.setDestinationPath("new-volume-uuid"); + + when(nasFeignClient.cloneFile(anyString(), any(FileCloneRequest.class))).thenReturn(new JobResponse()); + + CloudStackVolume result = strategy.cloneCloudStackVolume(request); + + assertEquals("new-volume-uuid", result.getFile().getPath()); + ArgumentCaptor captor = ArgumentCaptor.forClass(FileCloneRequest.class); + verify(nasFeignClient).cloneFile(anyString(), captor.capture()); + assertEquals("./.snapshot/snap_cs200/source-file-uuid", captor.getValue().getSourcePath()); + assertEquals("new-volume-uuid", captor.getValue().getDestinationPath()); + } + + @Test + public void testCloneCloudStackVolume_InvalidRequest_ThrowsException() { + assertThrows(CloudRuntimeException.class, () -> strategy.cloneCloudStackVolume(null)); + } + + @Test + public void testResizeCloudStackVolume_SendsResizeCommand() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + ResizeVolumePayload payload = new ResizeVolumePayload( + 21474836480L, null, null, null, false, "i-2-VM", null, false); + + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid"); + when(volumeObject.getpayload()).thenReturn(payload); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPath()).thenReturn("volume-uuid"); + when(volumeVO.getSize()).thenReturn(5368709120L); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))).thenReturn(new Answer(null, true, "Success")); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + strategy.resizeCloudStackVolume(request, 21474836480L); + + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(ResizeVolumeCommand.class); + verify(endPoint).sendMessage(commandCaptor.capture()); + ResizeVolumeCommand command = commandCaptor.getValue(); + assertEquals("volume-uuid", command.getPath()); + assertEquals(5368709120L, command.getCurrentSize()); + assertEquals(21474836480L, command.getNewSize()); + assertEquals("i-2-VM", command.getInstanceName()); + verify(nasFeignClient, never()).updateFile(anyString(), anyString(), anyString(), any(FileInfo.class)); + } + + @Test + public void testResizeCloudStackVolume_AppliesQosAfterFileGrow() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + ResizeVolumePayload payload = new ResizeVolumePayload( + 21474836480L, 100L, 200L, null, false, "i-2-VM", null, false); + + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid"); + when(volumeObject.getpayload()).thenReturn(payload); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPath()).thenReturn("volume-uuid"); + when(volumeVO.getSize()).thenReturn(5368709120L); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))).thenReturn(new Answer(null, true, "Success")); + + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_100_to_200_iops_svm1"); + qosPolicy.setUuid("qos-uuid"); + FileInfo fileInfo = new FileInfo(); + fileInfo.setQosPolicy(qosPolicy); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + request.setFlexVolumeUuid("flex-uuid"); + request.setFile(fileInfo); + + strategy.resizeCloudStackVolume(request, 21474836480L); + + verify(endPoint).sendMessage(any(ResizeVolumeCommand.class)); + verify(nasFeignClient).updateFile(anyString(), eq("flex-uuid"), eq("volume-uuid"), + argThat(file -> file.getQosPolicy() != null + && "cs_100_to_200_iops_svm1".equals(file.getQosPolicy().getName()) + && "qos-uuid".equals(file.getQosPolicy().getUuid()))); + } + + @Test + public void testResizeCloudStackVolume_WithoutPayloadUsesDetachedInstanceName() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid"); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPath()).thenReturn("volume-uuid"); + when(volumeVO.getSize()).thenReturn(5368709120L); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))).thenReturn(new Answer(null, true, "Success")); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + strategy.resizeCloudStackVolume(request, 21474836480L); + + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(ResizeVolumeCommand.class); + verify(endPoint).sendMessage(commandCaptor.capture()); + assertEquals("none", commandCaptor.getValue().getInstanceName()); + } + + @Test + public void testResizeCloudStackVolume_AgentFailureThrowsException() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))) + .thenReturn(new Answer(null, false, "qemu-img resize failed")); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + assertEquals("qemu-img resize failed", exception.getMessage()); + } + + @Test + public void testResizeCloudStackVolume_InvalidSizeDoesNotSendCommand() { + VolumeObject volumeObject = mock(VolumeObject.class); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(request, 0L)); + + verify(epSelector, never()).select(any(org.apache.cloudstack.engine.subsystem.api.storage.DataObject.class)); + } + + @Test + public void testResizeCloudStackVolume_NegativeSizeDoesNotSendCommand() { + VolumeObject volumeObject = mock(VolumeObject.class); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(request, -1L)); + + verify(epSelector, never()).select(any(org.apache.cloudstack.engine.subsystem.api.storage.DataObject.class)); + } + + @Test + public void testDeleteFileByPath_Treats404AsSuccess() { + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(404); + doThrow(feignException).when(nasFeignClient).deleteFile(anyString(), eq("flexvol-uuid"), eq("template-uuid")); + + strategy.deleteFileByPath("flexvol-uuid", "template-uuid"); + + verify(nasFeignClient).deleteFile(anyString(), eq("flexvol-uuid"), eq("template-uuid")); + } + + @Test + public void testDeleteFileByPath_Non404Feign_Throws() { + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(500); + when(feignException.getMessage()).thenReturn("server error"); + doThrow(feignException).when(nasFeignClient).deleteFile(anyString(), eq("flexvol-uuid"), eq("template-uuid")); + + assertThrows(CloudRuntimeException.class, + () -> strategy.deleteFileByPath("flexvol-uuid", "template-uuid")); + } + + @Test + public void testCloneCloudStackVolume_MissingFlexVolUuid_Throws() { + FileInfo source = new FileInfo(); + source.setPath("template-uuid"); + CloudStackVolume request = new CloudStackVolume(); + request.setDatastoreId("1"); + request.setFile(source); + request.setDestinationPath("volume-uuid"); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.VOLUME_NAME, "flexvol1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(details); + + assertThrows(CloudRuntimeException.class, () -> strategy.cloneCloudStackVolume(request)); + verify(nasFeignClient, never()).cloneFile(anyString(), any(FileCloneRequest.class)); + } + + @Test + public void testCloneCloudStackVolume_MissingDatastoreId_Throws() { + FileInfo source = new FileInfo(); + source.setPath("template-uuid"); + CloudStackVolume request = new CloudStackVolume(); + request.setFile(source); + request.setDestinationPath("volume-uuid"); + + assertThrows(CloudRuntimeException.class, () -> strategy.cloneCloudStackVolume(request)); + } + + @Test + public void testCloneCloudStackVolume_FeignException_Throws() { + VolumeObject volumeObject = mock(VolumeObject.class); + Map details = new HashMap<>(); + details.put(OntapStorageConstants.VOLUME_NAME, "flexvol1"); + details.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-1"); + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(details); + + FileInfo source = new FileInfo(); + source.setPath("template-uuid"); + CloudStackVolume request = new CloudStackVolume(); + request.setDatastoreId("1"); + request.setVolumeInfo(volumeObject); + request.setFile(source); + request.setDestinationPath("volume-uuid"); + + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(500); + when(feignException.getMessage()).thenReturn("clone failed"); + when(nasFeignClient.cloneFile(anyString(), any(FileCloneRequest.class))).thenThrow(feignException); + + assertThrows(CloudRuntimeException.class, () -> strategy.cloneCloudStackVolume(request)); + } + + @Test + public void testResizeCloudStackVolume_InvalidRequest_Throws() { + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(null, 100L)); + CloudStackVolume empty = new CloudStackVolume(); + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(empty, 100L)); + + VolumeObject volumeObject = mock(VolumeObject.class); + CloudStackVolume withVol = new CloudStackVolume(); + withVol.setVolumeInfo(volumeObject); + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(withVol, 0L)); + } + + @Test + public void testResizeCloudStackVolume_KvmHostFails_Throws() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + + when(volumeObject.getId()).thenReturn(100L); + when(volumeObject.getUuid()).thenReturn("volume-uuid"); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPath()).thenReturn("volume-uuid"); + when(volumeVO.getSize()).thenReturn(5368709120L); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))).thenReturn(new Answer(null, false, "resize failed")); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + } + + @Test + public void testResizeCloudStackVolume_NoEndpoint_Throws() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPath()).thenReturn("volume-uuid"); + when(volumeVO.getSize()).thenReturn(5368709120L); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(null); + + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + assertThrows(CloudRuntimeException.class, () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + } + + @Test + public void testResizeCloudStackVolume_VolumeNotFound_Throws() { + VolumeObject volumeObject = mock(VolumeObject.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(null); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + + assertTrue(exception.getMessage().contains("Volume not found")); + verify(epSelector, never()).select(any(org.apache.cloudstack.engine.subsystem.api.storage.DataObject.class)); + } + + @Test + public void testResizeCloudStackVolume_StoragePoolNotFound_Throws() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(null); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + + assertTrue(exception.getMessage().contains("Storage Pool not found")); + verify(epSelector, never()).select(any(org.apache.cloudstack.engine.subsystem.api.storage.DataObject.class)); + } + + @Test + public void testResizeCloudStackVolume_NullAnswer_Throws() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))).thenReturn(null); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + + assertTrue(exception.getMessage().contains("Failed to resize qcow2 on KVM host")); + } + + @Test + public void testResizeCloudStackVolume_AgentException_Propagates() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))) + .thenThrow(new CloudRuntimeException("KVM agent unreachable")); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + + assertEquals("KVM agent unreachable", exception.getMessage()); + } + + @Test + public void testResizeCloudStackVolume_DeletedFile_ThrowsAgentError() { + VolumeObject volumeObject = mock(VolumeObject.class); + VolumeVO volumeVO = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + EndPoint endPoint = mock(EndPoint.class); + when(volumeObject.getId()).thenReturn(100L); + when(volumeDao.findById(100L)).thenReturn(volumeVO); + when(volumeVO.getPath()).thenReturn("deleted-volume.qcow2"); + when(volumeVO.getPoolId()).thenReturn(1L); + when(primaryDataStoreDao.findById(1L)).thenReturn(storagePool); + when(epSelector.select(volumeObject)).thenReturn(endPoint); + when(endPoint.sendMessage(any(ResizeVolumeCommand.class))) + .thenReturn(new Answer(null, false, + "qemu-img: Could not open 'deleted-volume.qcow2': No such file or directory")); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeObject); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.resizeCloudStackVolume(request, 21474836480L)); + + assertTrue(exception.getMessage().contains("No such file or directory")); + assertTrue(exception.getMessage().contains("deleted-volume.qcow2")); + } } diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java index ec9023a6c760..3a37936009b1 100644 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java @@ -18,11 +18,14 @@ */ package org.apache.cloudstack.storage.service; -import com.cloud.host.HostVO; -import com.cloud.utils.exception.CloudRuntimeException; -import feign.FeignException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.feign.client.SANFeignClient; import org.apache.cloudstack.storage.feign.model.Igroup; @@ -30,36 +33,31 @@ import org.apache.cloudstack.storage.feign.model.Lun; import org.apache.cloudstack.storage.feign.model.LunMap; import org.apache.cloudstack.storage.feign.model.OntapStorage; +import org.apache.cloudstack.storage.feign.model.VolumeQosPolicy; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; import org.apache.cloudstack.storage.service.model.AccessGroup; import org.apache.cloudstack.storage.service.model.CloudStackVolume; import org.apache.cloudstack.storage.service.model.ProtocolType; import org.apache.cloudstack.storage.utils.OntapStorageConstants; import org.apache.cloudstack.storage.utils.OntapStorageUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; +import org.mockito.Mock; +import org.mockito.MockedStatic; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; @@ -68,6 +66,14 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.cloud.host.HostVO; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.exception.CloudRuntimeException; + +import feign.FeignException; @ExtendWith(MockitoExtension.class) class UnifiedSANStrategyTest { @@ -75,6 +81,9 @@ class UnifiedSANStrategyTest { @Mock private SANFeignClient sanFeignClient; + @Mock + private org.apache.cloudstack.storage.feign.client.NASFeignClient nasFeignClient; + @Mock private OntapStorage ontapStorage; @@ -87,6 +96,9 @@ class UnifiedSANStrategyTest { @Mock private StoragePoolDetailsDao storagePoolDetailsDao; + @Mock + private VolumeDetailsDao volumeDetailsDao; + private UnifiedSANStrategy unifiedSANStrategy; private String authHeader; @@ -105,6 +117,10 @@ void setUp() { sanFeignClientField.setAccessible(true); sanFeignClientField.set(unifiedSANStrategy, sanFeignClient); + java.lang.reflect.Field nasFeignClientField = StorageStrategy.class.getDeclaredField("nasFeignClient"); + nasFeignClientField.setAccessible(true); + nasFeignClientField.set(unifiedSANStrategy, nasFeignClient); + // Also inject the storage field from parent class to ensure proper mocking java.lang.reflect.Field storageField = StorageStrategy.class.getDeclaredField("storage"); storageField.setAccessible(true); @@ -114,6 +130,11 @@ void setUp() { java.lang.reflect.Field storagePoolDetailsDaoField = UnifiedSANStrategy.class.getDeclaredField("storagePoolDetailsDao"); storagePoolDetailsDaoField.setAccessible(true); storagePoolDetailsDaoField.set(unifiedSANStrategy, storagePoolDetailsDao); + + // Inject volumeDetailsDao, used to resolve the LUN UUID during resize + java.lang.reflect.Field volumeDetailsDaoField = UnifiedSANStrategy.class.getDeclaredField("volumeDetailsDao"); + volumeDetailsDaoField.setAccessible(true); + volumeDetailsDaoField.set(unifiedSANStrategy, volumeDetailsDao); } catch (Exception e) { throw new RuntimeException(e); } @@ -157,6 +178,172 @@ void testCreateCloudStackVolume_Success() { } } + @Test + void testCreateTemplateCache_Success() { + org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool = + mock(org.apache.cloudstack.storage.datastore.db.StoragePoolVO.class); + when(storagePool.getId()).thenReturn(1L); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getHypervisor()).thenReturn(com.cloud.hypervisor.Hypervisor.HypervisorType.KVM); + + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo = + mock(org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo.class); + when(templateInfo.getId()).thenReturn(50L); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.SVM_NAME, "svm1"); + + Lun createdLun = new Lun(); + createdLun.setName("/vol/vol1/cs_tmpl_50"); + createdLun.setUuid("template-lun-uuid"); + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(createdLun)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenReturn(response); + + CloudStackVolume result = unifiedSANStrategy.createTemplateCache( + storagePool, templateInfo, details, 5368709120L); + + assertNotNull(result); + assertEquals("template-lun-uuid", result.getLun().getUuid()); + assertEquals("/vol/vol1/cs_tmpl_50", result.getLun().getName()); + + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).createLun(eq(authHeader), eq(true), lunCaptor.capture()); + assertEquals("/vol/vol1/cs_tmpl_50", lunCaptor.getValue().getName()); + assertEquals(5368709120L, lunCaptor.getValue().getSpace().getSize()); + } + } + + @Test + void testCreateTemplateCache_UnknownSize_ThrowsException() { + org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool = + mock(org.apache.cloudstack.storage.datastore.db.StoragePoolVO.class); + when(storagePool.getId()).thenReturn(1L); + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo = + mock(org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo.class); + when(templateInfo.getId()).thenReturn(50L); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createTemplateCache(storagePool, templateInfo, Map.of(), 0L)); + verify(sanFeignClient, never()).createLun(any(), anyBoolean(), any()); + } + + @Test + void testCreateTemplateCache_CreateFails_BestEffortDeletesLeftoverLun() { + org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool = + mock(org.apache.cloudstack.storage.datastore.db.StoragePoolVO.class); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getHypervisor()).thenReturn(com.cloud.hypervisor.Hypervisor.HypervisorType.KVM); + + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo = + mock(org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo.class); + when(templateInfo.getId()).thenReturn(50L); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.SVM_NAME, "svm1"); + + FeignException createEx = mock(FeignException.class); + when(createEx.status()).thenReturn(500); + when(createEx.getMessage()).thenReturn("create failed"); + + Lun leftover = new Lun(); + leftover.setName("/vol/vol1/cs_tmpl_50"); + leftover.setUuid("leftover-uuid"); + OntapResponse leftoverResponse = new OntapResponse<>(); + leftoverResponse.setRecords(List.of(leftover)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenThrow(createEx); + when(sanFeignClient.getLunResponse(eq(authHeader), anyMap())).thenReturn(leftoverResponse); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createTemplateCache(storagePool, templateInfo, details, 5368709120L)); + + verify(sanFeignClient).deleteLun(eq(authHeader), eq("leftover-uuid"), anyMap()); + } + } + + @Test + void testCreateTemplateCache_CreateFails_NoLeftoverLun_SkipsDelete() { + org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool = + mock(org.apache.cloudstack.storage.datastore.db.StoragePoolVO.class); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getHypervisor()).thenReturn(com.cloud.hypervisor.Hypervisor.HypervisorType.KVM); + + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo = + mock(org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo.class); + when(templateInfo.getId()).thenReturn(50L); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.SVM_NAME, "svm1"); + + FeignException createEx = mock(FeignException.class); + when(createEx.status()).thenReturn(500); + when(createEx.getMessage()).thenReturn("create failed"); + + OntapResponse empty = new OntapResponse<>(); + empty.setRecords(List.of()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenThrow(createEx); + when(sanFeignClient.getLunResponse(eq(authHeader), anyMap())).thenReturn(empty); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createTemplateCache(storagePool, templateInfo, details, 5368709120L)); + + verify(sanFeignClient, never()).deleteLun(any(), any(), anyMap()); + } + } + + @Test + void testCreateTemplateCache_CleanupFailureDoesNotMaskCreateError() { + org.apache.cloudstack.storage.datastore.db.StoragePoolVO storagePool = + mock(org.apache.cloudstack.storage.datastore.db.StoragePoolVO.class); + when(storagePool.getName()).thenReturn("vol1"); + when(storagePool.getHypervisor()).thenReturn(com.cloud.hypervisor.Hypervisor.HypervisorType.KVM); + + org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo templateInfo = + mock(org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo.class); + when(templateInfo.getId()).thenReturn(50L); + + Map details = new HashMap<>(); + details.put(OntapStorageConstants.SVM_NAME, "svm1"); + + FeignException createEx = mock(FeignException.class); + when(createEx.status()).thenReturn(500); + when(createEx.getMessage()).thenReturn("create failed"); + + Lun leftover = new Lun(); + leftover.setName("/vol/vol1/cs_tmpl_50"); + leftover.setUuid("leftover-uuid"); + OntapResponse leftoverResponse = new OntapResponse<>(); + leftoverResponse.setRecords(List.of(leftover)); + + FeignException deleteEx = mock(FeignException.class); + when(deleteEx.status()).thenReturn(500); + when(deleteEx.getMessage()).thenReturn("delete failed"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenThrow(createEx); + when(sanFeignClient.getLunResponse(eq(authHeader), anyMap())).thenReturn(leftoverResponse); + doThrow(deleteEx).when(sanFeignClient).deleteLun(eq(authHeader), eq("leftover-uuid"), anyMap()); + + CloudRuntimeException thrown = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createTemplateCache(storagePool, templateInfo, details, 5368709120L)); + assertTrue(thrown.getMessage().contains("create failed") || thrown.getMessage().contains("Failed to create")); + } + } + @Test void testCreateCloudStackVolume_NullRequest_ThrowsException() { assertThrows(CloudRuntimeException.class, @@ -188,6 +375,28 @@ void testCreateCloudStackVolume_FeignException_ThrowsCloudRuntimeException() { } } + @Test + void testCreateCloudStackVolume_MinThroughputRejected_PropagatesOntapError() { + Lun lun = new Lun(); + lun.setName("/vol/vol1/lun1"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + FeignException feignException = mock(FeignException.class); + when(feignException.getMessage()).thenReturn("Bad Request"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))) + .thenThrow(feignException); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createCloudStackVolume(request)); + assertTrue(ex.getMessage().contains("Failed to create Lun")); + } + } + @Test void testDeleteCloudStackVolume_Success() { // Setup @@ -201,7 +410,7 @@ void testDeleteCloudStackVolume_Success() { utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) .thenReturn(authHeader); - doNothing().when(sanFeignClient).deleteLun(eq(authHeader), eq("lun-uuid-123"), anyMap()); + when(sanFeignClient.deleteLun(eq(authHeader), eq("lun-uuid-123"), anyMap())).thenReturn(null); // Execute unifiedSANStrategy.deleteCloudStackVolume(request); @@ -789,29 +998,343 @@ void testValidateInitiatorInAccessGroup_IgroupNotFound_ReturnsFalse() { } @Test - void testCopyCloudStackVolume_NullRequest_DoesNotThrow() { - // copyCloudStackVolume is not yet implemented (no-op), so it should not throw - assertDoesNotThrow(() -> unifiedSANStrategy.copyCloudStackVolume(null)); + void testCloneCloudStackVolume_Success() { + Lun.Source source = new Lun.Source(); + source.setUuid("source-lun-uuid"); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); + Lun lun = new Lun(); + lun.setName("/vol/vol1/cloned"); + lun.setClone(clone); + + VolumeInfo volumeInfo = mock(VolumeInfo.class); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(volumeInfo); + + Lun clonedLun = new Lun(); + clonedLun.setName("/vol/vol1/cloned"); + clonedLun.setUuid("cloned-lun-uuid"); + + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(clonedLun)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))) + .thenReturn(response); + + CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request); + + assertNotNull(result); + assertEquals("cloned-lun-uuid", result.getLun().getUuid()); + assertSame(volumeInfo, result.getVolumeInfo()); + verify(sanFeignClient).createLun(eq(authHeader), eq(true), any(Lun.class)); + } + } + + @Test + void testCloneCloudStackVolume_NullRequest_ThrowsException() { + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.cloneCloudStackVolume(null)); } @Test - void testCopyCloudStackVolume_NullLun_DoesNotThrow() { - // copyCloudStackVolume is not yet implemented (no-op), so it should not throw + void testCloneCloudStackVolume_MissingSource_ThrowsException() { + Lun lun = new Lun(); + lun.setName("/vol/vol1/cloned"); CloudStackVolume request = new CloudStackVolume(); - request.setLun(null); + request.setLun(lun); - assertDoesNotThrow(() -> unifiedSANStrategy.copyCloudStackVolume(request)); + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.cloneCloudStackVolume(request)); } @Test - void testCopyCloudStackVolume_ValidRequest_DoesNotThrow() { - // copyCloudStackVolume is not yet implemented (no-op), so it should not throw + void testCloneCloudStackVolume_SnapshotSource_SendsSnapshotQualifiedName() { + Lun.Source source = new Lun.Source(); + source.setName("/vol/vol1/.snapshot/snap_cs200/source_lun"); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); Lun lun = new Lun(); - lun.setName("/vol/vol1/lun1"); + lun.setName("/vol/vol1/new_lun"); + lun.setClone(clone); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + Lun createdLun = new Lun(); + createdLun.setName("/vol/vol1/new_lun"); + createdLun.setUuid("new-lun-uuid"); + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(createdLun)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenReturn(response); + + CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request); + + assertEquals("new-lun-uuid", result.getLun().getUuid()); + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).createLun(eq(authHeader), eq(true), lunCaptor.capture()); + assertEquals("/vol/vol1/.snapshot/snap_cs200/source_lun", + lunCaptor.getValue().getClone().getSource().getName()); + assertNull(lunCaptor.getValue().getClone().getSource().getUuid()); + } + } + + @Test + void testResizeCloudStackVolume_ValidRequest_PatchesSize() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + + unifiedSANStrategy.resizeCloudStackVolume(request, 21474836480L); + + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).updateLun(any(), eq("lun-uuid-123"), lunCaptor.capture()); + assertEquals(21474836480L, lunCaptor.getValue().getSpace().getSize()); + assertNull(lunCaptor.getValue().getQosPolicy()); + } + + @Test + void testResizeCloudStackVolume_PatchesSizeAndQosTogether() { + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_0_to_5000_iops_svm1"); + qosPolicy.setUuid("qos-uuid"); + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + lun.setQosPolicy(qosPolicy); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + + unifiedSANStrategy.resizeCloudStackVolume(request, 21474836480L); + + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).updateLun(any(), eq("lun-uuid-123"), lunCaptor.capture()); + assertEquals(21474836480L, lunCaptor.getValue().getSpace().getSize()); + assertEquals("cs_0_to_5000_iops_svm1", lunCaptor.getValue().getQosPolicy().getName()); + assertEquals("qos-uuid", lunCaptor.getValue().getQosPolicy().getUuid()); + } + + @Test + void testResizeCloudStackVolume_NoUuid_Throws() { + VolumeInfo volumeInfo = mock(VolumeInfo.class); + when(volumeInfo.getId()).thenReturn(100L); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(new Lun()); + request.setVolumeInfo(volumeInfo); + + // LUN UUID is absent on the request and cannot be resolved from volume details + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)).thenReturn(null); + + assertThrows(CloudRuntimeException.class, () -> unifiedSANStrategy.resizeCloudStackVolume(request, 100L)); + verify(sanFeignClient, never()).updateLun(any(), any(), any()); + } + + @Test + void testResizeCloudStackVolume_BlankUuidDetail_Throws() { + VolumeInfo volumeInfo = mock(VolumeInfo.class); + when(volumeInfo.getId()).thenReturn(100L); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeInfo); + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, " ", false)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.resizeCloudStackVolume(request, 100L)); + + assertTrue(exception.getMessage().contains("LUN UUID not found")); + verify(sanFeignClient, never()).updateLun(any(), any(), any()); + } + + @Test + void testResizeCloudStackVolume_NoVolumeInfo_Throws() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + assertThrows(CloudRuntimeException.class, () -> unifiedSANStrategy.resizeCloudStackVolume(request, 100L)); + verify(sanFeignClient, never()).updateLun(any(), any(), any()); + } + + @Test + void testResizeCloudStackVolume_UuidResolvedFromVolumeDetails() { + VolumeInfo volumeInfo = mock(VolumeInfo.class); + when(volumeInfo.getId()).thenReturn(100L); + CloudStackVolume request = new CloudStackVolume(); + request.setVolumeInfo(volumeInfo); + + when(volumeDetailsDao.findDetail(100L, OntapStorageConstants.LUN_DOT_UUID)) + .thenReturn(new VolumeDetailVO(100L, OntapStorageConstants.LUN_DOT_UUID, "lun-uuid-123", false)); + + unifiedSANStrategy.resizeCloudStackVolume(request, 21474836480L); + + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).updateLun(any(), eq("lun-uuid-123"), lunCaptor.capture()); + assertEquals(21474836480L, lunCaptor.getValue().getSpace().getSize()); + } + + @Test + void testResizeCloudStackVolume_InvalidSize_Throws() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + + assertThrows(CloudRuntimeException.class, () -> unifiedSANStrategy.resizeCloudStackVolume(request, 0L)); + assertThrows(CloudRuntimeException.class, () -> unifiedSANStrategy.resizeCloudStackVolume(null, 100L)); + verify(sanFeignClient, never()).updateLun(any(), any(), any()); + } + + @Test + void testResizeCloudStackVolume_FeignExceptions_Throw() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + for (int status : new int[] {401, 403, 409, 500, -1}) { + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(status); + when(feignException.getMessage()).thenReturn( + status == -1 ? "connection timed out" : "resize failed"); + doThrow(feignException).when(sanFeignClient) + .updateLun(eq(authHeader), eq("lun-uuid-123"), any(Lun.class)); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.resizeCloudStackVolume(request, 21474836480L)); + } + } + } + + @Test + void testResizeCloudStackVolume_DeletedLun_ThrowsSpecificMessage() { + Lun lun = new Lun(); + lun.setUuid("deleted-lun-uuid"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + FeignException notFound = mock(FeignException.class); + when(notFound.status()).thenReturn(404); + when(notFound.getMessage()).thenReturn("LUN not found"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + utilityMock.when(() -> OntapStorageUtils.isOntapObjectNotFoundError(notFound)) + .thenReturn(true); + doThrow(notFound).when(sanFeignClient) + .updateLun(eq(authHeader), eq("deleted-lun-uuid"), any(Lun.class)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.resizeCloudStackVolume(request, 21474836480L)); + + assertTrue(exception.getMessage().contains("no longer exists on ONTAP")); + assertTrue(exception.getMessage().contains("deleted-lun-uuid")); + } + } + + @Test + void testResizeCloudStackVolume_UnexpectedException_IsWrapped() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + doThrow(new IllegalStateException("unexpected failure")).when(sanFeignClient) + .updateLun(eq(authHeader), eq("lun-uuid-123"), any(Lun.class)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.resizeCloudStackVolume(request, 21474836480L)); + + assertTrue(exception.getMessage().contains("Unexpected error")); + assertTrue(exception.getMessage().contains("unexpected failure")); + } + } + + @Test + void testResizeCloudStackVolume_SameSizePatchIsIdempotent() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + request.setVolumeInfo(mock(VolumeInfo.class)); + + unifiedSANStrategy.resizeCloudStackVolume(request, 10737418240L); + + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).updateLun(any(), eq("lun-uuid-123"), lunCaptor.capture()); + assertEquals(10737418240L, lunCaptor.getValue().getSpace().getSize()); + } + + @Test + void testCloneCloudStackVolume_EmptyRecords_ThrowsException() { + Lun.Source source = new Lun.Source(); + source.setUuid("source-lun-uuid"); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); + Lun lun = new Lun(); + lun.setName("/vol/vol1/cloned"); + lun.setClone(clone); + + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of()); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenReturn(response); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.cloneCloudStackVolume(request)); + } + } + + @Test + void testCloneCloudStackVolume_FeignException_ThrowsException() { + Lun.Source source = new Lun.Source(); + source.setUuid("source-lun-uuid"); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); + Lun lun = new Lun(); + lun.setName("/vol/vol1/cloned"); + lun.setClone(clone); + CloudStackVolume request = new CloudStackVolume(); request.setLun(lun); - assertDoesNotThrow(() -> unifiedSANStrategy.copyCloudStackVolume(request)); + FeignException feignException = mock(FeignException.class); + when(feignException.status()).thenReturn(500); + when(feignException.getMessage()).thenReturn("clone failed"); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenThrow(feignException); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.cloneCloudStackVolume(request)); + } } @Test @@ -821,10 +1344,84 @@ void testSetOntapStorage() { } @Test - void testUpdateCloudStackVolume_ReturnsNull() { + void testUpdateCloudStackVolume_AppliesQosPolicy() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_0_to_5000_iops_svm1"); + qosPolicy.setUuid("qos-uuid"); + lun.setQosPolicy(qosPolicy); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.updateLun(eq(authHeader), eq("lun-uuid-123"), any(Lun.class))) + .thenReturn(null); + + CloudStackVolume result = unifiedSANStrategy.updateCloudStackVolume(request); + + assertSame(request, result); + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).updateLun(eq(authHeader), eq("lun-uuid-123"), lunCaptor.capture()); + assertEquals("cs_0_to_5000_iops_svm1", lunCaptor.getValue().getQosPolicy().getName()); + assertEquals("qos-uuid", lunCaptor.getValue().getQosPolicy().getUuid()); + } + } + + @Test + void testUpdateCloudStackVolume_UnexpectedException_IsWrapped() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_0_to_5000_iops_svm1"); + lun.setQosPolicy(qosPolicy); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.updateLun(eq(authHeader), eq("lun-uuid-123"), any(Lun.class))) + .thenThrow(new IllegalStateException("decoder failed")); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.updateCloudStackVolume(request)); + assertTrue(ex.getMessage().contains("Failed to update LUN")); + assertTrue(ex.getCause() instanceof IllegalStateException); + } + } + + @Test + void testUpdateCloudStackVolume_NoQosPolicy_SkipsPatch() { + Lun lun = new Lun(); + lun.setUuid("lun-uuid-123"); CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + CloudStackVolume result = unifiedSANStrategy.updateCloudStackVolume(request); - assertNull(result); + + assertSame(request, result); + verify(sanFeignClient, never()).updateLun(any(), any(), any()); + } + + @Test + void testUpdateCloudStackVolume_InvalidRequest_ThrowsException() { + CloudStackVolume lunWithoutUuid = new CloudStackVolume(); + lunWithoutUuid.setLun(new Lun()); + + assertInvalidIscsiVolumeUpdate(null); + assertInvalidIscsiVolumeUpdate(new CloudStackVolume()); + assertInvalidIscsiVolumeUpdate(lunWithoutUuid); + + verify(sanFeignClient, never()).updateLun(any(), any(), any()); + } + + private void assertInvalidIscsiVolumeUpdate(CloudStackVolume request) { + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.updateCloudStackVolume(request)); + assertEquals("Invalid iSCSI volume request", ex.getMessage()); } @Test @@ -1419,6 +2016,85 @@ void testCreateCloudStackVolume_EmptyResponse_ThrowsException() { } } + + @Test + void testCreateCloudStackVolume_IncompleteLunMissingName_ThrowsException() { + Lun lun = new Lun(); + lun.setName("/vol/vol1/lun1"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + Lun incomplete = new Lun(); + incomplete.setUuid("lun-uuid-123"); + // name intentionally null + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(incomplete)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))) + .thenReturn(response); + + CloudRuntimeException ex = assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createCloudStackVolume(request)); + assertTrue(ex.getMessage().contains("incomplete LUN")); + } + } + + @Test + void testCreateCloudStackVolume_IncompleteLunMissingUuid_ThrowsException() { + Lun lun = new Lun(); + lun.setName("/vol/vol1/lun1"); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + Lun incomplete = new Lun(); + incomplete.setName("/vol/vol1/lun1"); + // uuid intentionally null + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(incomplete)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))) + .thenReturn(response); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.createCloudStackVolume(request)); + } + } + + @Test + void testCloneCloudStackVolume_IncompleteLunMissingUuid_ThrowsException() { + Lun.Source source = new Lun.Source(); + source.setUuid("source-lun-uuid"); + Lun.Clone clone = new Lun.Clone(); + clone.setSource(source); + Lun lun = new Lun(); + lun.setName("/vol/vol1/cloned"); + lun.setClone(clone); + + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + Lun incomplete = new Lun(); + incomplete.setName("/vol/vol1/cloned"); + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(incomplete)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))) + .thenReturn(response); + + assertThrows(CloudRuntimeException.class, + () -> unifiedSANStrategy.cloneCloudStackVolume(request)); + } + } + @Test void testCreateCloudStackVolume_NullResponse_ThrowsException() { Lun lun = new Lun(); @@ -1802,4 +2478,35 @@ void testEnsureLunMapped_ExistingMapping_ReturnsExistingNumber() { verify(sanFeignClient, never()).createLunMap(any(), anyBoolean(), any(LunMap.class)); } } + + @Test + void testCreateCloudStackVolume_PassesQosPolicyOnLunCreate() { + Lun lun = new Lun(); + lun.setName("/vol/vol1/lun1"); + VolumeQosPolicy qosPolicy = new VolumeQosPolicy(); + qosPolicy.setName("cs_100_to200_iops_svm1"); + lun.setQosPolicy(qosPolicy); + CloudStackVolume request = new CloudStackVolume(); + request.setLun(lun); + + Lun createdLun = new Lun(); + createdLun.setName("/vol/vol1/lun1"); + createdLun.setUuid("lun-uuid-123"); + OntapResponse response = new OntapResponse<>(); + response.setRecords(List.of(createdLun)); + + try (MockedStatic utilityMock = mockStatic(OntapStorageUtils.class)) { + utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password")) + .thenReturn(authHeader); + when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))) + .thenReturn(response); + + unifiedSANStrategy.createCloudStackVolume(request); + + ArgumentCaptor lunCaptor = ArgumentCaptor.forClass(Lun.class); + verify(sanFeignClient).createLun(eq(authHeader), eq(true), lunCaptor.capture()); + assertNotNull(lunCaptor.getValue().getQosPolicy()); + assertEquals("cs_100_to200_iops_svm1", lunCaptor.getValue().getQosPolicy().getName()); + } + } } diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/utils/OntapStorageUtilsTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/utils/OntapStorageUtilsTest.java index 372a75ad257d..1fa61966af88 100644 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/utils/OntapStorageUtilsTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/utils/OntapStorageUtilsTest.java @@ -18,9 +18,11 @@ */ package org.apache.cloudstack.storage.utils; +import com.cloud.utils.exception.CloudRuntimeException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class OntapStorageUtilsTest { @@ -79,4 +81,41 @@ public void getIgroupName_truncates_whenOneCharOverMaxLength() { assertEquals(OntapStorageConstants.IGROUP_NAME_MAX_LENGTH, result.length()); } + + @Test + public void isOntapSnapshotNotFoundError_matchesEntryDoesNotExist() { + CloudRuntimeException ex = new CloudRuntimeException("Job failed with error: entry doesn't exist"); + assertTrue(OntapStorageUtils.isOntapObjectNotFoundError(ex)); + } + + @Test + public void isOntapSnapshotNotFoundError_rejectsUnrelatedErrors() { + assertFalse(OntapStorageUtils.isOntapObjectNotFoundError( + new CloudRuntimeException("Job failed with error: permission denied"))); + } + + @Test + public void toFlexVolRelativePath_stripsVolPrefix() { + assertEquals("lun1", OntapStorageUtils.toFlexVolRelativePath("/vol/vol1/lun1", "vol1")); + assertEquals("file-uuid", OntapStorageUtils.toFlexVolRelativePath("file-uuid", "vol1")); + assertEquals("file-uuid", OntapStorageUtils.toFlexVolRelativePath("/file-uuid", "vol1")); + } + + @Test + public void toLunCloneSourcePathInSnapshot_buildsSnapshotQualifiedPath() { + assertEquals("/vol/vol1/.snapshot/snap_cs200/source_lun", + OntapStorageUtils.toLunCloneSourcePathInSnapshot("/vol/vol1/source_lun", "vol1", "snap_cs200")); + assertEquals("/vol/vol1/.snapshot/snap_cs200/source_lun", + OntapStorageUtils.toLunCloneSourcePathInSnapshot("source_lun", "vol1", "snap_cs200")); + } + + @Test + public void toLunCloneSourcePathInSnapshot_rejectsBlankInputs() { + org.junit.jupiter.api.Assertions.assertThrows(com.cloud.exception.InvalidParameterValueException.class, + () -> OntapStorageUtils.toLunCloneSourcePathInSnapshot("/vol/vol1/lun", "vol1", null)); + org.junit.jupiter.api.Assertions.assertThrows(com.cloud.exception.InvalidParameterValueException.class, + () -> OntapStorageUtils.toLunCloneSourcePathInSnapshot("/vol/vol1/lun", "", "snap")); + org.junit.jupiter.api.Assertions.assertThrows(com.cloud.exception.InvalidParameterValueException.class, + () -> OntapStorageUtils.toLunCloneSourcePathInSnapshot("", "vol1", "snap")); + } } diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategyTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategyTest.java index b069ab7246a0..aa0d48c2e850 100644 --- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategyTest.java +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategyTest.java @@ -18,15 +18,22 @@ */ package org.apache.cloudstack.storage.vmsnapshot; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -44,7 +51,18 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.client.SnapshotFeignClient; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroup; +import org.apache.cloudstack.storage.feign.model.ConsistencyGroupSnapshot; +import org.apache.cloudstack.storage.feign.model.FlexVolSnapshot; +import org.apache.cloudstack.storage.feign.model.Job; +import org.apache.cloudstack.storage.feign.model.response.JobResponse; +import org.apache.cloudstack.storage.feign.model.response.OntapResponse; +import org.apache.cloudstack.storage.service.StorageStrategy; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Logger; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -127,6 +145,10 @@ class OntapVMSnapshotStrategyTest { private VolumeDataFactory volumeDataFactory; @Mock private VolumeDetailsDao volumeDetailsDao; + @Mock + private StorageStrategy storageStrategy; + @Mock + private SnapshotFeignClient snapshotFeignClient; @Spy @InjectMocks @@ -226,14 +248,18 @@ void testCanHandle_AllocatedDiskType_VmxenHypervisor_ReturnsCantHandle() { } @Test - void testCanHandle_AllocatedDiskType_VmNotRunning_ReturnsCantHandle() { + void testCanHandle_AllocatedDiskType_VmStopped_ReturnsHighest() { UserVmVO userVm = createMockUserVm(Hypervisor.HypervisorType.KVM, VirtualMachine.State.Stopped); when(userVmDao.findById(VM_ID)).thenReturn(userVm); VMSnapshotVO vmSnapshot = createMockVmSnapshot(VMSnapshot.State.Allocated, VMSnapshot.Type.Disk); + VolumeVO vol = createMockVolume(VOLUME_ID_1, POOL_ID_1); + when(volumeDao.findByInstance(VM_ID)).thenReturn(Collections.singletonList(vol)); + StoragePoolVO pool = createOntapManagedPool(POOL_ID_1); + when(storagePool.findById(POOL_ID_1)).thenReturn(pool); StrategyPriority result = strategy.canHandle(vmSnapshot); - assertEquals(StrategyPriority.CANT_HANDLE, result); + assertEquals(StrategyPriority.HIGHEST, result); } @Test @@ -532,6 +558,86 @@ void testGroupVolumesByFlexVol_VolumeNotFound_ThrowsException() { () -> strategy.groupVolumesByFlexVol(Collections.singletonList(volumeTO1))); } + @Test + void testCreateTemporaryConsistencyGroup_includesSvmName() { + SnapshotFeignClient client = mock(SnapshotFeignClient.class); + StorageStrategy storageStrategy = mock(StorageStrategy.class); + when(client.createConsistencyGroup(any(), any())).thenReturn(createJobResponse("job-cg-create")); + OntapResponse cgResponse = new OntapResponse<>(); + ConsistencyGroup cgRecord = new ConsistencyGroup(); + cgRecord.setUuid("cg-uuid-1"); + cgResponse.setRecords(Collections.singletonList(cgRecord)); + when(client.getConsistencyGroups(any(), any())).thenReturn(cgResponse); + + String cgUuid = strategy.createTemporaryConsistencyGroup(client, storageStrategy, "auth", + "cg-name", new OntapVMSnapshotStrategy.ConsistencyGroupScope("10.0.0.1", "vs0", "svm-uuid-1"), + java.util.Set.of("flexvol-uuid-1", "flexvol-uuid-2")); + + assertEquals("cg-uuid-1", cgUuid); + org.mockito.ArgumentCaptor payloadCaptor = org.mockito.ArgumentCaptor.forClass(ConsistencyGroup.class); + verify(client).createConsistencyGroup(eq("auth"), payloadCaptor.capture()); + ConsistencyGroup payload = payloadCaptor.getValue(); + assertEquals("cg-name", payload.getName()); + assertEquals("svm-uuid-1", payload.getSvm().getUuid()); + assertEquals(2, payload.getVolumes().size()); + assertEquals(OntapStorageConstants.CG_VOLUME_PROVISIONING_ACTION_ADD, + payload.getVolumes().get(0).getProvisioningOptions().getAction()); + } + + @Test + void testResolveConsistencyGroupScope_rejectsDifferentStorageIpWithSameSvmName() { + Map groups = new HashMap<>(); + Map poolDetails1 = new HashMap<>(); + poolDetails1.put(OntapStorageConstants.STORAGE_IP, "10.1.1.1"); + poolDetails1.put(OntapStorageConstants.SVM_NAME, "vs0"); + groups.put("flexvol-uuid-1", new OntapVMSnapshotStrategy.FlexVolGroupInfo(poolDetails1, POOL_ID_1)); + + Map poolDetails2 = new HashMap<>(); + poolDetails2.put(OntapStorageConstants.STORAGE_IP, "10.2.2.2"); + poolDetails2.put(OntapStorageConstants.SVM_NAME, "vs0"); + groups.put("flexvol-uuid-2", new OntapVMSnapshotStrategy.FlexVolGroupInfo(poolDetails2, POOL_ID_2)); + + assertThrows(CloudRuntimeException.class, () -> strategy.resolveConsistencyGroupScope(groups)); + } + + @Test + void testResolveConsistencyGroupScope_acceptsSameClusterAndSvmUuid() { + Map groups = new HashMap<>(); + Map poolDetails1 = new HashMap<>(); + poolDetails1.put(OntapStorageConstants.STORAGE_IP, "10.1.1.1"); + poolDetails1.put(OntapStorageConstants.SVM_NAME, "vs0"); + poolDetails1.put(OntapStorageConstants.SVM_UUID, "svm-uuid-shared"); + groups.put("flexvol-uuid-1", new OntapVMSnapshotStrategy.FlexVolGroupInfo(poolDetails1, POOL_ID_1)); + + Map poolDetails2 = new HashMap<>(); + poolDetails2.put(OntapStorageConstants.STORAGE_IP, "10.1.1.1"); + poolDetails2.put(OntapStorageConstants.SVM_NAME, "vs0"); + poolDetails2.put(OntapStorageConstants.SVM_UUID, "svm-uuid-shared"); + groups.put("flexvol-uuid-2", new OntapVMSnapshotStrategy.FlexVolGroupInfo(poolDetails2, POOL_ID_2)); + + OntapVMSnapshotStrategy.ConsistencyGroupScope scope = strategy.resolveConsistencyGroupScope(groups); + assertEquals("svm-uuid-shared", scope.svmUuid); + assertEquals("10.1.1.1", scope.storageIp); + } + + @Test + void testResolveConsistencyGroupScope_rejectsDifferentSvmUuidOnSameCluster() { + Map groups = new HashMap<>(); + Map poolDetails1 = new HashMap<>(); + poolDetails1.put(OntapStorageConstants.STORAGE_IP, "10.1.1.1"); + poolDetails1.put(OntapStorageConstants.SVM_NAME, "vs0"); + poolDetails1.put(OntapStorageConstants.SVM_UUID, "svm-uuid-1"); + groups.put("flexvol-uuid-1", new OntapVMSnapshotStrategy.FlexVolGroupInfo(poolDetails1, POOL_ID_1)); + + Map poolDetails2 = new HashMap<>(); + poolDetails2.put(OntapStorageConstants.STORAGE_IP, "10.1.1.1"); + poolDetails2.put(OntapStorageConstants.SVM_NAME, "vs0"); + poolDetails2.put(OntapStorageConstants.SVM_UUID, "svm-uuid-2"); + groups.put("flexvol-uuid-2", new OntapVMSnapshotStrategy.FlexVolGroupInfo(poolDetails2, POOL_ID_2)); + + assertThrows(CloudRuntimeException.class, () -> strategy.resolveConsistencyGroupScope(groups)); + } + // ══════════════════════════════════════════════════════════════════════════ // Tests: FlexVolSnapshotDetail parse/toString // ══════════════════════════════════════════════════════════════════════════ @@ -585,6 +691,44 @@ void testFlexVolSnapshotDetail_Parse5Parts_ThrowsException() { () -> OntapVMSnapshotStrategy.FlexVolSnapshotDetail.parse("a::b::c::d::e")); } + @Test + void testRollbackFlexVolSnapshot_UsesSharedDelete() { + OntapVMSnapshotStrategy.FlexVolSnapshotDetail detail = new OntapVMSnapshotStrategy.FlexVolSnapshotDetail( + "flexvol-uuid-1", "snapshot-uuid-1", "snapshot-name-1", "volume-301.qcow2", POOL_ID_1, "NFS3"); + doReturn(storageStrategy).when(strategy).resolveStorageStrategy(any()); + + assertDoesNotThrow(() -> strategy.rollbackFlexVolSnapshot(detail)); + + verify(storageStrategy).deleteFlexVolSnapshotForCloudStackVolume( + "flexvol-uuid-1", "snapshot-uuid-1", "snapshot-name-1"); + verify(storageStrategy, never()).getSnapshotFeignClient(); + verify(storageStrategy, never()).getAuthHeader(); + verify(storageStrategy, never()).jobPollForSuccess(any(), anyInt(), anyInt()); + } + + @Test + void testRollbackFlexVolSnapshot_DeleteFailureIsLoggedAndNotThrown() { + OntapVMSnapshotStrategy.FlexVolSnapshotDetail detail = new OntapVMSnapshotStrategy.FlexVolSnapshotDetail( + "flexvol-uuid-1", "snapshot-uuid-1", "snapshot-name-1", "volume-301.qcow2", POOL_ID_1, "NFS3"); + doReturn(storageStrategy).when(strategy).resolveStorageStrategy(any()); + doThrow(new CloudRuntimeException("delete failed")).when(storageStrategy) + .deleteFlexVolSnapshotForCloudStackVolume(any(), any(), any()); + + Logger logger = (Logger) LogManager.getLogger(OntapVMSnapshotStrategy.class); + Appender appender = mock(Appender.class); + when(appender.getName()).thenReturn("rollback-test-appender"); + when(appender.isStarted()).thenReturn(true); + logger.addAppender(appender); + try { + assertDoesNotThrow(() -> strategy.rollbackFlexVolSnapshot(detail)); + + verify(appender).append(argThat(event -> event.getMessage().getFormattedMessage() + .contains("Rollback of FlexVol snapshot failed: delete failed"))); + } finally { + logger.removeAppender(appender); + } + } + // ══════════════════════════════════════════════════════════════════════════ // Tests: buildSnapshotName // ══════════════════════════════════════════════════════════════════════════ @@ -593,10 +737,11 @@ void testFlexVolSnapshotDetail_Parse5Parts_ThrowsException() { void testBuildSnapshotName_Format() { VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); when(vmSnapshot.getId()).thenReturn(SNAPSHOT_ID); + when(vmSnapshot.getName()).thenReturn("UI VM Snapshot"); String name = strategy.buildSnapshotName(vmSnapshot); - assertEquals(true, name.startsWith("vmsnap_200_")); + assertEquals(true, name.startsWith("UI_VM_Snapshot_vm200")); assertEquals(true, name.length() <= OntapStorageConstants.MAX_SNAPSHOT_NAME_LENGTH); } @@ -732,6 +877,116 @@ void testTakeVMSnapshot_OperationTimeout_ThrowsCloudRuntimeException() throws Ex assertEquals(true, ex.getMessage().contains("timed out")); } + @Test + void testTakeVMSnapshot_SingleFlexVolSuccess_UsesDirectSnapshotNotCg() throws Exception { + VMSnapshotVO vmSnapshot = createTakeSnapshotVmSnapshot(); + setupTakeSnapshotCommon(vmSnapshot); + setupSingleVolumeForTakeSnapshot(); + + String snapshotName = strategy.buildSnapshotName(vmSnapshot); + setupSingleFlexVolFlowMocks(snapshotName); + + FreezeThawVMAnswer freezeAnswer = mock(FreezeThawVMAnswer.class); + when(freezeAnswer.getResult()).thenReturn(true); + FreezeThawVMAnswer thawAnswer = mock(FreezeThawVMAnswer.class); + when(thawAnswer.getResult()).thenReturn(true); + when(agentMgr.send(eq(HOST_ID), any(FreezeThawVMCommand.class))) + .thenReturn(freezeAnswer) + .thenReturn(thawAnswer); + + strategy.takeVMSnapshot(vmSnapshot); + + verify(snapshotFeignClient, times(1)).createSnapshot(any(), eq("flexvol-uuid-1"), any()); + verify(snapshotFeignClient, never()).createConsistencyGroup(any(), any()); + verify(snapshotFeignClient, never()).createConsistencyGroupSnapshot(any(), any(), any()); + verify(snapshotFeignClient, never()).commitConsistencyGroupSnapshot(any(), any(), any(), any()); + verify(snapshotFeignClient, never()).deleteConsistencyGroup(any(), any()); + verify(vmSnapshotDetailsDao, atLeastOnce()).persist(any(VMSnapshotDetailsVO.class)); + } + + @Test + void testTakeVMSnapshot_RollbackDeleteFailurePreservesOriginalError() throws Exception { + VMSnapshotVO vmSnapshot = createTakeSnapshotVmSnapshot(); + setupTakeSnapshotCommon(vmSnapshot); + setupSingleVolumeForTakeSnapshot(); + String snapshotName = strategy.buildSnapshotName(vmSnapshot); + setupSingleFlexVolFlowMocks(snapshotName); + + FreezeThawVMAnswer freezeAnswer = mock(FreezeThawVMAnswer.class); + when(freezeAnswer.getResult()).thenReturn(true); + FreezeThawVMAnswer thawAnswer = mock(FreezeThawVMAnswer.class); + when(thawAnswer.getResult()).thenReturn(true); + when(agentMgr.send(eq(HOST_ID), any(FreezeThawVMCommand.class))) + .thenReturn(freezeAnswer) + .thenReturn(thawAnswer); + doThrow(new CloudRuntimeException("snapshot persistence failed")).when(vmSnapshotDetailsDao) + .persist(any(VMSnapshotDetailsVO.class)); + doThrow(new CloudRuntimeException("rollback delete failed")).when(storageStrategy) + .deleteFlexVolSnapshotForCloudStackVolume(any(), any(), any()); + when(vmSnapshotDetailsDao.listDetails(SNAPSHOT_ID)).thenReturn(Collections.emptyList()); + doReturn(true).when(vmSnapshotHelper).vmSnapshotStateTransitTo(any(), eq(VMSnapshot.Event.OperationFailed)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> strategy.takeVMSnapshot(vmSnapshot)); + + assertEquals("snapshot persistence failed", exception.getMessage()); + verify(storageStrategy).deleteFlexVolSnapshotForCloudStackVolume( + eq("flexvol-uuid-1"), any(), eq(snapshotName)); + } + + @Test + void testTakeVMSnapshot_TemporaryCgTwoPhaseSuccess_PersistsDetailsAndCleansUpCg() throws Exception { + VMSnapshotVO vmSnapshot = createTakeSnapshotVmSnapshot(); + setupTakeSnapshotCommon(vmSnapshot); + setupMultiFlexVolForTakeSnapshot(); + + String snapshotName = strategy.buildSnapshotName(vmSnapshot); + setupTemporaryCgFlowMocks(snapshotName); + + FreezeThawVMAnswer freezeAnswer = mock(FreezeThawVMAnswer.class); + when(freezeAnswer.getResult()).thenReturn(true); + FreezeThawVMAnswer thawAnswer = mock(FreezeThawVMAnswer.class); + when(thawAnswer.getResult()).thenReturn(true); + when(agentMgr.send(eq(HOST_ID), any(FreezeThawVMCommand.class))) + .thenReturn(freezeAnswer) + .thenReturn(thawAnswer); + + strategy.takeVMSnapshot(vmSnapshot); + + verify(snapshotFeignClient, times(1)).createConsistencyGroup(any(), any()); + verify(snapshotFeignClient, times(1)).createConsistencyGroupSnapshot(any(), eq("cg-uuid-1"), any()); + verify(snapshotFeignClient, times(1)).commitConsistencyGroupSnapshot(any(), eq("cg-uuid-1"), eq("cg-snap-uuid-1"), any()); + verify(snapshotFeignClient, times(1)).deleteConsistencyGroup(any(), eq("cg-uuid-1")); + verify(vmSnapshotDetailsDao, atLeastOnce()).persist(any(VMSnapshotDetailsVO.class)); + } + + @Test + void testTakeVMSnapshot_TemporaryCgStartFails_TransitionsToOperationFailed() throws Exception { + VMSnapshotVO vmSnapshot = createTakeSnapshotVmSnapshot(); + setupTakeSnapshotCommon(vmSnapshot); + setupMultiFlexVolForTakeSnapshot(); + + String snapshotName = strategy.buildSnapshotName(vmSnapshot); + setupTemporaryCgFlowMocks(snapshotName); + when(snapshotFeignClient.createConsistencyGroupSnapshot(any(), eq("cg-uuid-1"), any())) + .thenThrow(new CloudRuntimeException("start phase failed")); + + FreezeThawVMAnswer freezeAnswer = mock(FreezeThawVMAnswer.class); + when(freezeAnswer.getResult()).thenReturn(true); + FreezeThawVMAnswer thawAnswer = mock(FreezeThawVMAnswer.class); + when(thawAnswer.getResult()).thenReturn(true); + when(agentMgr.send(eq(HOST_ID), any(FreezeThawVMCommand.class))) + .thenReturn(freezeAnswer) + .thenReturn(thawAnswer); + when(vmSnapshotDetailsDao.listDetails(SNAPSHOT_ID)).thenReturn(Collections.emptyList()); + doReturn(true).when(vmSnapshotHelper).vmSnapshotStateTransitTo(any(), eq(VMSnapshot.Event.OperationFailed)); + + assertThrows(CloudRuntimeException.class, () -> strategy.takeVMSnapshot(vmSnapshot)); + + verify(snapshotFeignClient, times(1)).deleteConsistencyGroup(any(), eq("cg-uuid-1")); + verify(vmSnapshotHelper, atLeastOnce()).vmSnapshotStateTransitTo(any(), eq(VMSnapshot.Event.OperationFailed)); + } + // ══════════════════════════════════════════════════════════════════════════ // Tests: Quiesce Behavior // ══════════════════════════════════════════════════════════════════════════ @@ -746,20 +1001,9 @@ void testTakeVMSnapshot_QuiesceFalse_SkipsFreezeThaw() throws Exception { setupTakeSnapshotCommon(vmSnapshot); setupSingleVolumeForTakeSnapshot(); + setupSingleFlexVolFlowMocks(strategy.buildSnapshotName(vmSnapshot)); - // The FlexVolume snapshot flow will try to call Utility.getStrategyByStoragePoolDetails - // which is a static method that makes real connections. We expect this to fail in unit tests. - // The important thing is that freeze/thaw was NOT called before the failure. - when(vmSnapshotDetailsDao.listDetails(SNAPSHOT_ID)).thenReturn(Collections.emptyList()); - doReturn(true).when(vmSnapshotHelper).vmSnapshotStateTransitTo(any(), eq(VMSnapshot.Event.OperationFailed)); - - // Since Utility.getStrategyByStoragePoolDetails is static and creates real Feign clients, - // this will fail. We just verify that freeze was never called. - try { - strategy.takeVMSnapshot(vmSnapshot); - } catch (Exception e) { - // Expected — static utility can't be mocked in unit test - } + strategy.takeVMSnapshot(vmSnapshot); // No freeze/thaw commands should be sent when quiesce is false verify(agentMgr, never()).send(eq(HOST_ID), any(FreezeThawVMCommand.class)); @@ -790,16 +1034,9 @@ void testTakeVMSnapshot_WithParentSnapshot_SetsParentId() throws Exception { when(agentMgr.send(eq(HOST_ID), any(FreezeThawVMCommand.class))) .thenReturn(freezeAnswer) .thenReturn(thawAnswer); + setupSingleFlexVolFlowMocks(strategy.buildSnapshotName(vmSnapshot)); - when(vmSnapshotDetailsDao.listDetails(SNAPSHOT_ID)).thenReturn(Collections.emptyList()); - doReturn(true).when(vmSnapshotHelper).vmSnapshotStateTransitTo(any(), eq(VMSnapshot.Event.OperationFailed)); - - // FlexVol snapshot flow will fail on static method, but parent should already be set - try { - strategy.takeVMSnapshot(vmSnapshot); - } catch (Exception e) { - // Expected - } + strategy.takeVMSnapshot(vmSnapshot); // Verify parent was set on the VM snapshot before the FlexVol snapshot attempt verify(vmSnapshot).setParent(199L); @@ -820,15 +1057,9 @@ void testTakeVMSnapshot_WithNoParentSnapshot_SetsParentNull() throws Exception { when(agentMgr.send(eq(HOST_ID), any(FreezeThawVMCommand.class))) .thenReturn(freezeAnswer) .thenReturn(thawAnswer); + setupSingleFlexVolFlowMocks(strategy.buildSnapshotName(vmSnapshot)); - when(vmSnapshotDetailsDao.listDetails(SNAPSHOT_ID)).thenReturn(Collections.emptyList()); - doReturn(true).when(vmSnapshotHelper).vmSnapshotStateTransitTo(any(), eq(VMSnapshot.Event.OperationFailed)); - - try { - strategy.takeVMSnapshot(vmSnapshot); - } catch (Exception e) { - // Expected - } + strategy.takeVMSnapshot(vmSnapshot); verify(vmSnapshot).setParent(null); } @@ -866,6 +1097,9 @@ private UserVmVO setupTakeSnapshotCommon(VMSnapshotVO vmSnapshot) throws Excepti when(vmSnapshotDao.findCurrentSnapshotByVmId(VM_ID)).thenReturn(null); doReturn(true).when(vmSnapshotHelper).vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.CreateRequested); + doNothing().when(strategy).processAnswer(any(), any(), any(), any()); + doNothing().when(strategy).publishUsageEvent(any(), any(), any(), any()); + doNothing().when(strategy).publishUsageEvent(any(), any(), any(), anyLong(), anyLong()); return userVm; } @@ -880,6 +1114,7 @@ private void setupSingleVolumeForTakeSnapshot() { VolumeVO volumeVO = mock(VolumeVO.class); when(volumeVO.getId()).thenReturn(VOLUME_ID_1); when(volumeVO.getPoolId()).thenReturn(POOL_ID_1); + when(volumeVO.getPath()).thenReturn("volume-301.qcow2"); when(volumeVO.getVmSnapshotChainSize()).thenReturn(null); when(volumeDao.findById(VOLUME_ID_1)).thenReturn(volumeVO); @@ -899,4 +1134,139 @@ private void setupSingleVolumeForTakeSnapshot() { when(volumeInfo.getName()).thenReturn("vol-1"); when(volumeDataFactory.getVolume(VOLUME_ID_1)).thenReturn(volumeInfo); } + + private void setupMultiFlexVolForTakeSnapshot() { + VolumeObjectTO volumeTO1 = mock(VolumeObjectTO.class); + when(volumeTO1.getId()).thenReturn(VOLUME_ID_1); + when(volumeTO1.getSize()).thenReturn(10737418240L); + VolumeObjectTO volumeTO2 = mock(VolumeObjectTO.class); + when(volumeTO2.getId()).thenReturn(VOLUME_ID_2); + when(volumeTO2.getSize()).thenReturn(10737418240L); + List volumeTOs = Arrays.asList(volumeTO1, volumeTO2); + when(vmSnapshotHelper.getVolumeTOList(VM_ID)).thenReturn(volumeTOs); + + VolumeVO volumeVO1 = mock(VolumeVO.class); + when(volumeVO1.getId()).thenReturn(VOLUME_ID_1); + when(volumeVO1.getPoolId()).thenReturn(POOL_ID_1); + when(volumeVO1.getPath()).thenReturn("volume-301.qcow2"); + when(volumeVO1.getVmSnapshotChainSize()).thenReturn(null); + when(volumeDao.findById(VOLUME_ID_1)).thenReturn(volumeVO1); + + VolumeVO volumeVO2 = mock(VolumeVO.class); + when(volumeVO2.getId()).thenReturn(VOLUME_ID_2); + when(volumeVO2.getPoolId()).thenReturn(POOL_ID_2); + when(volumeVO2.getPath()).thenReturn("volume-302.qcow2"); + when(volumeVO2.getVmSnapshotChainSize()).thenReturn(null); + when(volumeDao.findById(VOLUME_ID_2)).thenReturn(volumeVO2); + + Map poolDetails1 = new HashMap<>(); + poolDetails1.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-1"); + poolDetails1.put(OntapStorageConstants.USERNAME, "admin"); + poolDetails1.put(OntapStorageConstants.PASSWORD, "pass"); + poolDetails1.put(OntapStorageConstants.STORAGE_IP, "10.0.0.1"); + poolDetails1.put(OntapStorageConstants.SVM_NAME, "svm1"); + poolDetails1.put(OntapStorageConstants.SVM_UUID, "svm-uuid-shared"); + poolDetails1.put(OntapStorageConstants.SIZE, "107374182400"); + poolDetails1.put(OntapStorageConstants.PROTOCOL, "NFS3"); + when(storagePoolDetailsDao.listDetailsKeyPairs(POOL_ID_1)).thenReturn(poolDetails1); + + Map poolDetails2 = new HashMap<>(); + poolDetails2.put(OntapStorageConstants.VOLUME_UUID, "flexvol-uuid-2"); + poolDetails2.put(OntapStorageConstants.USERNAME, "admin"); + poolDetails2.put(OntapStorageConstants.PASSWORD, "pass"); + poolDetails2.put(OntapStorageConstants.STORAGE_IP, "10.0.0.1"); + poolDetails2.put(OntapStorageConstants.SVM_NAME, "svm1"); + poolDetails2.put(OntapStorageConstants.SVM_UUID, "svm-uuid-shared"); + poolDetails2.put(OntapStorageConstants.SIZE, "107374182400"); + poolDetails2.put(OntapStorageConstants.PROTOCOL, "NFS3"); + when(storagePoolDetailsDao.listDetailsKeyPairs(POOL_ID_2)).thenReturn(poolDetails2); + + VolumeInfo volumeInfo1 = mock(VolumeInfo.class); + when(volumeInfo1.getId()).thenReturn(VOLUME_ID_1); + when(volumeDataFactory.getVolume(VOLUME_ID_1)).thenReturn(volumeInfo1); + VolumeInfo volumeInfo2 = mock(VolumeInfo.class); + when(volumeInfo2.getId()).thenReturn(VOLUME_ID_2); + when(volumeDataFactory.getVolume(VOLUME_ID_2)).thenReturn(volumeInfo2); + } + + private JobResponse createJobResponse(String uuid) { + Job job = new Job(); + job.setUuid(uuid); + JobResponse response = new JobResponse(); + response.setJob(job); + return response; + } + + private void setupSingleFlexVolFlowMocks(String snapshotName) { + doReturn(storageStrategy).when(strategy).resolveStorageStrategy(any()); + when(storageStrategy.getSnapshotFeignClient()).thenReturn(snapshotFeignClient); + when(storageStrategy.getAuthHeader()).thenReturn("Basic dGVzdDp0ZXN0"); + when(storageStrategy.jobPollForSuccess(any(), anyInt(), anyInt())).thenReturn(true); + + when(snapshotFeignClient.createSnapshot(any(), eq("flexvol-uuid-1"), any())) + .thenReturn(createJobResponse("job-fv-snap")); + + OntapResponse flexVolSnapshots = new OntapResponse<>(); + FlexVolSnapshot flexVolSnapshot = new FlexVolSnapshot(); + flexVolSnapshot.setUuid("fv-snap-uuid-1"); + flexVolSnapshot.setName(snapshotName); + flexVolSnapshots.setRecords(Collections.singletonList(flexVolSnapshot)); + when(snapshotFeignClient.getSnapshots(any(), eq("flexvol-uuid-1"), any())) + .thenReturn(flexVolSnapshots); + } + + private void setupTemporaryCgFlowMocks(String snapshotName) { + doReturn(storageStrategy).when(strategy).resolveStorageStrategy(any()); + when(storageStrategy.getSnapshotFeignClient()).thenReturn(snapshotFeignClient); + when(storageStrategy.getAuthHeader()).thenReturn("Basic dGVzdDp0ZXN0"); + when(storageStrategy.jobPollForSuccess(any(), anyInt(), anyInt())).thenReturn(true); + when(storageStrategy.pollJobIfPresentAndGetCompletedJob(any(), any())).thenAnswer(invocation -> { + Job completedJob = new Job(); + completedJob.setState(OntapStorageConstants.JOB_SUCCESS); + String operationName = invocation.getArgument(1); + if (operationName != null && operationName.startsWith("start CG snapshot")) { + completedJob.setDescription( + "POST /api/application/consistency-groups/cg-uuid-1/snapshots/cg-snap-uuid-1"); + } + return completedJob; + }); + + when(snapshotFeignClient.createConsistencyGroup(any(), any())).thenReturn(createJobResponse("job-cg-create")); + OntapResponse cgResponse = new OntapResponse<>(); + ConsistencyGroup cgRecord = new ConsistencyGroup(); + cgRecord.setUuid("cg-uuid-1"); + cgResponse.setRecords(Collections.singletonList(cgRecord)); + when(snapshotFeignClient.getConsistencyGroups(any(), any())).thenReturn(cgResponse); + + when(snapshotFeignClient.createConsistencyGroupSnapshot(any(), eq("cg-uuid-1"), any())) + .thenReturn(createJobResponse("job-cg-start")); + OntapResponse cgSnapshotResponse = new OntapResponse<>(); + ConsistencyGroupSnapshot cgSnapshotRecord = new ConsistencyGroupSnapshot(); + cgSnapshotRecord.setUuid("cg-snap-uuid-1"); + cgSnapshotRecord.setName(snapshotName); + cgSnapshotResponse.setRecords(Collections.singletonList(cgSnapshotRecord)); + when(snapshotFeignClient.getConsistencyGroupSnapshots(any(), eq("cg-uuid-1"), any())) + .thenReturn(cgSnapshotResponse); + when(snapshotFeignClient.commitConsistencyGroupSnapshot(any(), eq("cg-uuid-1"), eq("cg-snap-uuid-1"), any())) + .thenReturn(createJobResponse("job-cg-commit")); + + when(snapshotFeignClient.deleteConsistencyGroup(any(), eq("cg-uuid-1"))) + .thenReturn(createJobResponse("job-cg-delete")); + + OntapResponse flexVolSnapshots = new OntapResponse<>(); + FlexVolSnapshot flexVolSnapshot = new FlexVolSnapshot(); + flexVolSnapshot.setUuid("fv-snap-uuid-1"); + flexVolSnapshot.setName(snapshotName); + flexVolSnapshots.setRecords(Collections.singletonList(flexVolSnapshot)); + when(snapshotFeignClient.getSnapshots(any(), eq("flexvol-uuid-1"), any())) + .thenReturn(flexVolSnapshots); + + OntapResponse flexVolSnapshots2 = new OntapResponse<>(); + FlexVolSnapshot flexVolSnapshot2 = new FlexVolSnapshot(); + flexVolSnapshot2.setUuid("fv-snap-uuid-2"); + flexVolSnapshot2.setName(snapshotName); + flexVolSnapshots2.setRecords(Collections.singletonList(flexVolSnapshot2)); + when(snapshotFeignClient.getSnapshots(any(), eq("flexvol-uuid-2"), any())) + .thenReturn(flexVolSnapshots2); + } } diff --git a/private-cicd/Jenkinsfile b/private-cicd/Jenkinsfile new file mode 100644 index 000000000000..8d1dead65701 --- /dev/null +++ b/private-cicd/Jenkinsfile @@ -0,0 +1,1597 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@NonCPS +def sourceKeyFromParameters(parameters) { + def sourceMode = parameters?.getParameter('SOURCE_MODE')?.value?.toString() ?: 'pull_request' + if (sourceMode == 'branch') { + def sourceBranch = parameters?.getParameter('SOURCE_BRANCH')?.value?.toString() + return sourceBranch ? "branch:${sourceBranch}" : null + } + def prId = parameters?.getParameter('PR_ID')?.value?.toString() + return prId ? "pull_request:${prId}" : null +} + +@NonCPS +def shaFromParameters(parameters) { + def sourceMode = parameters?.getParameter('SOURCE_MODE')?.value?.toString() ?: 'pull_request' + def name = sourceMode == 'branch' ? 'SOURCE_SHA' : 'PR_HEAD_SHA' + return parameters?.getParameter(name)?.value?.toString() +} + +@NonCPS +boolean matchingPresubmitParameters(parameters, String prId, String sha) { + def sourceMode = parameters?.getParameter('SOURCE_MODE')?.value?.toString() + if (sourceMode != 'pull_request') { + return false + } + def id = parameters?.getParameter('PR_ID')?.value?.toString() + def head = parameters?.getParameter('PR_HEAD_SHA')?.value?.toString() + return id == prId && (!sha || head == sha) +} + +@NonCPS +boolean isPullRequestWorker(parameters) { + return parameters?.getParameter('SOURCE_MODE')?.value?.toString() == + 'pull_request' +} + +@NonCPS +boolean presubmitKnows(String jobName, String prId, String sha) { + def jenkins = jenkins.model.Jenkins.get() + def job = jenkins.getItemByFullName(jobName) + if (job == null) { + throw new IllegalStateException("Cannot find Jenkins job '${jobName}'.") + } + def queued = jenkins.queue.items.any { item -> + item.task?.fullName == job.fullName && + matchingPresubmitParameters( + item.getAction(hudson.model.ParametersAction), prId, sha) + } + return queued || job.getBuilds().any { build -> + matchingPresubmitParameters( + build.getAction(hudson.model.ParametersAction), prId, sha) + } +} + +@NonCPS +int activeWorkerCount(String jobName) { + def jenkins = jenkins.model.Jenkins.get() + def job = jenkins.getItemByFullName(jobName) + if (job == null) { + throw new IllegalStateException("Cannot find Jenkins job '${jobName}'.") + } + def queued = jenkins.queue.items.findAll { item -> + item.task?.fullName == job.fullName && + isPullRequestWorker(item.getAction(hudson.model.ParametersAction)) + } + def running = job.getBuilds().findAll { build -> + build.isBuilding() && + isPullRequestWorker(build.getAction(hudson.model.ParametersAction)) + } + return queued.size() + running.size() +} + +@NonCPS +String latestDiscoveryWatermark(String jobName, int buildNumber) { + def job = jenkins.model.Jenkins.get().getItemByFullName(jobName) + if (job == null) { + throw new IllegalStateException("Cannot find Jenkins job '${jobName}'.") + } + def build = job.getBuilds().find { candidate -> + if (candidate.number >= buildNumber || candidate.isBuilding()) { + return false + } + def sourceMode = candidate.getAction(hudson.model.ParametersAction) + ?.getParameter('SOURCE_MODE')?.value?.toString() + if (sourceMode != 'discover') { + return false + } + def environment = candidate.getAction( + org.jenkinsci.plugins.workflow.cps.EnvActionImpl + )?.getOverriddenEnvironment() + return environment?.get('PRESUBMIT_DISCOVERY_NEXT_WATERMARK') + } + return build?.getAction(org.jenkinsci.plugins.workflow.cps.EnvActionImpl) + ?.getOverriddenEnvironment()?.get('PRESUBMIT_DISCOVERY_NEXT_WATERMARK') +} + +// The diff number counts how many distinct commits have been presubmitted for +// this PR, so a re-run of the same commit keeps its number. Only the builds +// still kept by the build discarder can be inspected. +@NonCPS +int diffNumberForSource(String jobName, int buildNumber, String sourceKey, String sha) { + def job = jenkins.model.Jenkins.get().getItemByFullName(jobName) + if (job == null || !sourceKey) { + return 1 + } + + def previousShas = [] as Set + job.getBuilds().findAll { + it.number < buildNumber + }.each { build -> + def parameters = build.getAction(hudson.model.ParametersAction) + if (sourceKeyFromParameters(parameters) == sourceKey) { + def previousSha = shaFromParameters(parameters) + if (previousSha) { + previousShas << previousSha + } + } + } + previousShas.remove(sha) + return previousShas.size() + 1 +} + +@NonCPS +boolean hasIntegrationStarted(build) { + def environment = build.getAction( + org.jenkinsci.plugins.workflow.cps.EnvActionImpl + )?.getOverriddenEnvironment() + return environment?.get('PRESUBMIT_INTEGRATION_STARTED') == 'true' +} + +@NonCPS +def abortOlderBuildsForSource(String jobName, int buildNumber, String sourceKey) { + def job = jenkins.model.Jenkins.get().getItemByFullName(jobName) + if (job == null) { + throw new IllegalStateException("Cannot find Jenkins job '${jobName}'.") + } + + def results = [] + job.getBuilds().findAll { + it.isBuilding() && it.number < buildNumber + }.each { build -> + def parameters = build.getAction(hudson.model.ParametersAction) + if (sourceKeyFromParameters(parameters) == sourceKey) { + if (hasIntegrationStarted(build)) { + results << "Not aborting build #${build.number}: ONTAP integration has started." + } else { + build.doStop() + results << "Aborted superseded build #${build.number}." + } + } + } + return results +} + +@NonCPS +boolean containsPlaceholder(Object value) { + if (value instanceof Map) { + return value.values().any { containsPlaceholder(it) } + } + if (value instanceof Collection) { + return value.any { containsPlaceholder(it) } + } + return value != null && value.toString().contains('PLACEHOLDER-') +} + +@NonCPS +boolean isValidBranchName(String branchName) { + if (!(branchName ==~ /^[A-Za-z0-9._\/-]+$/) || + branchName.startsWith('-') || + branchName.startsWith('/') || + branchName.endsWith('/') || + branchName.endsWith('.') || + branchName.contains('..') || + branchName.contains('//') || + branchName.contains('@{')) { + return false + } + return !branchName.split('/').any { + it.isEmpty() || it.startsWith('.') || it.endsWith('.lock') + } +} + +def validateInventory(Map inventory) { + if (containsPlaceholder(inventory)) { + error('VM inventory still contains PLACEHOLDER- values.') + } + + def allVms = inventory.vms ?: [] + if (!(allVms instanceof Collection) || allVms.isEmpty()) { + error('VM inventory must contain a non-empty vms list.') + } + def enabledInventoryVms = allVms.findAll { + it.enabled == null || it.enabled.toString().toBoolean() + } + def resources = enabledInventoryVms.collect { it.lock_resource as String } + if (resources.any { !it } || resources.toSet().size() != resources.size()) { + error('Enabled inventory VMs need unique, non-empty lock_resource values.') + } + + def requiredCapabilities = ['ubuntu22', 'kvm', 'ontap'] as Set + def enabledVms = enabledInventoryVms.findAll { + def capabilities = (it.capabilities ?: []).collect { capability -> + capability as String + } as Set + capabilities.containsAll(requiredCapabilities) + } + if (enabledVms.isEmpty()) { + error('VM inventory has no enabled VM with ubuntu22, kvm, and ontap capabilities.') + } + + enabledVms.each { vm -> + ['id', 'lock_resource', 'ssh_host', 'vcenter_vm', 'snapshot', 'bridge'].each { key -> + if (!vm[key]?.toString()?.trim()) { + error("Inventory VM '${vm.id ?: vm.lock_resource ?: 'unknown'}' has no usable '${key}'.") + } + } + [ + management_server: vm.dedicated_ips?.management_server, + kvm_host: vm.dedicated_ips?.kvm_host, + public_gateway: vm.dedicated_ips?.public?.gateway, + public_netmask: vm.dedicated_ips?.public?.netmask, + public_start: vm.dedicated_ips?.public?.start, + public_end: vm.dedicated_ips?.public?.end, + pod_gateway: vm.dedicated_ips?.pod?.gateway, + pod_netmask: vm.dedicated_ips?.pod?.netmask, + pod_start: vm.dedicated_ips?.pod?.start, + pod_end: vm.dedicated_ips?.pod?.end, + zone_name: vm.zone?.name, + network_type: vm.zone?.network_type, + dns1: vm.zone?.dns1, + internal_dns1: vm.zone?.internal_dns1, + guest_cidr: vm.zone?.guest_cidr, + guest_vlan_range: vm.zone?.guest_vlan_range, + pod_name: vm.zone?.pod_name, + cluster_name: vm.zone?.cluster_name, + primary_storage_url: vm.zone?.primary_storage_url, + secondary_storage_url: vm.zone?.secondary_storage_url, + systemvm_template_url: vm.zone?.systemvm_template_url, + template_name: vm.zone?.template_name, + ontap_storage_ip: vm.ontap?.storage_ip, + ontap_svm_name: vm.ontap?.svm_name + ].each { key, value -> + if (!value?.toString()?.trim()) { + error("Inventory VM '${vm.id}' has no usable '${key}'.") + } + } + } + return enabledVms +} + +// Discovery only needs list-eligible-prs.py, so it fetches one commit without +// tags and checks out only private-cicd/scripts. A full checkout of CloudStack +// takes minutes and makes each poll outlast the next timer trigger. +// The GitSCM getters are sandboxed, so fall back to the full checkout when the +// controller has not approved them yet. +void checkoutDiscoveryScripts() { + try { + checkout([ + $class: 'GitSCM', + branches: scm.branches, + userRemoteConfigs: scm.userRemoteConfigs, + extensions: [ + [$class: 'CloneOption', shallow: true, depth: 1, noTags: true], + [ + $class: 'SparseCheckoutPaths', + sparseCheckoutPaths: [[path: 'private-cicd/scripts']] + ] + ] + ]) + } catch (Exception e) { + echo "Shallow discovery checkout unavailable, using the full checkout: ${e}" + checkout(scm) + } +} + +// Optional manual gate after a stage, enabled by PAUSE_BETWEEN_STAGES. +// The pod stays reserved while an input waits, so the timeout is required. +void pauseForVerification(String stageName) { + if (!params.PAUSE_BETWEEN_STAGES) { + return + } + timeout(time: 60, unit: 'MINUTES') { + input(message: "Finished '${stageName}'. Proceed to the next stage?", ok: 'Proceed') + } +} + +void recordStageEvent(String stageName, String status) { + try { + withEnv([ + "PRESUBMIT_STAGE_NAME=${stageName}", + "PRESUBMIT_STAGE_STATUS=${status}" + ]) { + sh ''' + stage_events="${WORKSPACE}@tmp/presubmit-stage-events.tsv" + mkdir -p "$(dirname "$stage_events")" + printf '%s\t%s\t%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + "$PRESUBMIT_STAGE_NAME" "$PRESUBMIT_STAGE_STATUS" >> "$stage_events" + ''' + } + } catch (Exception e) { + echo "Unable to record presubmit stage event: ${e}" + } +} + +String presubmitSubjectBase() { + def recipientName = params.PR_AUTHOR_LOGIN?.trim() + if (!recipientName && env.PRESUBMIT_MAIL_TO?.contains('@')) { + recipientName = env.PRESUBMIT_MAIL_TO.substring(0, env.PRESUBMIT_MAIL_TO.indexOf('@')) + } + def sourceName = params.SOURCE_MODE == 'branch' ? + "branch ${params.SOURCE_BRANCH}" : "PR-${params.PR_ID}" + def title = params.PR_TITLE?.replaceAll(/[\r\n]+/, ' ')?.trim() + if (title?.length() > 80) { + title = title.take(77) + '...' + } + def titlePart = title ? " ${title}" : '' + def mention = recipientName ? "@${recipientName}: " : '' + def diffNumber = env.PRESUBMIT_DIFF_NUMBER?.trim() + def diffPart = diffNumber ? + "diff #${diffNumber} (${env.RESOLVED_SOURCE_SHA.take(12)})" : + "diff ${env.RESOLVED_SOURCE_SHA.take(12)}" + return "${mention}CloudStack presubmit ${sourceName}${titlePart}, ${diffPart}" +} + +void sendPresubmitMail(String subject, String body) { + def recipient = env.PRESUBMIT_MAIL_TO?.trim() + if (!recipient) { + echo 'Presubmit mail skipped: no recipient could be resolved.' + return + } + try { + emailext( + subject: subject, + body: body, + mimeType: 'text/html', + to: recipient + ) + } catch (Exception e) { + echo "Presubmit mail was not sent: ${e}" + } +} + +boolean hasGithubAppCredential() { + def credentialId = params.GITHUB_APP_CREDENTIALS_ID?.trim() + return credentialId && !credentialId.startsWith('PLACEHOLDER-') +} + +List githubEnvironment() { + return [ + "GITHUB_REPOSITORY=${params.PR_REPOSITORY}", + "GITHUB_HEAD_SHA=${params.PR_HEAD_SHA}", + "PR_ID=${params.PR_ID}", + "PR_TITLE=${params.PR_TITLE ?: ''}" + ] +} + +void createGithubCheck() { + if (params.SOURCE_MODE != 'pull_request' || !hasGithubAppCredential()) { + echo 'GitHub Check creation skipped: GitHub App credential is not configured.' + return + } + try { + withCredentials([ + usernamePassword( + credentialsId: params.GITHUB_APP_CREDENTIALS_ID, + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + ) + ]) { + withEnv(githubEnvironment()) { + def checkRunId = sh( + script: 'private-cicd/scripts/github-check-run.sh create', + returnStdout: true + ).trim() + if (!(checkRunId ==~ /^[0-9]+$/)) { + throw new IllegalStateException("GitHub returned an invalid Check Run ID '${checkRunId}'.") + } + env.GITHUB_CHECK_RUN_ID = checkRunId + echo "Created GitHub Check Run ${checkRunId} on ${params.PR_HEAD_SHA}." + } + } + } catch (Exception e) { + // A missing check cannot be noticed from the build result, because + // reporting deliberately never changes it. Surface it on the job page. + currentBuild.description = ( + (currentBuild.description ? currentBuild.description + ' | ' : '') + + 'GitHub Check not published' + ) + echo "GitHub Check creation failed; presubmit will continue: ${e}" + echo 'Run this job with SOURCE_MODE=diagnose and the same PR_HEAD_SHA ' + + 'to identify the cause.' + } +} + +void completeGithubCheck(String conclusion) { + if (!env.GITHUB_CHECK_RUN_ID?.trim() || !hasGithubAppCredential()) { + return + } + try { + withCredentials([ + usernamePassword( + credentialsId: params.GITHUB_APP_CREDENTIALS_ID, + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + ) + ]) { + withEnv(githubEnvironment() + [ + "GITHUB_CHECK_CONCLUSION=${conclusion}" + ]) { + sh 'private-cicd/scripts/github-check-run.sh complete' + echo "Completed GitHub Check Run ${env.GITHUB_CHECK_RUN_ID} with conclusion ${conclusion}." + } + } + } catch (Exception e) { + echo "GitHub Check completion failed; presubmit result is unchanged: ${e}" + } +} + +void resolvePresubmitMailRecipient() { + def webhookEmail = params.PR_AUTHOR_EMAIL?.trim() + env.PRESUBMIT_MAIL_TO = webhookEmail?.contains('@') && + !webhookEmail.equalsIgnoreCase('null') ? webhookEmail : '' + if (env.PRESUBMIT_MAIL_TO || params.SOURCE_MODE != 'pull_request' || + !hasGithubAppCredential()) { + return + } + try { + withCredentials([ + usernamePassword( + credentialsId: params.GITHUB_APP_CREDENTIALS_ID, + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + ) + ]) { + withEnv(githubEnvironment()) { + def commitEmail = sh( + script: 'private-cicd/scripts/github-check-run.sh commit-email', + returnStdout: true + ).trim() + if (commitEmail && !commitEmail.endsWith('@users.noreply.github.com')) { + env.PRESUBMIT_MAIL_TO = commitEmail + } + } + } + } catch (Exception e) { + echo "Unable to resolve PR author email from GitHub: ${e}" + } +} + +String htmlEscape(String value) { + return (value ?: '') + .replace('&', '&') + .replace('<', '<') + .replace('>', '>') + .replace('"', '"') + .replace("'", ''') +} + +// Mirrors the table styling of scripts/generate-presubmit-report.py so the +// start mail and the final report mail look like the same document. +String presubmitInfoRow(String label, String valueHtml) { + def labelStyle = 'text-align:left;background:#f2f4f7;' + + 'border:1px solid #d0d5dd;padding:8px' + def valueStyle = 'border:1px solid #d0d5dd;padding:8px' + return "${htmlEscape(label)}" + + "${valueHtml}" +} + +String presubmitStartHtml() { + def source = params.SOURCE_MODE == 'branch' ? + "branch ${params.SOURCE_BRANCH}" : "PR-${params.PR_ID}" + def sourceHtml = params.PR_URL ==~ /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/pull\/[0-9]+$/ ? + "${htmlEscape(source)}" : + htmlEscape(source) + def diffNumber = env.PRESUBMIT_DIFF_NUMBER?.trim() + def rows = presubmitInfoRow('Source', sourceHtml) + + presubmitInfoRow('Title', htmlEscape(params.PR_TITLE ?: 'not provided')) + if (diffNumber) { + rows += presubmitInfoRow('Diff', "#${htmlEscape(diffNumber)}") + } + rows += presubmitInfoRow( + 'Commit', "${htmlEscape(env.RESOLVED_SOURCE_SHA)}") + rows += presubmitInfoRow('Started', htmlEscape(env.PRESUBMIT_STARTED_AT)) + def consoleUrl = "${env.BUILD_URL}console" + return """
+

CloudStack ONTAP presubmit

+

STARTED

+ +${rows} +
+

Jenkins build · +Console

+

A final mail with the stage table, stage durations, and the ONTAP test +results follows when this run finishes.

+
""" +} + +// Stage bodies live in the methods below rather than inline in the stages +// further down. The JVM caps one method at 64KB of bytecode and Jenkins +// compiles the whole body of the pipeline directive into a single method, so +// inlining these again reintroduces JENKINS-37984 +// ("Method too large: WorkflowScript.___cps___NNNN"). +// These methods call steps, so none of them may be @NonCPS. + +// Worker stages run only for a real build request, so discovery and the +// GitHub App diagnostic skip them, as does a request that `Check eligibility` +// rejected by setting NOT_BUILT. +boolean isWorkerStageEnabled() { + return params.SOURCE_MODE in ['pull_request', 'branch'] && + currentBuild.result != 'NOT_BUILT' +} + +boolean isDiagnoseRun() { + return params.SOURCE_MODE == 'diagnose' +} + +void discoverPullRequests() { + currentBuild.displayName = "#${env.BUILD_NUMBER} discover" + // Skipping instead of queueing keeps a slow poll from building + // a backlog of discovery runs that each hold an agent pod. + lock(resource: 'cloudstack-presubmit-discovery', skipIfLocked: true) { + env.PRESUBMIT_DISCOVERY_RAN = 'true' + deleteDir() + checkoutDiscoveryScripts() + if (!(params.EXPECTED_REPOSITORY ==~ /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/)) { + error('EXPECTED_REPOSITORY must be owner/repository.') + } + if (!isValidBranchName(params.DISCOVERY_BASE_BRANCH)) { + error('DISCOVERY_BASE_BRANCH is not a valid Git branch name.') + } + if (!(params.GITHUB_APP_CREDENTIALS_ID ==~ /^[A-Za-z0-9._-]+$/)) { + error('GITHUB_APP_CREDENTIALS_ID contains unsupported characters.') + } + if (!(params.MAX_ACTIVE_WORKERS ==~ /^[1-9][0-9]*$/)) { + error('MAX_ACTIVE_WORKERS must be a positive integer.') + } + def cap = params.MAX_ACTIVE_WORKERS.toInteger() + def previousWatermark = latestDiscoveryWatermark( + env.JOB_NAME, + env.BUILD_NUMBER.toInteger() + ) + def isInitialDiscovery = !previousWatermark + def thisStart = new Date(currentBuild.startTimeInMillis).format( + "yyyy-MM-dd'T'HH:mm:ssXXX", + TimeZone.getTimeZone('UTC') + ) + def watermark = previousWatermark ?: thisStart + withCredentials([ + usernamePassword( + credentialsId: params.GITHUB_APP_CREDENTIALS_ID, + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + ) + ]) { + withEnv([ + "DISCOVERY_REPOSITORY=${params.EXPECTED_REPOSITORY}", + "DISCOVERY_BASE_BRANCH=${params.DISCOVERY_BASE_BRANCH}", + "DISCOVERY_UPDATED_AFTER=${watermark}" + ]) { + sh ''' + python3 private-cicd/scripts/list-eligible-prs.py \ + --repository "$DISCOVERY_REPOSITORY" \ + --base-branch "$DISCOVERY_BASE_BRANCH" \ + --updated-after "$DISCOVERY_UPDATED_AFTER" \ + > eligible-prs.json + ''' + } + } + def pulls = readJSON(file: 'eligible-prs.json') + if (!(pulls instanceof List)) { + error('eligible-prs.json is not a JSON array.') + } + if (isInitialDiscovery) { + echo "Seeded discovery watermark at ${watermark}; existing PRs were not queued." + pulls = [] + } + int queuedCount = 0 + int deferredCount = 0 + int knownActive = cap + try { + knownActive = activeWorkerCount(env.JOB_NAME) + } catch (Exception e) { + echo "Unable to inspect the worker queue; discovery will not queue new workers: ${e}" + } + pulls.each { pull -> + def prId = pull.pr_id?.toString() + def sha = pull.pr_head_sha?.toString() + if (!(prId ==~ /^[0-9]+$/) || + !(sha ==~ /^[0-9a-fA-F]{40}$/)) { + echo "Skipping malformed pull request payload: ${pull}" + return + } + if (presubmitKnows(env.JOB_NAME, prId, sha)) { + echo "PR-${prId} ${sha.take(12)} already has a queued or recorded presubmit." + return + } + if (knownActive + queuedCount >= cap) { + echo "Deferred PR-${prId} ${sha.take(12)}: ${knownActive + queuedCount} active workers at cap ${cap}." + deferredCount++ + return + } + def action = presubmitKnows( + env.JOB_NAME, prId, '') ? + 'synchronize' : 'opened' + echo "Queueing PR-${prId} ${action} at ${sha.take(12)}." + build( + job: env.JOB_NAME, + wait: false, + propagate: false, + parameters: [ + string(name: 'SOURCE_MODE', value: 'pull_request'), + string(name: 'PR_ID', value: prId), + string(name: 'PR_ACTION', value: action), + string(name: 'PR_DRAFT', value: 'false'), + string(name: 'PR_REPOSITORY', value: params.EXPECTED_REPOSITORY), + string(name: 'PR_BASE_BRANCH', value: params.DISCOVERY_BASE_BRANCH), + string(name: 'PR_HEAD_BRANCH', value: pull.pr_head_branch ?: ''), + string(name: 'PR_HEAD_SHA', value: sha), + string(name: 'PR_AUTHOR_LOGIN', value: pull.pr_author_login ?: ''), + string(name: 'PR_AUTHOR_EMAIL', value: pull.pr_author_email ?: ''), + string(name: 'PR_URL', value: pull.pr_url ?: ''), + string(name: 'PR_TITLE', value: pull.pr_title ?: '') + ] + ) + queuedCount++ + } + env.PRESUBMIT_DISCOVERY_NEXT_WATERMARK = + deferredCount > 0 ? watermark : thisStart + currentBuild.displayName = + "#${env.BUILD_NUMBER} discover: queued ${queuedCount}, deferred ${deferredCount}" + currentBuild.description = isInitialDiscovery ? + "seeded at ${watermark}" : "since ${watermark}" + // Jenkins treats NOT_BUILT as worse than FAILURE and + // never improves a result, so this must stay last. + // Setting it earlier hides a broken discovery run. + currentBuild.result = 'NOT_BUILT' + } + if (env.PRESUBMIT_DISCOVERY_RAN != 'true') { + currentBuild.displayName = + "#${env.BUILD_NUMBER} discover: skipped, lock held" + currentBuild.result = 'NOT_BUILT' + } +} + +void validateSourceRequest() { + sh 'rm -f "${WORKSPACE}@tmp/presubmit-stage-events.tsv"' + env.PRESUBMIT_INTEGRATION_STARTED = 'false' + env.PRESUBMIT_STARTED_AT = new Date().format( + "yyyy-MM-dd'T'HH:mm:ssXXX", + TimeZone.getTimeZone('UTC') + ) + recordStageEvent('Validate source request', 'STARTED') + if (!(params.SOURCE_MODE in ['pull_request', 'branch'])) { + error("Unsupported SOURCE_MODE '${params.SOURCE_MODE}'.") + } + if (params.SOURCE_MODE == 'branch') { + if (!isValidBranchName(params.SOURCE_BRANCH)) { + error('SOURCE_BRANCH is not a valid Git branch name.') + } + if (!(params.SOURCE_SHA ==~ /^[0-9a-fA-F]{40}$/)) { + error('SOURCE_SHA must be a full 40-character Git SHA.') + } + } else { + if (!(params.PR_ID ==~ /^[0-9]+$/)) { + error('PR_ID must be the numeric GitHub pull request number.') + } + if (params.PR_REPOSITORY != params.EXPECTED_REPOSITORY) { + error("Repository '${params.PR_REPOSITORY}' is not '${params.EXPECTED_REPOSITORY}'.") + } + if (!(params.PR_HEAD_BRANCH ==~ /^[A-Za-z0-9._\/-]+$/)) { + error('PR_HEAD_BRANCH contains unsupported characters.') + } + if (!(params.PR_HEAD_SHA ==~ /^[0-9a-fA-F]{40}$/) && params.PR_ACTION != 'closed') { + error('PR_HEAD_SHA must be a full 40-character Git SHA.') + } + } + if (!(params.CLOUDSTACK_GIT_URL ==~ /^https:\/\/[A-Za-z0-9._\/-]+\.git$/)) { + error('CLOUDSTACK_GIT_URL must be an HTTPS Git URL.') + } + if (params.GIT_CREDENTIALS_ID && + !(params.GIT_CREDENTIALS_ID ==~ /^[A-Za-z0-9._-]+$/)) { + error('GIT_CREDENTIALS_ID contains unsupported characters.') + } + if (!(params.VCENTER_HOST ==~ /^[A-Za-z0-9.-]+$/)) { + error('VCENTER_HOST contains unsupported characters.') + } + [ + params.GITHUB_APP_CREDENTIALS_ID, + params.VM_INVENTORY_CREDENTIALS_ID, + params.VCENTER_CREDENTIALS_ID, + params.VM_SSH_CREDENTIALS_ID, + params.MYSQL_ROOT_CREDENTIALS_ID, + params.CLOUD_DB_CREDENTIALS_ID, + params.KVM_HOST_CREDENTIALS_ID, + params.ONTAP_CREDENTIALS_ID, + params.CLOUDSTACK_ADMIN_CREDENTIALS_ID + ].each { credentialId -> + if (!(credentialId ==~ /^[A-Za-z0-9._-]+$/)) { + error("Credential ID contains unsupported characters: ${credentialId}") + } + } + try { + def sourceKey = params.SOURCE_MODE == 'branch' ? + "branch:${params.SOURCE_BRANCH}" : + "pull_request:${params.PR_ID}" + env.PRESUBMIT_DIFF_NUMBER = diffNumberForSource( + env.JOB_NAME, + env.BUILD_NUMBER.toInteger(), + sourceKey, + env.RESOLVED_SOURCE_SHA + ).toString() + } catch (Exception e) { + env.PRESUBMIT_DIFF_NUMBER = '' + echo "Unable to determine the diff number: ${e}" + } + // The diff number goes in the title so an author can pick + // their own run out of a build history dominated by + // five-minute discovery runs. + def diffLabel = env.PRESUBMIT_DIFF_NUMBER ? + " diff #${env.PRESUBMIT_DIFF_NUMBER}" : '' + if (params.SOURCE_MODE == 'branch') { + currentBuild.displayName = + "#${env.BUILD_NUMBER} branch ${params.SOURCE_BRANCH}${diffLabel}" + currentBuild.description = "Branch ${params.SOURCE_BRANCH} @ ${params.SOURCE_SHA.take(12)}" + } else { + currentBuild.displayName = + "#${env.BUILD_NUMBER} PR-${params.PR_ID}${diffLabel} ${params.PR_ACTION}" + currentBuild.description = + "${env.RESOLVED_SOURCE_SHA?.take(12)} ${params.PR_TITLE ?: params.PR_URL}" + } +} + +void checkPresubmitEligibility() { + recordStageEvent('Check eligibility', 'STARTED') + if (params.SOURCE_MODE == 'pull_request') { + def eligibleActions = ['opened', 'synchronize', 'reopened', 'ready_for_review'] + if (params.PR_ACTION == 'closed' || + params.PR_DRAFT.toBoolean() || + params.PR_BASE_BRANCH != 'main' || + !(params.PR_ACTION in eligibleActions)) { + currentBuild.result = 'NOT_BUILT' + echo "Ignoring PR-${params.PR_ID}: action=${params.PR_ACTION}, draft=${params.PR_DRAFT}, base=${params.PR_BASE_BRANCH}" + } + } +} + +void abortSupersededRun() { + recordStageEvent('Abort superseded run', 'STARTED') + def sourceKey = params.SOURCE_MODE == 'branch' ? + "branch:${params.SOURCE_BRANCH}" : + "pull_request:${params.PR_ID}" + try { + abortOlderBuildsForSource( + env.JOB_NAME, + env.BUILD_NUMBER.toInteger(), + sourceKey + ).each { result -> + echo result + } + } catch (Exception e) { + echo "Unable to inspect older builds; no build was aborted: ${e}" + } +} + +void checkoutCiScripts() { + recordStageEvent('Checkout CI scripts', 'STARTED') + deleteDir() + checkout(scm) +} + +// A full worker run takes over an hour, so reporting problems are diagnosed +// with this short build instead of a rebuild of the whole presubmit. +void diagnoseGithubApp() { + currentBuild.displayName = "#${env.BUILD_NUMBER} diagnose GitHub App" + if (!hasGithubAppCredential()) { + error('GITHUB_APP_CREDENTIALS_ID is unset or still a placeholder.') + } + def headSha = params.PR_HEAD_SHA?.trim() ?: params.SOURCE_SHA?.trim() + if (!(headSha ==~ /^[0-9a-f]{40}$/)) { + error('Set PR_HEAD_SHA to a full 40-character commit SHA to diagnose.') + } + def repository = params.PR_REPOSITORY?.trim() ?: params.EXPECTED_REPOSITORY + withCredentials([ + usernamePassword( + credentialsId: params.GITHUB_APP_CREDENTIALS_ID, + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + ) + ]) { + withEnv([ + "GITHUB_REPOSITORY=${repository}", + "GITHUB_HEAD_SHA=${headSha}", + "PR_ID=${params.PR_ID ?: ''}" + ]) { + sh 'private-cicd/scripts/github-check-run.sh diagnose' + } + } +} + +void startPullRequestReporting() { + recordStageEvent('Start PR reporting', 'STARTED') + createGithubCheck() + resolvePresubmitMailRecipient() + sendPresubmitMail( + "${presubmitSubjectBase()}: STARTED", + presubmitStartHtml() + ) +} + +void validateBuilder() { + recordStageEvent('Validate builder', 'STARTED') + def required = [ + VCENTER_HOST : params.VCENTER_HOST, + VM_INVENTORY_CREDENTIALS_ID : params.VM_INVENTORY_CREDENTIALS_ID, + VCENTER_CREDENTIALS_ID : params.VCENTER_CREDENTIALS_ID, + VM_SSH_CREDENTIALS_ID : params.VM_SSH_CREDENTIALS_ID, + MYSQL_ROOT_CREDENTIALS_ID : params.MYSQL_ROOT_CREDENTIALS_ID, + CLOUD_DB_CREDENTIALS_ID : params.CLOUD_DB_CREDENTIALS_ID, + KVM_HOST_CREDENTIALS_ID : params.KVM_HOST_CREDENTIALS_ID, + ONTAP_CREDENTIALS_ID : params.ONTAP_CREDENTIALS_ID, + CLOUDSTACK_ADMIN_CREDENTIALS_ID : params.CLOUDSTACK_ADMIN_CREDENTIALS_ID + ] + required.each { key, value -> + if (!value?.trim() || value.startsWith('PLACEHOLDER-')) { + error("${key} is missing or still uses a placeholder.") + } + } + withCredentials([ + file( + credentialsId: params.VM_INVENTORY_CREDENTIALS_ID, + variable: 'BOUND_VM_INVENTORY_FILE' + ) + ]) { + def inventory = readYaml(file: env.BOUND_VM_INVENTORY_FILE) ?: [:] + validateInventory(inventory) + echo 'VM inventory Secret file parsed and validated before lock acquisition.' + } + container('cloudstack-driver') { + sh 'private-cicd/scripts/check-build-prereqs.sh' + } + pauseForVerification('Validate builder') +} + +void checkoutSource() { + recordStageEvent('Checkout source', 'STARTED') + dir(env.SOURCE_DIR) { + deleteDir() + def isBranch = params.SOURCE_MODE == 'branch' + def sourceBranch = isBranch ? params.SOURCE_BRANCH : params.PR_HEAD_BRANCH + def sourceSha = isBranch ? params.SOURCE_SHA : params.PR_HEAD_SHA + def sourceLabel = isBranch ? "branch:${sourceBranch}" : "pull_request:${params.PR_ID}" + def remoteRef = isBranch ? + "refs/remotes/origin/source/${sourceBranch}" : + "refs/remotes/origin/pr/${params.PR_ID}" + def remote = [ + url: params.CLOUDSTACK_GIT_URL, + refspec: isBranch ? + "+refs/heads/${sourceBranch}:${remoteRef}" : + "+refs/pull/${params.PR_ID}/head:${remoteRef}" + ] + if (params.GIT_CREDENTIALS_ID?.trim()) { + remote.credentialsId = params.GIT_CREDENTIALS_ID + } + checkout([ + $class: 'GitSCM', + branches: [[name: sourceSha]], + doGenerateSubmoduleConfigurations: false, + extensions: [[ + $class: 'CloneOption', + honorRefspec: true, + noTags: true, + shallow: false + ]], + userRemoteConfigs: [remote] + ]) + withEnv([ + "EXPECTED_SOURCE_SHA=${sourceSha}", + "REMOTE_SOURCE_REF=${remoteRef}", + "SOURCE_LABEL=${sourceLabel}", + "RESOLVED_SOURCE_BRANCH=${sourceBranch}" + ]) { + sh ''' + mkdir -p "$WORKSPACE/$PRESUBMIT_RESULTS" + actual_sha="$(git rev-parse HEAD)" + if [ "$actual_sha" != "$EXPECTED_SOURCE_SHA" ]; then + echo "Checked out $actual_sha instead of $EXPECTED_SOURCE_SHA" >&2 + exit 1 + fi + if ! git merge-base --is-ancestor "$EXPECTED_SOURCE_SHA" "$REMOTE_SOURCE_REF"; then + echo "$EXPECTED_SOURCE_SHA is not reachable from $REMOTE_SOURCE_REF" >&2 + exit 1 + fi + if [ -n "$(git status --porcelain)" ]; then + echo "Source checkout is not clean" >&2 + git status --short >&2 + exit 1 + fi + printf 'SOURCE_MODE=%s\\nSOURCE_LABEL=%s\\nSOURCE_BRANCH=%s\\nSOURCE_SHA=%s\\n' \ + "$SOURCE_MODE" "$SOURCE_LABEL" "$RESOLVED_SOURCE_BRANCH" "$actual_sha" \ + > "$WORKSPACE/$PRESUBMIT_RESULTS/source.properties" + ''' + } + } + pauseForVerification('Checkout source') +} + +void runUnitTests() { + recordStageEvent('Build and unit tests', 'STARTED') + container('cloudstack-driver') { + sh ''' + SOURCE_DIR="$WORKSPACE/$SOURCE_DIR" \ + RESULT_DIR="$WORKSPACE/$PRESUBMIT_RESULTS/unit-tests" \ + SOURCE_ID="$RESOLVED_SOURCE_ID" \ + SOURCE_SHA="$RESOLVED_SOURCE_SHA" \ + ENABLE_SIMULATOR=false \ + MAVEN_THREADS=1C \ + MAVEN_OPTS="-Xms1g -Xmx8g" \ + private-cicd/scripts/mvn-full.sh + ''' + } + pauseForVerification('Build and unit tests') +} + +void buildDebianPackages() { + recordStageEvent('Build Debian packages', 'STARTED') + container('cloudstack-driver') { + sh ''' + SOURCE_DIR="$WORKSPACE/$SOURCE_DIR" \ + RESULT_DIR="$WORKSPACE/$SOURCE_DIR/dist/deb-all" \ + REUSE_MAVEN_BUILD=true \ + MAVEN_OPTS="-Xms1g -Xmx8g" \ + private-cicd/scripts/build-debs.sh + ''' + } + pauseForVerification('Build Debian packages') +} + +void deployAndRunOntapIntegration() { + recordStageEvent('Deploy and run ONTAP integration', 'WAITING_FOR_VM') + withCredentials([ + file( + credentialsId: params.VM_INVENTORY_CREDENTIALS_ID, + variable: 'BOUND_VM_INVENTORY_FILE' + ) + ]) { + // The lock step declares `resource` as mandatory, so label-based + // locking must pass it explicitly as null. + lock(resource: null, label: 'cloudstack-presubmit-vm', quantity: 1, variable: 'LOCKED_VM') { + env.PRESUBMIT_INTEGRATION_STARTED = 'true' + recordStageEvent('Deploy and run ONTAP integration', 'STARTED') + container('cloudstack-driver') { + def inventory = readYaml(file: env.BOUND_VM_INVENTORY_FILE) ?: [:] + def enabledVms = validateInventory(inventory) + def vm = enabledVms.find { + (it.lock_resource as String) == env.LOCKED_VM + } + if (vm == null) { + error("Locked resource '${env.LOCKED_VM}' is not a compatible enabled inventory VM.") + } + ['id', 'ssh_host', 'vcenter_vm', 'snapshot', 'bridge'].each { key -> + if (!vm[key] || (vm[key] as String).startsWith('PLACEHOLDER-')) { + error("Inventory VM '${env.LOCKED_VM}' has no usable '${key}'.") + } + } + + env.VM_ID = vm.id as String + env.VM_HOST = vm.ssh_host as String + env.VM_SSH_USER = (vm.ssh_user ?: 'root') as String + env.VM_NAME = vm.vcenter_vm as String + env.VM_SNAPSHOT = vm.snapshot as String + env.VM_BRIDGE = vm.bridge as String + def sourceDescription = params.SOURCE_MODE == 'branch' ? + "branch=${params.SOURCE_BRANCH}" : + "PR-${params.PR_ID}" + currentBuild.description = "${sourceDescription} VM=${env.VM_ID}" + echo "Reserved compatible VM '${env.VM_ID}' via '${env.LOCKED_VM}'." + + withCredentials([ + usernamePassword( + credentialsId: params.VCENTER_CREDENTIALS_ID, + usernameVariable: 'VCENTER_USER', + passwordVariable: 'VCENTER_PASSWORD' + ) + ]) { + sh ''' + python3 private-cicd/scripts/vcenter-revert.py \ + --host "$VCENTER_HOST" \ + --vm "$VM_NAME" \ + --snapshot "$VM_SNAPSHOT" \ + --power-on \ + --insecure + ''' + } + + // This gate waits while the VM lock is held, so no other + // build can reserve the VM until it is released. + pauseForVerification('Snapshot revert') + + withCredentials([ + usernamePassword( + credentialsId: params.VM_SSH_CREDENTIALS_ID, + usernameVariable: 'VM_SSH_USER', + passwordVariable: 'VM_SSH_PASSWORD' + ), + usernamePassword( + credentialsId: params.MYSQL_ROOT_CREDENTIALS_ID, + usernameVariable: 'MYSQL_ROOT_USER', + passwordVariable: 'MYSQL_ROOT_PASSWORD' + ), + usernamePassword( + credentialsId: params.CLOUD_DB_CREDENTIALS_ID, + usernameVariable: 'CLOUD_DB_USER', + passwordVariable: 'CLOUD_DB_PASSWORD' + ), + usernamePassword( + credentialsId: params.KVM_HOST_CREDENTIALS_ID, + usernameVariable: 'KVM_HOST_USER', + passwordVariable: 'KVM_HOST_PASSWORD' + ), + usernamePassword( + credentialsId: params.ONTAP_CREDENTIALS_ID, + usernameVariable: 'ONTAP_USER', + passwordVariable: 'ONTAP_PASSWORD' + ), + usernamePassword( + credentialsId: params.CLOUDSTACK_ADMIN_CREDENTIALS_ID, + usernameVariable: 'CLOUDSTACK_ADMIN_USER', + passwordVariable: 'CLOUDSTACK_ADMIN_PASSWORD' + ) + ]) { + sh ''' + SOURCE_DIR="$WORKSPACE/$SOURCE_DIR" \ + DEB_DIR="$WORKSPACE/$SOURCE_DIR/dist/deb-all" \ + RESULT_DIR="$WORKSPACE/$PRESUBMIT_RESULTS/phase2" \ + INVENTORY_FILE="$BOUND_VM_INVENTORY_FILE" \ + SOURCE_ID="$RESOLVED_SOURCE_ID" \ + SOURCE_SHA="$RESOLVED_SOURCE_SHA" \ + BRIDGE="$VM_BRIDGE" \ + private-cicd/scripts/run-phase2-remote.sh + ''' + } + } + } + } +} + +void publishPresubmitResults() { + if (params.SOURCE_MODE == 'discover') { + echo 'Discovery build completed; worker reporting was skipped.' + return + } + if (isDiagnoseRun()) { + echo 'Diagnostic build completed; no mail, Check or test reporting.' + return + } + junit( + testResults: 'presubmit-results/unit-tests/surefire/**/*.xml', + allowEmptyResults: true, + keepLongStdio: true + ) + def result = currentBuild.currentResult ?: 'FAILURE' + def conclusion = result == 'SUCCESS' ? 'success' : + (result == 'ABORTED' ? 'cancelled' : 'failure') + if (result != 'NOT_BUILT') { + env.PRESUBMIT_FINISHED_AT = new Date().format( + "yyyy-MM-dd'T'HH:mm:ssXXX", + TimeZone.getTimeZone('UTC') + ) + completeGithubCheck(conclusion) + if (!env.PRESUBMIT_MAIL_TO?.trim()) { + resolvePresubmitMailRecipient() + } + def sourceName = params.SOURCE_MODE == 'branch' ? + "branch ${params.SOURCE_BRANCH}" : "PR-${params.PR_ID}" + try { + if (fileExists('private-cicd/scripts/generate-presubmit-report.py')) { + withEnv([ + "PRESUBMIT_REPORT_RESULT=${result}", + "PRESUBMIT_REPORT_SOURCE=${sourceName}", + "PRESUBMIT_REPORT_SOURCE_URL=${params.PR_URL ?: ''}", + "PRESUBMIT_REPORT_TITLE=${params.PR_TITLE ?: ''}", + "PRESUBMIT_REPORT_SHA=${env.RESOLVED_SOURCE_SHA}", + "PRESUBMIT_REPORT_DIFF=${env.PRESUBMIT_DIFF_NUMBER ?: ''}", + "PRESUBMIT_REPORT_VM=${env.VM_ID ?: ''}" + ]) { + sh ''' + python3 private-cicd/scripts/generate-presubmit-report.py \ + --results "$PRESUBMIT_RESULTS" \ + --stage-events "${WORKSPACE}@tmp/presubmit-stage-events.tsv" \ + --build-url "$BUILD_URL" \ + --result "$PRESUBMIT_REPORT_RESULT" \ + --source "$PRESUBMIT_REPORT_SOURCE" \ + --source-url "$PRESUBMIT_REPORT_SOURCE_URL" \ + --title "$PRESUBMIT_REPORT_TITLE" \ + --source-sha "$PRESUBMIT_REPORT_SHA" \ + --diff-number "$PRESUBMIT_REPORT_DIFF" \ + --vm "$PRESUBMIT_REPORT_VM" \ + --started-at "$PRESUBMIT_STARTED_AT" \ + --finished-at "$PRESUBMIT_FINISHED_AT" \ + --debian-log "$SOURCE_DIR/dist/deb-all/deb-build.log" + ''' + } + } else { + echo 'HTML report generator was unavailable; sending a minimal final mail.' + } + } catch (Exception e) { + echo "Unable to generate HTML presubmit report: ${e}" + } + try { + archiveArtifacts( + artifacts: 'presubmit-results/**/*, cloudstack-src/dist/deb-all/**/*', + allowEmptyArchive: true, + fingerprint: true + ) + } catch (Exception e) { + echo "Unable to archive presubmit artifacts: ${e}" + } + def failedStage = fileExists("${env.PRESUBMIT_RESULTS}/failed-stage.txt") ? + readFile("${env.PRESUBMIT_RESULTS}/failed-stage.txt").trim() : '' + def finalSubject = failedStage ? + "${presubmitSubjectBase()}: ${result} in ${failedStage}" : + "${presubmitSubjectBase()}: ${result}" + def mailBody = fileExists("${env.PRESUBMIT_RESULTS}/email-body.html") ? + readFile("${env.PRESUBMIT_RESULTS}/email-body.html") : + """

CloudStack ONTAP presubmit: ${htmlEscape(result)}

+

Source: ${htmlEscape(sourceName)}

+

Commit: ${htmlEscape(env.RESOLVED_SOURCE_SHA)}

+

Open Jenkins build and logs

""" + sendPresubmitMail( + finalSubject, + mailBody + ) + } +} + +pipeline { + agent { + kubernetes { + defaultContainer 'cloudstack-driver' + yaml ''' +apiVersion: v1 +kind: Pod +spec: + containers: + - name: jnlp + image: docker.repo.eng.netapp.com/global/devts-daas-prod/jnlp/jnlp-slave:latest + resources: + requests: + memory: 512Mi + cpu: 100m + limits: + memory: 1Gi + cpu: 500m + - name: cloudstack-driver + image: docker.repo.eng.netapp.com/slocharl/cloudstack-presubmit-driver:v202608261233 + command: [sleep] + args: [infinity] + resources: + requests: + memory: 1Gi + cpu: "500m" + ephemeral-storage: 5Gi + limits: + memory: 8Gi + cpu: "2" + ephemeral-storage: 20Gi +''' + } + } + + options { + buildDiscarder(logRotator(numToKeepStr: '500')) + skipDefaultCheckout() + timestamps() + timeout(time: 8, unit: 'HOURS') + } + + triggers { + cron('H/5 * * * *') + } + + parameters { + choice( + name: 'SOURCE_MODE', + choices: ['discover', 'pull_request', 'branch', 'diagnose'], + description: 'Timer builds use discover. Self-queued workers use pull_request. Use branch for a manual exact-SHA build, or diagnose to probe GitHub App Checks access using PR_HEAD_SHA.' + ) + string(name: 'SOURCE_BRANCH', defaultValue: '', description: 'Manual branch mode: remote branch name, for example feature/CSTACKEX-223.') + string(name: 'SOURCE_SHA', defaultValue: '', description: 'Manual branch mode: exact full 40-character commit SHA.') + string(name: 'PR_ID', defaultValue: '', description: 'GitHub pull request number.') + string(name: 'PR_ACTION', defaultValue: '', description: 'GitHub pull_request action.') + string(name: 'PR_DRAFT', defaultValue: 'false', description: 'GitHub pull_request.draft.') + string(name: 'PR_REPOSITORY', defaultValue: '', description: 'GitHub repository full_name.') + string(name: 'PR_BASE_BRANCH', defaultValue: 'main', description: 'GitHub pull_request.base.ref.') + string(name: 'PR_HEAD_BRANCH', defaultValue: '', description: 'GitHub pull_request.head.ref.') + string(name: 'PR_HEAD_SHA', defaultValue: '', description: 'GitHub pull_request.head.sha.') + string(name: 'PR_AUTHOR_LOGIN', defaultValue: '', description: 'GitHub login used in presubmit mail subjects.') + string(name: 'PR_AUTHOR_EMAIL', defaultValue: '', description: 'PR author email, if available.') + string(name: 'PR_URL', defaultValue: '', description: 'GitHub pull request URL.') + string(name: 'PR_TITLE', defaultValue: '', description: 'GitHub pull request title.') + string(name: 'EXPECTED_REPOSITORY', defaultValue: 'NetApp/cloudstack', description: 'Only this GitHub repository may trigger presubmit.') + string(name: 'DISCOVERY_BASE_BRANCH', defaultValue: 'main', description: 'Discover only open pull requests targeting this branch.') + string( + name: 'MAX_ACTIVE_WORKERS', + defaultValue: '5', + description: 'Discover queues at most this many pull_request workers at once. Extra eligible SHAs wait for a later poll.' + ) + string(name: 'CLOUDSTACK_GIT_URL', defaultValue: 'https://github.com/NetApp/cloudstack.git', description: 'Repository containing branch and pull-request refs.') + string(name: 'GIT_CREDENTIALS_ID', defaultValue: '', description: 'Optional Jenkins Git credential ID.') + string( + name: 'GITHUB_APP_CREDENTIALS_ID', + defaultValue: 'cloudstack-presubmit-github-app', + description: 'Jenkins GitHub App credential used for Check Runs and author email lookup.' + ) + string( + name: 'VM_INVENTORY_CREDENTIALS_ID', + defaultValue: 'cloudstack-presubmit-inventory', + description: 'Jenkins Secret file credential containing the reserved-VM inventory YAML.' + ) + string(name: 'VCENTER_HOST', defaultValue: 'cstack-netapp-lab.rtp.openenglab.netapp.com', description: 'vCenter hostname.') + string(name: 'VCENTER_CREDENTIALS_ID', defaultValue: 'cloudstack-vcenter', description: 'vCenter username/password credential ID.') + string(name: 'VM_SSH_CREDENTIALS_ID', defaultValue: 'cloudstack-presubmit-ssh', description: 'Reserved-VM SSH username/password credential ID.') + string(name: 'MYSQL_ROOT_CREDENTIALS_ID', defaultValue: 'cloudstack-mysql-root', description: 'MySQL administrator credential ID.') + string(name: 'CLOUD_DB_CREDENTIALS_ID', defaultValue: 'cloudstack-db', description: 'CloudStack database credential ID.') + string(name: 'KVM_HOST_CREDENTIALS_ID', defaultValue: 'cloudstack-kvm-host', description: 'KVM host credential ID used by zone setup.') + string(name: 'ONTAP_CREDENTIALS_ID', defaultValue: 'cloudstack-ontap', description: 'ONTAP SVM credential ID.') + string(name: 'CLOUDSTACK_ADMIN_CREDENTIALS_ID', defaultValue: 'cloudstack-admin', description: 'CloudStack admin credential ID.') + booleanParam(name: 'PAUSE_BETWEEN_STAGES', defaultValue: false, description: 'Pause for manual verification after each stage. Keep this off for automatic PR workers.') + } + + environment { + SOURCE_DIR = 'cloudstack-src' + PRESUBMIT_RESULTS = 'presubmit-results' + // Branch mode leaves PR_ID and PR_HEAD_SHA empty, so scripts receive a + // mode-independent identity. The branch is sanitised because it reaches + // a filesystem path on the target VM. + RESOLVED_SOURCE_ID = "${params.SOURCE_MODE == 'branch' ? 'branch-' + params.SOURCE_BRANCH.replaceAll('[^A-Za-z0-9._-]', '-') : 'pr-' + params.PR_ID}" + RESOLVED_SOURCE_SHA = "${params.SOURCE_MODE == 'branch' ? params.SOURCE_SHA : params.PR_HEAD_SHA}" + } + + stages { + stage('Discover PR revisions') { + when { + expression { + return params.SOURCE_MODE == 'discover' + } + } + steps { + script { + discoverPullRequests() + } + } + } + + stage('Validate source request') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + validateSourceRequest() + } + } + post { + success { + script { + recordStageEvent('Validate source request', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Validate source request', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Validate source request', 'ABORTED') + } + } + } + } + + stage('Check eligibility') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + checkPresubmitEligibility() + } + } + post { + success { + script { + recordStageEvent( + 'Check eligibility', + currentBuild.currentResult == 'NOT_BUILT' ? 'SKIPPED' : 'SUCCESS' + ) + } + } + failure { + script { + recordStageEvent('Check eligibility', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Check eligibility', 'ABORTED') + } + } + } + } + + stage('Abort superseded run') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + abortSupersededRun() + } + } + post { + success { + script { + recordStageEvent('Abort superseded run', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Abort superseded run', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Abort superseded run', 'ABORTED') + } + } + } + } + + stage('Checkout CI scripts') { + when { + expression { + return isWorkerStageEnabled() || isDiagnoseRun() + } + } + steps { + script { + checkoutCiScripts() + } + } + post { + success { + script { + recordStageEvent('Checkout CI scripts', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Checkout CI scripts', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Checkout CI scripts', 'ABORTED') + } + } + } + } + + stage('Diagnose GitHub App') { + when { + expression { + return isDiagnoseRun() + } + } + steps { + script { + diagnoseGithubApp() + } + } + } + + stage('Start PR reporting') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + startPullRequestReporting() + } + } + post { + success { + script { + recordStageEvent('Start PR reporting', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Start PR reporting', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Start PR reporting', 'ABORTED') + } + } + } + } + + stage('Validate builder') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + validateBuilder() + } + } + post { + success { + script { + recordStageEvent('Validate builder', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Validate builder', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Validate builder', 'ABORTED') + } + } + } + } + + stage('Checkout source') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + checkoutSource() + } + } + post { + success { + script { + recordStageEvent('Checkout source', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Checkout source', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Checkout source', 'ABORTED') + } + } + } + } + + stage('Build and unit tests') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + runUnitTests() + } + } + post { + success { + script { + recordStageEvent('Build and unit tests', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Build and unit tests', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Build and unit tests', 'ABORTED') + } + } + } + } + + stage('Build Debian packages') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + buildDebianPackages() + } + } + post { + success { + script { + recordStageEvent('Build Debian packages', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Build Debian packages', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Build Debian packages', 'ABORTED') + } + } + } + } + + stage('Deploy and run ONTAP integration') { + when { + expression { + return isWorkerStageEnabled() + } + } + steps { + script { + deployAndRunOntapIntegration() + } + } + post { + success { + script { + recordStageEvent('Deploy and run ONTAP integration', 'SUCCESS') + } + } + failure { + script { + recordStageEvent('Deploy and run ONTAP integration', 'FAILURE') + } + } + aborted { + script { + recordStageEvent('Deploy and run ONTAP integration', 'ABORTED') + } + } + } + } + } + + post { + always { + script { + publishPresubmitResults() + } + } + } +} diff --git a/private-cicd/README.md b/private-cicd/README.md new file mode 100644 index 000000000000..ae48b35a9a75 --- /dev/null +++ b/private-cicd/README.md @@ -0,0 +1,96 @@ + + +# Private CloudStack CI/CD + +This downstream-only directory implements the NetApp ONTAP presubmit for +CloudStack. Do not include it in pull requests to `apache/cloudstack`. + +For setup, operation, testing, security boundaries, troubleshooting, and the +current implementation contract, see +[`docs/PRIVATE-CICD-GUIDE.md`](docs/PRIVATE-CICD-GUIDE.md). + +## How it works + +The production Jenkins job loads [`Jenkinsfile`](Jenkinsfile) and helper scripts +from the protected NetApp `main` branch. Every five minutes a short discovery +build polls GitHub and self-queues a worker for each unseen PR head SHA. Each +worker checks out the exact commit into a separate `cloudstack-src` directory, +then: + +1. validates the request, builder, credentials, and VM inventory; +2. runs the full Maven build and unit tests; +3. builds and verifies CloudStack Debian packages; +4. locks and reverts a compatible lab VM; +5. installs and configures CloudStack, MySQL, KVM, NFS, and iSCSI; +6. creates the test zone and runs the ONTAP iSCSI and NFS3 suites; +7. redacts and archives results, publishes a GitHub Check, and sends mail. + +Eligible pull-request revisions receive the required Check +`cloudstack-ontap-presubmit`. GitHub API and SMTP failures are reported without +replacing the underlying build or test result. + +## Entry points + +- **Pull request:** `SOURCE_MODE=discover` notices an open non-draft PR update + to `main` and self-queues at most five `SOURCE_MODE=pull_request` workers in + the same job. Extra SHAs wait for a later poll. Workers then wait on the + `cloudstack-presubmit-vm` lock for a lab VM. +- **Manual branch:** a separate triggerless job uses `SOURCE_MODE=branch` with + a remote branch and exact 40-character commit SHA. +- **Local validation:** `scripts/validate-local.sh` checks the CI files without + building CloudStack. +- **Direct lab validation:** the scripts can be run gate by gate on a disposable + Ubuntu 22.04 nested-KVM VM. + +Different sources may run concurrently. A newer run aborts an older run only +for the same pull-request ID or manual source branch, and only before the older +run acquires its integration-test VM. + +## Configuration and secrets + +[`config/vm-inventory.yaml.example`](config/vm-inventory.yaml.example) is the +only tracked inventory file. Create populated inventory outside Git and upload +it to Jenkins as a Secret file. Keep passwords, tokens, and private keys in +dedicated Jenkins credentials; generated runtime configuration is not archived. + +The Kubernetes pod uses a `jnlp` agent container and a separate +`cloudstack-driver` build container. The immutable driver image reference, +resource requests, build commands, credential mapping, and snapshot contract +are documented in the comprehensive guide. + +## Local validation + +```bash +./private-cicd/scripts/validate-local.sh + +# Also build the driver image +./private-cicd/scripts/validate-local.sh --with-docker +``` + +Local validation checks shell syntax, compiles Python, and parses YAML when a +supported parser is available. The Docker option builds the driver image. These +commands do not run Maven, create Debian packages, deploy CloudStack, or run +ONTAP tests. + +## Related code + +- ONTAP plugin: [`../plugins/storage/volume/ontap/`](../plugins/storage/volume/ontap/) +- ONTAP integration tests: + [`../test/integration/plugins/ontap/`](../test/integration/plugins/ontap/) diff --git a/private-cicd/benchmark/ontap/.gitignore b/private-cicd/benchmark/ontap/.gitignore new file mode 100644 index 000000000000..fc8b8af6d831 --- /dev/null +++ b/private-cicd/benchmark/ontap/.gitignore @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +config.yaml +config_sanity.yaml +.venv/ +__pycache__/ +*.pyc +results/*.csv +results/*.md +results/*.log diff --git a/private-cicd/benchmark/ontap/AGENTS.md b/private-cicd/benchmark/ontap/AGENTS.md new file mode 100644 index 000000000000..40ea169fdbb6 --- /dev/null +++ b/private-cicd/benchmark/ontap/AGENTS.md @@ -0,0 +1,41 @@ + +# ONTAP scale benchmark + +Scale matrix: [ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix](https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350). + +Test ids written to the CSVs and report map to the page sections as follows: + +| Test id | Page section | What it measures | +| --- | --- | --- | +| `pool_sequential_create` | 5.1.1 | Sequential create (1 pool at a time, cumulative to N) | +| `pool_sequential_delete` | 5.1.2 | Sequential delete (1 pool at a time, from N remaining) | +| `pool_concurrent_create` | 6.1.1 | Parallel pool creation | +| `pool_concurrent_delete` | 6.1.2 | Parallel pool deletion | +| `vm_sequential_create` | 5.2.1 | Sequential create (1 VM at a time, cumulative to N) | +| `vm_sequential_delete` | 5.2.2 | Sequential delete (1 VM at a time, from N remaining) | +| `vm_concurrent_create` | 6.2.1 | Parallel VM creation | +| `vm_concurrent_delete` | 6.2.2 | Parallel VM deletion | +| Report "Results log" table | 9 | Results log | + +Keep section numbers and the page link out of the code. How to run each script is in `README.md` and `USAGE.md`. +- Protocols, checkpoints, and concurrency levels live in `config.yaml`. `run.py` is the entry point. Do not add a CLI flag for a field that already exists in config. +- Use `benchmark_support` for run ids, resource names, `mean_seconds`, and logging. Sequential and concurrency scripts must use the same helpers and the same log format. Do not hand-roll a second average or a second logger. +- Run ids are letters, digits, and underscores. Storage pool names disallow `-`. VM hostnames disallow `_`; `format_vm_name` / `vm_name_token` are the only rewrite, and cleanup must use them so created VMs are the ones deleted. +- Cleanup must stay inside the configured name prefix, zone, cluster (pools), and storage provider. Do not delete a resource just because its name contains a short substring. diff --git a/private-cicd/benchmark/ontap/README.md b/private-cicd/benchmark/ontap/README.md new file mode 100644 index 000000000000..d9dc8693895c --- /dev/null +++ b/private-cicd/benchmark/ontap/README.md @@ -0,0 +1,233 @@ + +# ONTAP plugin - storage pool benchmark (Step 1) + +> Looking for a quick "what does script X do / how do I run it" cheat +> sheet instead of the full design write-up below? See [USAGE.md](USAGE.md). + +Downstream-only benchmarking tool (not for Apache upstream) that drives +`createStoragePool` / `deleteStoragePool` over the CloudStack HTTP/REST API +to reproduce the **Storage Pool** rows of the Sequential Scale Matrix (5.1) +and Parallel/Concurrency Matrix (6.1) from the Confluence page +["ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix"](https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350). + +This is step 1 of the automation follow-up (storage pool only). VM instance +benchmarks are now also covered (see below); volume/snapshot benchmarks will +be added as separate scripts later, reusing `cloudstack_client.py`. + +The storage-pool benchmark is split into **two independent scripts** (on +purpose — a sequential run is cheap/safe and worth reviewing before deciding +to launch a concurrency run, which is the one most likely to stress the +mgmt-server/plugin job queue): + +- `benchmark_storage_pool_sequential.py` — Section 5.1 (5.1.1 / 5.1.2) +- `benchmark_storage_pool_concurrency.py` — Section 6.1 (6.1.1 / 6.1.2) + +Both share the same createStoragePool/deleteStoragePool call shape, CSV +formats, and cleanup logic via `storage_pool_common.py`. Run them with the +**same `--run-id`** to accumulate both into a single +`results/summary_.csv` (and therefore a single `render_report.py` +output covering 5.1.x and 6.1.x together). + +The VM-instance benchmark is split into **three scripts** sharing +`vm_instance_common.py` (same call shape/CSV formats/cleanup logic): + +- `benchmark_vm_instance_sequential.py` — Section 5.2 (5.2.1 / 5.2.2) +- `benchmark_vm_instance_concurrency.py` — Section 6.2 (6.2.1 / 6.2.2) +- `benchmark_vm_instance_combined.py` — runs both of the above back to back + under one shared `--run-id`, for when you already trust the config/ + environment and just want the full 5.2.x + 6.2.x matrix in one invocation + +See "VM instance benchmark" below for prerequisites and usage. + +## What it does + +1. **Sequential scale test (5.1.1 / 5.1.2)** — for each protocol (NFS3, + iSCSI), creates storage pools one at a time up to N=30, logging every + single create call, and reports cumulative-total/avg-per-op timing at + checkpoints N = 1, 5, 10, 20, 30. Then deletes them one at a time and + reports the same at "N remaining" = 30, 20, 10, 5, 1. +2. **Concurrency test (6.1.1 / 6.1.2)** — for each concurrency level + C = 2, 5, 10, 20, 30 and each protocol, creates C pools in parallel + (`ThreadPoolExecutor`), records wall-clock time for the whole batch plus + success/failure/avg-per-op, then deletes the same C pools in parallel. +3. Writes **every** individual API call (start/end timestamp, duration, + success/failure, pool id, error) to `results/raw_ops_.csv`, and a + checkpoint-level roll-up to `results/summary_.csv`. +4. `render_report.py` turns those two CSVs into markdown tables that match + the Confluence page's table layout (including min/avg/p95/max stats for + the "Results Log" section), ready to paste back into the page. + +## Setup + +```bash +cd private-cicd/benchmark/ontap +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp config.example.yaml config.yaml +``` + +Edit `config.yaml`: + +- `cloudstack.*` — API URL, admin username/password of your lab's + management server. +- `infrastructure.*` — zone/pod/cluster UUIDs to create pools under. +- `ontap.nfs3` / `ontap.iscsi` — connection details of the ONTAP SVM(s) you + want to benchmark against for each protocol. Delete a block if you only + have one protocol available. +- `benchmark.*` — checkpoints, concurrency levels, pool name prefix, output + directory. Defaults match the Confluence page (N/C = 1,5,10,20,30 / + 2,5,10,20,30). + +`config.yaml` and everything under `results/` are gitignored — never commit +real lab IPs/credentials or run output. + +## Running + +`run.py` is the entry point. Protocols, checkpoints, and concurrency levels +come from `config.yaml` (`ontap` / `benchmark` for pools, `vm_bench` for +VMs). Set `enabled_protocols` under `benchmark` or `vm_bench` to select +`[nfs3]`, `[iscsi]`, or `[nfs3, iscsi]`, and edit `sequential_checkpoints` or +`concurrency_levels` to change the matrix. + +Run ids are generated as `RUN__<4 hex chars>` (for example +`RUN_20260927_081500_a1b2`). Letters, digits, and underscores only: a hyphen +is rewritten to `_` because it is embedded in the storage pool name. Reuse +one id across sequential and concurrency when you want a single report. + +```bash +# Sanity-check without hitting a real management server: +python3 run.py --test storage-pool-sequential --config config.yaml --dry-run +python3 run.py --test storage-pool-concurrency --config config.yaml --dry-run + +# Sequential scale matrix, using the protocols and checkpoints in config.yaml: +python3 run.py --test storage-pool-sequential --config config.yaml + +# Concurrency matrix, using config.yaml's concurrency_levels: +python3 run.py --test storage-pool-concurrency --config config.yaml + +# Same --run-id across both tests to combine 5.1.x + 6.1.x into one report. +# Run sequential first, review it, then decide on concurrency: +python3 run.py --test storage-pool-sequential --config config.yaml --run-id RUN_20260927_081500_a1b2 +python3 run.py --test storage-pool-concurrency --config config.yaml --run-id RUN_20260927_081500_a1b2 +``` + +At the end of a real run each script deletes leftover pools from that run id. +The sweep only deletes pools that match this config's name prefix, name +pattern, zone, cluster, and storage provider, so other pools in a shared zone +are left alone. Pass `--skip-cleanup` to inspect the pools yourself. + +If a run crashes and leaves pools behind: + +```bash +# Deletes benchmark pools for this config's pool_name_prefix (still provider/zone/cluster scoped): +python3 run.py --test storage-pool-sequential --config config.yaml --cleanup-only + +# Or scope it to one specific run (underscores, not hyphens): +python3 run.py --test storage-pool-sequential --config config.yaml --cleanup-only RUN_20260927_081500_a1b2 +``` + +## Rendering the Confluence-ready report + +```bash +python3 render_report.py --run-id RUN_20260722_101500_a1b2 \ + --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 +``` + +This prints markdown tables for 5.1.1, 5.1.2, 6.1.1, 6.1.2, and a set of +pre-filled rows for the page's Section 9 "Results Log" table (with Min/Avg/ +P95/Max computed from the raw per-op log), and also saves them to +`results/report_.md`. + +## VM instance benchmark + +Drives `deployVirtualMachine` / `destroyVirtualMachine` to reproduce the +**VM Instance** rows of the same Confluence page's Sequential Scale Matrix +(5.2) and Parallel/Concurrency Matrix (6.2). Every VM gets a root disk (from +the service offering) AND a data disk (from the disk offering), both landing +on the same tagged storage pool. + +### Prerequisites (one-time setup, per protocol) + +Unlike the storage-pool benchmark's transient pools, the VM-instance +benchmark needs a **persistent, pre-created** `bench_vm_` pool plus +matching offerings - it does not create/destroy the pool itself: + +1. A storage pool (e.g. `bench_vm_nfs3_pool`) with a distinct storage tag + (e.g. `bench_vm_nfs3`), sized comfortably below your aggregate's capacity. +2. A service offering (root disk) and a disk offering (data disk), both + tagged with that same storage tag, so root + data land on the same pool. + Set the service offering's `rootdisksize` (GB) to comfortably exceed the + template's actual `qemu-img` **virtual** size (not its sparse/download + file size) - too small silently "succeeds" on NFS but hard-fails on iSCSI + with `qemu-img: Cannot grow device files` (see Confluence Issue #2). +3. Fill in `vm_bench.templateid` / `networkid` / per-protocol + `serviceofferingid` / `diskofferingid` in `config.yaml` with the above. + +### Running + +```bash +# Sanity-check without hitting a real management server: +python3 run.py --test vm-instance-sequential --config config.yaml --dry-run +python3 run.py --test vm-instance-concurrency --config config.yaml --dry-run + +# Sequential scale matrix (5.2.1/5.2.2), protocols from config.yaml: +python3 run.py --test vm-instance-sequential --config config.yaml + +# Concurrency matrix (6.2.1/6.2.2). Levels come from vm_bench.concurrency_levels: +python3 run.py --test vm-instance-concurrency --config config.yaml + +# Both matrices in one invocation: +python3 run.py --test vm-instance-combined --config config.yaml + +# Or run the two tests under the SAME --run-id (underscores only): +python3 run.py --test vm-instance-sequential --config config.yaml --run-id RUN_20260927_081500_a1b2 +python3 run.py --test vm-instance-concurrency --config config.yaml --run-id RUN_20260927_081500_a1b2 + +python3 render_report.py --run-id RUN_20260927_081500_a1b2 --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +``` + +All three scripts share the same `--skip-cleanup` / `--cleanup-only` safety +net as the storage-pool scripts (see above), via `vm_instance_common.py`. +Failed `deployVirtualMachine` calls are logged but their VM/disk are +intentionally left in place (not auto-destroyed) so you can inspect exactly +what got left behind - use `--cleanup-only` once you're done. + +## Notes / caveats + +- Auth is session-key based (`login` -> `sessionkey` + `JSESSIONID` cookie), + matching the approach documented on the Confluence API page. API + key/secret signed requests are not implemented (add to + `cloudstack_client.py` if your lab requires it). +- Timing is measured end-to-end from the caller's perspective, including any + async job polling (`queryAsyncJobResult`) — this is the number an operator + or automation would actually observe, not raw server-side processing time. +- The single `CloudStackClient`/`requests.Session` is shared across threads + during concurrency tests. We never mutate session state after login, so + this is safe; if you prefer full isolation, instantiate one + `CloudStackClient` per thread instead. +- `createStoragePool`/`deleteStoragePool`/`updateStoragePool` are + synchronous in this CloudStack version (no `jobid` in the response); + `enableStorageMaintenance`/`cancelStorageMaintenance` are async — the + client polls `queryAsyncJobResult` automatically for any response that + does contain a `jobid`, so the script works either way. +- This script intentionally covers **create/delete only** (per the current + ask). Maintenance mode and enable/disable pool are documented in + `cloudstack_client.py`'s API surface but not yet wired into a benchmark + phase — flag if you want those added to the matrix too. diff --git a/private-cicd/benchmark/ontap/USAGE.md b/private-cicd/benchmark/ontap/USAGE.md new file mode 100644 index 000000000000..98bfee11d0ea --- /dev/null +++ b/private-cicd/benchmark/ontap/USAGE.md @@ -0,0 +1,192 @@ + +# Usage Reference + +Quick reference for **what each file does** and **how to run it**. For the +"why" behind the design (what each benchmark measures, how it maps to the +Confluence test matrix, known caveats) see [README.md](README.md) instead - +this file is deliberately just a lookup table / cheat sheet. + +## 1. Prerequisites + +| # | Requirement | Notes | +| --- | --- | --- | +| 1 | Python 3.9+ | No other version-specific features used; anything 3.9-3.12 should work. | +| 2 | Network access to a CloudStack management server's API (`/client/api`) | Admin credentials with permission to create/delete storage pools, service/disk offerings, and deploy/destroy VMs. | +| 3 | Network access (HTTPS/443) from **your machine** to the ONTAP cluster's management LIF | Only needed if you run the storage-pool scripts' `--cleanup-only` helper functions that talk to ONTAP directly for orphan checks; the benchmark scripts themselves only ever call the CloudStack API, never ONTAP directly. | +| 4 | A CloudStack zone/pod/cluster with at least one KVM host, and the ONTAP plugin's SVM(s) reachable from the CloudStack management server | See README.md's "Test Environment" section on the Confluence page for the exact lab topology this was built/tested against. | +| 5 | For the VM-instance scripts only: a pre-created `bench_vm_` storage pool + matching service/disk offerings | One-time setup, see [Prerequisites (VM instance benchmark)](#4-vm-instance-benchmark-scripts) below. | + +### One-time environment setup + +```bash +cd private-cicd/benchmark/ontap +python3 -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +cp config.example.yaml config.yaml +``` + +Then edit `config.yaml` with your lab's real API URL, credentials, zone/pod/ +cluster IDs, and ONTAP connection details (see inline comments in +`config.example.yaml` for what each field means and why). + +> **About `.venv/`:** this is a completely standard, disposable Python +> virtual environment (`python3 -m venv .venv` + `pip install -r +> requirements.txt`, which just pulls in `requests` and `PyYAML` plus their +> small set of transitive dependencies). It is **never committed** (it's in +> `.gitignore`) because it is: +> - **Machine/OS-specific** - it stores an absolute path back to the Python +> interpreter that created it (`.venv/pyvenv.cfg`) and can contain +> platform-compiled binary wheels (e.g. a `.so`/`.pyd` file bundled with +> one of the dependencies) that won't load on a different OS/architecture. +> - **Python-version-specific** - the `lib/python3.9/...` directory layout +> is tied to whatever interpreter version created it. +> - **Fully reproducible** - anyone can regenerate an equivalent one in a +> few seconds with the two commands above; there is nothing hand-tuned or +> irreplaceable in it. +> +> If you ever see `.venv/` show up in `git status` as untracked, it means +> `.gitignore` isn't being picked up (e.g. it was created before the +> `.gitignore` entry was added) - just confirm `.venv/` is listed in +> `.gitignore` and it will stop appearing. + +## 2. Shared library modules (not run directly) + +| File | Purpose | +| --- | --- | +| `benchmark_support.py` | Shared run ids, resource names, `mean_seconds`, and the one log format used by every script. | +| `cloudstack_client.py` | Minimal CloudStack HTTP/REST client (session-key login, generic `call()`, automatic `queryAsyncJobResult` polling). Used by every benchmark script. | +| `storage_pool_common.py` | Shared helpers for the storage-pool scripts: `createStoragePool`/`deleteStoragePool` request building, CSV logging (`RawLogger`, `append_summary_csv`), cleanup-by-filter, config loading. | +| `vm_instance_common.py` | Shared helpers for the VM-instance scripts: `deployVirtualMachine`/`destroyVirtualMachine` request building (incl. force-purging data disks and `Destroy`-state volumes so ONTAP space is reclaimed immediately instead of waiting for CloudStack's 24h cleanup delay), CSV logging, cleanup-by-filter, config loading. | + +## 3. Storage-pool benchmark scripts + +Reproduces Confluence sections 5.1 (sequential) and 6.1 (concurrency) - +`createStoragePool` / `deleteStoragePool` timing at scale. + +| Script | What it does | Typical command | +| --- | --- | --- | +| `run.py` | Entry point. Loads `config.yaml` and runs the test named by `--test`. | `python3 run.py --test storage-pool-sequential --config config.yaml` | +| `benchmark_storage_pool_sequential.py` | Creates storage pools one at a time (per protocol) up to N=30, then deletes them one at a time, logging every call and reporting checkpoint totals at N = 1, 5, 10, 20, 30. | `python3 run.py --test storage-pool-sequential --config config.yaml` | +| `benchmark_storage_pool_concurrency.py` | For each concurrency level C (default 2, 5, 10, 20, 30), creates C pools in parallel via a thread pool, records wall-clock time for the whole batch, then deletes the same C pools in parallel. | `python3 run.py --test storage-pool-concurrency --config config.yaml` | + +Common flags (both scripts): + +| Flag | Meaning | +| --- | --- | +| `--config PATH` | Config YAML to use (default `config.yaml`). Protocols, checkpoints, and concurrency levels are read from here. | +| `--run-id ID` | Reuse a run id. Auto-generated otherwise as `RUN__<4 hex chars>`. Letters, digits, and underscores only — a hyphen is rewritten to `_` because storage-pool names (and the ONTAP volume behind them) disallow `-`. Use the **same id** across sequential and concurrency to merge one report. | +| `--dry-run` | Simulate timings with no real API calls - use this first to sanity-check your config. | +| `--skip-cleanup` | Skip the automatic post-run sweep for leftover pools from this run id. | +| `--cleanup-only [FILTER]` | Don't run the benchmark. Delete pools that match `FILTER` (default: config's `pool_name_prefix`) **and** this config's name pattern, zone, cluster, and storage provider. | + +`run.py` also requires `--test`. Protocols, checkpoints, and concurrency levels are not flags; change them in `config.yaml`. + +- `--test NAME` — **Required.** One of `storage-pool-sequential`, `storage-pool-concurrency`, `vm-instance-sequential`, `vm-instance-concurrency`, or `vm-instance-combined`. + +## 4. VM-instance benchmark scripts + +Reproduces Confluence sections 5.2 (sequential) and 6.2 (concurrency) - +`deployVirtualMachine` / `destroyVirtualMachine` timing at scale, with each +VM getting both a root disk and a data disk on the same tagged pool. + +### Prerequisites (one-time, per protocol - not created/destroyed by the scripts) + +1. A persistent storage pool (e.g. `bench_vm_nfs3_pool`) with a distinct + storage tag (e.g. `bench_vm_nfs3`), sized comfortably below your + aggregate's capacity. +2. A service offering (root disk) and a disk offering (data disk), both + tagged with that same storage tag, so root + data land on the same pool. + Set the service offering's `rootdisksize` (GB) to comfortably exceed the + template's actual `qemu-img` **virtual** size - too small silently + "succeeds" on NFS but hard-fails on iSCSI with `qemu-img: Cannot grow + device files`. +3. Fill in `vm_bench.templateid` / `networkid` / per-protocol + `serviceofferingid` / `diskofferingid` in `config.yaml`. + +### Scripts + +| Script | What it does | Typical command | +| --- | --- | --- | +| `benchmark_vm_instance_sequential.py` | Deploys VMs one at a time (per protocol) up to N=30, then destroys them one at a time, reporting checkpoint totals at N = 1, 2, 5, 10, 20, 30. | `python3 run.py --test vm-instance-sequential --config config.yaml` | +| `benchmark_vm_instance_concurrency.py` | For each concurrency level C, deploys C VMs in parallel, records wall-clock time, then destroys the same C VMs in parallel. | `python3 run.py --test vm-instance-concurrency --config config.yaml` | +| `benchmark_vm_instance_combined.py` | Runs the sequential matrix immediately followed by the concurrency matrix, under one shared run id, in a single invocation. Use once you already trust your config/environment. | `python3 run.py --test vm-instance-combined --config config.yaml` | + +Common flags: same as the storage-pool scripts above (`--config`, +`--run-id`, `--dry-run`, `--skip-cleanup`, `--cleanup-only [FILTER]`). +VM cleanup rewrites `_` in the filter to `-` so it matches the hostname +form of the run id, and it only destroys VMs in the configured zone whose +names match the benchmark prefix and pattern. Failed +`deployVirtualMachine` calls are intentionally **left in place** (not +auto-destroyed) so you can inspect what got left behind - clean them up +with `--cleanup-only` once done. + +## 5. Report rendering + +`render_report.py` turns a run's `raw_ops_.csv` + `summary_.csv` into markdown tables (section layout plus Min/Avg/P95/Max rows for the Results Log). Writes to stdout and to `results/report_.md`. + +Typical command: + +```bash +python3 render_report.py \ + --run-id RUN_20260927_081500_a1b2 \ + --cloudstack-build 4.23.0.0-SNAPSHOT \ + --ontap-version 9.17.1 +``` + +For VM-instance runs (which write `raw_ops_vm_.csv` / +`summary_vm_.csv` instead of the storage-pool scripts' `raw_ops_`/ +`summary_` prefix), pass the matching prefixes: + +```bash +python3 render_report.py --run-id RUN_20260927_081500_a1b2 \ + --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +``` + +| Flag | Meaning | +| --- | --- | +| `--run-id ID` | **Required.** The run id used by the benchmark script(s). | +| `--output-dir DIR` | Where the CSVs live / the report gets written (default `results`). | +| `--cloudstack-build STR` | Label only, stamped into the Results Log rows. | +| `--ontap-version STR` | Label only, stamped into the Results Log rows. | +| `--run-date YYYY-MM-DD` | Defaults to today. | +| `--raw-prefix STR` | CSV filename prefix for raw per-op data (default `raw_ops`; use `raw_ops_vm` for VM-instance runs). | +| `--summary-prefix STR` | CSV filename prefix for checkpoint summaries (default `summary`; use `summary_vm` for VM-instance runs). | +| `--report-suffix STR` | Suffix appended to the output report filename (e.g. `_vm`) to avoid clobbering a storage-pool report with the same run id. | + +## 6. Config files at a glance + +| File | Committed? | Purpose | +| --- | --- | --- | +| `config.example.yaml` | Yes | Sanitized template - copy to `config.yaml` and fill in your lab's real values. | +| `config.yaml` | **No** (gitignored) | Your real, lab-specific config (real IPs/credentials) - never commit this. | +| `config_sanity.yaml` | **No** (gitignored) | Optional scratch config some contributors keep locally for quick one-off dry-runs; not part of the checked-in tooling. | + +## 7. Output files at a glance + +All written under `results/` (gitignored - these are run artifacts, not +source): + +| Pattern | Produced by | Contents | +| --- | --- | --- | +| `raw_ops_.csv` | storage-pool scripts | One row per individual API call (timestamps, duration, success/failure, error). | +| `raw_ops_vm_.csv` | VM-instance scripts | Same, for VM deploy/destroy calls. | +| `summary_.csv` / `summary_vm_.csv` | both | One row per checkpoint/concurrency level (totals, averages, success/failure counts). | +| `report_.md` / `report_vm_.md` | `render_report.py` | Confluence-ready markdown tables generated from the two CSVs above. | diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py new file mode 100644 index 000000000000..433eb5528df6 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Concurrency storage-pool benchmark for the NetApp ONTAP CloudStack plugin. + +Drives createStoragePool / deleteStoragePool in parallel (ThreadPoolExecutor) +over the CloudStack HTTP/REST API. Concurrency levels come from config.yaml: + + Parallel pool creation + Parallel pool deletion + +See benchmark_storage_pool_sequential.py for the sequential scale matrix +(kept as a separate script/entry point on purpose - concurrency runs are the +ones most likely to hit mgmt-server/plugin job-queue saturation, so they're +easy to run, review, and re-run independently of the sequential matrix). + +Every individual API call is logged to results/raw_ops_.csv, and a +per-checkpoint roll-up is appended to results/summary_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_storage_pool_sequential.py +combines both into a single summary_.csv / report. + +Protocols and concurrency levels come from config.yaml. Prefer run.py. + +Usage: + python3 run.py --test storage-pool-concurrency --config config.yaml + python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run + python3 benchmark_storage_pool_concurrency.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from benchmark_support import ( + assert_pool_name_prefix, + configure_logging, + format_pool_name, + get_logger, + mean_seconds, + normalize_run_id, +) +from storage_pool_common import ( + append_summary_csv, + cleanup_by_filter, + create_pool, + delete_pool, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + RawLogger, + resolve_protocols, +) + +log = get_logger() + + +def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): + infra = cfg["infrastructure"] + ontap_cfg = cfg["ontap"][protocol_key] + prefix = cfg["benchmark"]["pool_name_prefix"] + + for level in levels: + log.info("Parallel pool create protocol=%s concurrency=%s", protocol_key, level) + names = [format_pool_name(prefix, protocol_key, run_id, i, concurrency_level=level) + for i in range(1, level + 1)] + + t0 = time.perf_counter() + results = [] + if dry_run: + results = [fake_create(n) for n in names] + else: + with ThreadPoolExecutor(max_workers=level) as ex: + futures = {ex.submit(create_pool, client, n, infra, ontap_cfg): n for n in names} + for fut in as_completed(futures): + results.append(fut.result()) + wall = time.perf_counter() - t0 + + for idx, r in enumerate(results, start=1): + raw_logger.log(run_id, "concurrent_create", "pool_concurrent_create", protocol_key, level, idx, r) + + succ = [r for r in results if r.success] + fail = [r for r in results if not r.success] + avg = mean_seconds([r.duration_sec for r in succ]) + notes = "Watch mgmt-server job-queue/thread-pool saturation as a confound" if level >= 30 else "" + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_create", "test_id": "pool_concurrent_create", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), + "success_count": len(succ), "failure_count": len(fail), "notes": notes, + }) + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall, avg, len(succ), len(fail), + ) + for r in fail: + log.error("%s: %s", r.pool_name, r.error) + + log.info("Parallel pool delete protocol=%s concurrency=%s", protocol_key, len(succ)) + t0 = time.perf_counter() + del_results = [] + if dry_run: + del_results = [fake_delete(r.pool_id, r.pool_name) for r in succ] + elif succ: + with ThreadPoolExecutor(max_workers=len(succ)) as ex: + futures = {ex.submit(delete_pool, client, r.pool_id, r.pool_name): r for r in succ} + for fut in as_completed(futures): + del_results.append(fut.result()) + wall_del = time.perf_counter() - t0 + + for idx, r in enumerate(del_results, start=1): + raw_logger.log(run_id, "concurrent_delete", "pool_concurrent_delete", protocol_key, level, idx, r) + + succ_d = [r for r in del_results if r.success] + fail_d = [r for r in del_results if not r.success] + avg_d = mean_seconds([r.duration_sec for r in succ_d]) + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_delete", "test_id": "pool_concurrent_delete", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), + "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", + }) + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall_del, avg_d, len(succ_d), len(fail_d), + ) + for r in fail_d: + log.error("%s: %s", r.pool_name, r.error) + if delay: + time.sleep(delay) + + +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_pool_name_prefix(cfg["benchmark"]["pool_name_prefix"]) + output_dir = cfg["benchmark"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) + + if cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = cfg["benchmark"]["pool_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) + return + + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") + delay = cfg["benchmark"].get("inter_op_delay_sec", 0) + levels = sorted(cfg["benchmark"]["concurrency_levels"]) + + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: concurrency") + log.info("Concurrency levels: %s", levels) + log.info("Dry run: %s", dry_run) + + client = None if dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info("Next: python3 render_report.py --run-id %s --output-dir %s", run_id, output_dir) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned pools remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete benchmark pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py new file mode 100644 index 000000000000..6f3e5f22e93a --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Sequential storage-pool benchmark for the NetApp ONTAP CloudStack plugin. + +Drives createStoragePool / deleteStoragePool one at a time over the CloudStack +HTTP/REST API. Checkpoints come from config.yaml: + + Sequential create (1 pool at a time, cumulative to N) + Sequential delete (1 pool at a time, from N remaining) + +See benchmark_storage_pool_concurrency.py for the parallel/concurrency matrix +(kept as a separate script/entry point on purpose, so a sequential run can be +kicked off and reviewed independently before committing to a concurrency run). + +Every individual API call is logged to results/raw_ops_.csv, and a +per-checkpoint roll-up is appended to results/summary_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_storage_pool_concurrency.py +combines both into a single summary_.csv / report. + +Protocols and checkpoints come from config.yaml. Prefer run.py. + +Usage: + python3 run.py --test storage-pool-sequential --config config.yaml + python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run + python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time + +from benchmark_support import ( + assert_pool_name_prefix, + configure_logging, + format_pool_name, + get_logger, + mean_seconds, + normalize_run_id, +) +from storage_pool_common import ( + append_summary_csv, + cleanup_by_filter, + create_pool, + delete_pool, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + RawLogger, + resolve_protocols, +) + +log = get_logger() + + +def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): + infra = cfg["infrastructure"] + ontap_cfg = cfg["ontap"][protocol_key] + checkpoints = sorted(cfg["benchmark"]["sequential_checkpoints"]) + max_n = max(checkpoints) + prefix = cfg["benchmark"]["pool_name_prefix"] + + created = [] + create_durations = [] + log.info("Sequential pool create protocol=%s up to N=%s", protocol_key, max_n) + for i in range(1, max_n + 1): + name = format_pool_name(prefix, protocol_key, run_id, i) + result = fake_create(name) if dry_run else create_pool(client, name, infra, ontap_cfg) + raw_logger.log(run_id, "sequential_create", "pool_sequential_create", protocol_key, max_n, i, result) + status = "OK" if result.success else "FAIL" + log.info("[%s/%s] create %s -> %s (%.3fs)", i, max_n, name, status, result.duration_sec) + if result.success: + created.append((name, result.pool_id)) + create_durations.append(result.duration_sec) + else: + log.error("%s", result.error) + if i in checkpoints: + total = sum(create_durations) + avg = mean_seconds(create_durations) + summary_rows.append({ + "run_id": run_id, "phase": "sequential_create", "test_id": "pool_sequential_create", + "protocol": protocol_key, "checkpoint": i, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": len(created), "failure_count": i - len(created), "notes": "", + }) + log.info( + "checkpoint N=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + i, protocol_key, total, avg, len(created), i - len(created), + ) + if delay: + time.sleep(delay) + + total_created = len(created) + log.info("Sequential pool delete protocol=%s from N=%s remaining", protocol_key, total_created) + if total_created in checkpoints: + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "pool_sequential_delete", + "protocol": protocol_key, "checkpoint": total_created, + "total_time_sec": 0, "avg_time_sec": 0, "success_count": 0, "failure_count": 0, + "notes": "baseline - no deletes issued yet", + }) + delete_durations = [] + deleted_ok = 0 + for idx, (name, pool_id) in enumerate(created, start=1): + result = fake_delete(pool_id, name) if dry_run else delete_pool(client, pool_id, name) + remaining = total_created - idx + raw_logger.log(run_id, "sequential_delete", "pool_sequential_delete", protocol_key, total_created, idx, result) + status = "OK" if result.success else "FAIL" + log.info( + "[%s/%s] delete %s -> %s (%.3fs) remaining=%s", + idx, total_created, name, status, result.duration_sec, remaining, + ) + if result.success: + delete_durations.append(result.duration_sec) + deleted_ok += 1 + else: + log.error("%s", result.error) + if remaining in checkpoints: + total = sum(delete_durations) + avg = mean_seconds(delete_durations) + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "pool_sequential_delete", + "protocol": protocol_key, "checkpoint": remaining, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", + }) + log.info( + "checkpoint remaining=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + remaining, protocol_key, total, avg, deleted_ok, idx - deleted_ok, + ) + if delay: + time.sleep(delay) + + +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_pool_name_prefix(cfg["benchmark"]["pool_name_prefix"]) + output_dir = cfg["benchmark"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) + + if cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = cfg["benchmark"]["pool_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) + return + + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") + delay = cfg["benchmark"].get("inter_op_delay_sec", 0) + + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: sequential") + log.info("Dry run: %s", dry_run) + + client = None if dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info("Next: python3 render_report.py --run-id %s --output-dir %s", run_id, output_dir) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned pools remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete benchmark pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_support.py b/private-cicd/benchmark/ontap/benchmark_support.py new file mode 100644 index 000000000000..bdafda215027 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_support.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Shared run-id, naming, statistics, and logging helpers for every benchmark script.""" + +import datetime +import logging +import re +import secrets +import statistics +import sys + +LOG_NAME = "ontap.benchmark" + + +def configure_logging(): + """One format for every benchmark script and the API client.""" + root = logging.getLogger(LOG_NAME) + if not root.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter( + "%(asctime)s %(levelname)s %(message)s", + "%Y-%m-%dT%H:%M:%S", + )) + root.addHandler(handler) + root.setLevel(logging.INFO) + root.propagate = False + return root + + +def get_logger(): + return logging.getLogger(LOG_NAME) + + +def mean_seconds(values): + """Average duration. Empty input is 0 so checkpoint rows stay numeric.""" + if not values: + return 0.0 + return statistics.mean(values) + + +def new_run_id(): + """Timestamp plus a short random suffix. Underscores only: pool names reject hyphens.""" + stamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") + return f"RUN_{stamp}_{secrets.token_hex(2)}" + + +def normalize_run_id(run_id): + """Accept a caller-supplied id, or generate one. Hyphens are rewritten to underscores.""" + if not run_id: + return new_run_id() + normalized = run_id.strip().replace("-", "_") + if not re.fullmatch(r"[A-Za-z0-9_]+", normalized): + sys.exit( + f"Run id {run_id!r} is not valid. Use letters, digits, and underscores " + "(hyphens are rewritten to underscores because storage pool names disallow '-')." + ) + if normalized != run_id.strip(): + get_logger().warning( + "Rewrote run id %r to %r so it can be embedded in a storage pool name", + run_id, normalized, + ) + return normalized + + +def format_pool_name(prefix, protocol_key, run_id, index, concurrency_level=None): + if concurrency_level is None: + return f"{prefix}_seq_{protocol_key}_{run_id}_{index:03d}" + return f"{prefix}_c{concurrency_level}_{protocol_key}_{run_id}_{index:03d}" + + +def format_vm_name(prefix, protocol_key, run_id, index, concurrency_level=None): + """VM names are hostnames: underscores in the run id become hyphens.""" + kind = "seq" if concurrency_level is None else f"c{concurrency_level}" + raw = f"{prefix}-{kind}-{protocol_key}-{run_id}-{index:03d}" + return raw.replace("_", "-") + + +def vm_name_token(value): + """Same underscore-to-hyphen rewrite used by format_vm_name, for cleanup filters.""" + return (value or "").replace("_", "-") + + +def assert_pool_name_prefix(prefix): + if not prefix or not re.fullmatch(r"[A-Za-z0-9_]+", prefix): + sys.exit( + f"benchmark.pool_name_prefix {prefix!r} must be letters, digits, and underscores " + "(storage pool names disallow '-')." + ) + + +def assert_vm_name_prefix(prefix): + if not prefix or "_" in prefix or not re.fullmatch(r"[A-Za-z0-9-]+", prefix): + sys.exit( + f"vm_bench.vm_name_prefix {prefix!r} must be letters, digits, and hyphens " + "(VM names are hostnames and disallow '_')." + ) diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py new file mode 100644 index 000000000000..af96f5a1779f --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Combined (sequential + concurrency) VM-instance benchmark for the NetApp +ONTAP CloudStack plugin. + +Convenience wrapper that runs benchmark_vm_instance_sequential.py's sequential +scale matrix immediately followed by benchmark_vm_instance_concurrency.py's +parallel matrix, for each protocol, under a single shared +--run-id - equivalent to running both scripts back to back, but in one +invocation and one combined results/summary_vm_.csv. + +Prefer the two standalone scripts (benchmark_vm_instance_sequential.py / +benchmark_vm_instance_concurrency.py) when you want to review the sequential +results before deciding whether to proceed with a concurrency run - this +combined script is for when you already trust the config/environment and just +want the full sequential and parallel matrix in one go. + +Protocols and concurrency levels come from config.yaml. Prefer run.py. + +Usage: + python3 run.py --test vm-instance-combined --config config.yaml + python3 benchmark_vm_instance_combined.py --config config.yaml --dry-run + python3 benchmark_vm_instance_combined.py --config config.yaml --cleanup-only +""" + +import argparse +import os + +from benchmark_support import ( + assert_vm_name_prefix, + configure_logging, + get_logger, + normalize_run_id, +) +from benchmark_vm_instance_concurrency import run_concurrency +from benchmark_vm_instance_sequential import run_sequential +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + load_config, + make_cloudstack_client, + RawLogger, + resolve_protocols, +) + +log = get_logger() + + +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_vm_name_prefix(cfg["vm_bench"]["vm_name_prefix"]) + output_dir = cfg["vm_bench"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) + + if cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = cfg["vm_bench"]["vm_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) + return + + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + levels = sorted(cfg["vm_bench"]["concurrency_levels"]) + + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: sequential + concurrency (combined)") + log.info("Concurrency levels: %s", levels) + log.info("Dry run: %s", dry_run) + + client = None if dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay) + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info( + "Next: python3 render_report.py --run-id %s --output-dir %s --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm", + run_id, output_dir, + ) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned VMs remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy benchmark VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py new file mode 100644 index 000000000000..19ec53b80b8e --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Concurrency VM-instance benchmark for the NetApp ONTAP CloudStack plugin. + +Drives deployVirtualMachine / destroyVirtualMachine in parallel +(ThreadPoolExecutor) over the CloudStack HTTP/REST API against pre- +provisioned bench_vm_nfs3 / bench_vm_iscsi storage pools (see README.md +"VM instance benchmark prerequisites"). Concurrency levels come from config.yaml: + + Parallel VM creation + Parallel VM deletion + +Each deployed VM gets a root disk (from the service offering) AND one data +disk (from the disk offering) landing on the SAME tagged storage pool. + +See benchmark_vm_instance_sequential.py for the sequential scale matrix, or +benchmark_vm_instance_combined.py to run both in one invocation (kept as +separate scripts/entry points on purpose - concurrency runs are the ones most +likely to hit mgmt-server/plugin job-queue saturation or pool capacity limits, +so they're easy to run, review, and re-run independently of the sequential +matrix). + +Every individual API call is logged to results/raw_ops_vm_.csv, and a +per-checkpoint roll-up is appended to results/summary_vm_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_vm_instance_sequential.py +combines both into a single summary_vm_.csv / report. + +Protocols and concurrency levels come from config.yaml. Prefer run.py. + +Usage: + python3 run.py --test vm-instance-concurrency --config config.yaml + python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run + python3 benchmark_vm_instance_concurrency.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from benchmark_support import ( + assert_vm_name_prefix, + configure_logging, + format_vm_name, + get_logger, + mean_seconds, + normalize_run_id, +) +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + deploy_vm, + destroy_vm, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + RawLogger, + resolve_protocols, +) + +log = get_logger() + + +def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): + infra = cfg["infrastructure"] + vm_cfg = cfg["vm_bench"] + proto_cfg = vm_cfg["protocols"][protocol_key] + prefix = vm_cfg["vm_name_prefix"] + + for level in levels: + log.info("Parallel VM create protocol=%s concurrency=%s", protocol_key, level) + names = [format_vm_name(prefix, protocol_key, run_id, i, concurrency_level=level) + for i in range(1, level + 1)] + + t0 = time.perf_counter() + results = [] + if dry_run: + results = [fake_create(n) for n in names] + else: + with ThreadPoolExecutor(max_workers=level) as ex: + futures = {ex.submit(deploy_vm, client, n, infra, vm_cfg, proto_cfg): n for n in names} + for fut in as_completed(futures): + results.append(fut.result()) + wall = time.perf_counter() - t0 + + for idx, r in enumerate(results, start=1): + raw_logger.log(run_id, "concurrent_create", "vm_concurrent_create", protocol_key, level, idx, r) + + succ = [r for r in results if r.success] + fail = [r for r in results if not r.success] + avg = mean_seconds([r.duration_sec for r in succ]) + notes = "Watch mgmt-server job-queue/thread-pool + single-host KVM saturation as a confound" if level >= 20 else "" + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_create", "test_id": "vm_concurrent_create", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), + "success_count": len(succ), "failure_count": len(fail), "notes": notes, + }) + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall, avg, len(succ), len(fail), + ) + for r in fail: + log.error("%s: %s", r.vm_name, r.error) + + log.info("Parallel VM delete protocol=%s concurrency=%s", protocol_key, len(succ)) + t0 = time.perf_counter() + del_results = [] + if dry_run: + del_results = [fake_delete(r.vm_id, r.vm_name) for r in succ] + elif succ: + with ThreadPoolExecutor(max_workers=len(succ)) as ex: + futures = {ex.submit(destroy_vm, client, r.vm_id, r.vm_name): r for r in succ} + for fut in as_completed(futures): + del_results.append(fut.result()) + wall_del = time.perf_counter() - t0 + + for idx, r in enumerate(del_results, start=1): + raw_logger.log(run_id, "concurrent_delete", "vm_concurrent_delete", protocol_key, level, idx, r) + + succ_d = [r for r in del_results if r.success] + fail_d = [r for r in del_results if not r.success] + avg_d = mean_seconds([r.duration_sec for r in succ_d]) + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_delete", "test_id": "vm_concurrent_delete", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), + "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", + }) + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall_del, avg_d, len(succ_d), len(fail_d), + ) + for r in fail_d: + log.error("%s: %s", r.vm_name, r.error) + if delay: + time.sleep(delay) + + +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_vm_name_prefix(cfg["vm_bench"]["vm_name_prefix"]) + output_dir = cfg["vm_bench"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) + + if cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = cfg["vm_bench"]["vm_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) + return + + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + levels = sorted(cfg["vm_bench"]["concurrency_levels"]) + + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: concurrency") + log.info("Concurrency levels: %s", levels) + log.info("Dry run: %s", dry_run) + + client = None if dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info( + "Next: python3 render_report.py --run-id %s --output-dir %s --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm", + run_id, output_dir, + ) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned VMs remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy benchmark VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py new file mode 100644 index 000000000000..76afabe21954 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Sequential VM-instance benchmark for the NetApp ONTAP CloudStack plugin. + +Drives deployVirtualMachine / destroyVirtualMachine one at a time over the +CloudStack HTTP/REST API against pre-provisioned bench_vm_nfs3 / bench_vm_iscsi +storage pools (see README.md "VM instance benchmark prerequisites"). +Checkpoints come from config.yaml: + + Sequential create (1 VM at a time, cumulative to N) + Sequential delete (1 VM at a time, from N remaining) + +Each deployed VM gets a root disk (from the service offering) AND one data +disk (from the disk offering) landing on the SAME tagged storage pool. + +See benchmark_vm_instance_concurrency.py for the parallel/concurrency matrix, +or benchmark_vm_instance_combined.py to run both in one invocation (kept as +separate scripts/entry points on purpose, so a sequential run can be kicked +off and reviewed independently before committing to a concurrency run). + +Every individual API call is logged to results/raw_ops_vm_.csv, and a +per-checkpoint roll-up is appended to results/summary_vm_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_vm_instance_concurrency.py +combines both into a single summary_vm_.csv / report. + +Protocols and checkpoints come from config.yaml. Prefer run.py. + +Usage: + python3 run.py --test vm-instance-sequential --config config.yaml + python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run + python3 benchmark_vm_instance_sequential.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time + +from benchmark_support import ( + assert_vm_name_prefix, + configure_logging, + format_vm_name, + get_logger, + mean_seconds, + normalize_run_id, +) +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + deploy_vm, + destroy_vm, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + RawLogger, + resolve_protocols, +) + +log = get_logger() + + +def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): + infra = cfg["infrastructure"] + vm_cfg = cfg["vm_bench"] + proto_cfg = vm_cfg["protocols"][protocol_key] + checkpoints = sorted(vm_cfg["sequential_checkpoints"]) + max_n = max(checkpoints) + prefix = vm_cfg["vm_name_prefix"] + + created = [] + create_durations = [] + log.info("Sequential VM create protocol=%s up to N=%s", protocol_key, max_n) + for i in range(1, max_n + 1): + name = format_vm_name(prefix, protocol_key, run_id, i) + result = fake_create(name) if dry_run else deploy_vm(client, name, infra, vm_cfg, proto_cfg) + raw_logger.log(run_id, "sequential_create", "vm_sequential_create", protocol_key, max_n, i, result) + status = "OK" if result.success else "FAIL" + log.info("[%s/%s] create %s -> %s (%.3fs)", i, max_n, name, status, result.duration_sec) + if result.success: + created.append((name, result.vm_id)) + create_durations.append(result.duration_sec) + else: + log.error("%s", result.error) + if i in checkpoints: + total = sum(create_durations) + avg = mean_seconds(create_durations) + summary_rows.append({ + "run_id": run_id, "phase": "sequential_create", "test_id": "vm_sequential_create", + "protocol": protocol_key, "checkpoint": i, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": len(created), "failure_count": i - len(created), "notes": "", + }) + log.info( + "checkpoint N=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + i, protocol_key, total, avg, len(created), i - len(created), + ) + if delay: + time.sleep(delay) + + total_created = len(created) + log.info("Sequential VM delete protocol=%s from N=%s remaining", protocol_key, total_created) + if total_created in checkpoints: + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "vm_sequential_delete", + "protocol": protocol_key, "checkpoint": total_created, + "total_time_sec": 0, "avg_time_sec": 0, "success_count": 0, "failure_count": 0, + "notes": "baseline - no deletes issued yet", + }) + delete_durations = [] + deleted_ok = 0 + for idx, (name, vm_id) in enumerate(created, start=1): + result = fake_delete(vm_id, name) if dry_run else destroy_vm(client, vm_id, name) + remaining = total_created - idx + raw_logger.log(run_id, "sequential_delete", "vm_sequential_delete", protocol_key, total_created, idx, result) + status = "OK" if result.success else "FAIL" + log.info( + "[%s/%s] delete %s -> %s (%.3fs) remaining=%s", + idx, total_created, name, status, result.duration_sec, remaining, + ) + if result.success: + delete_durations.append(result.duration_sec) + deleted_ok += 1 + else: + log.error("%s", result.error) + if remaining in checkpoints: + total = sum(delete_durations) + avg = mean_seconds(delete_durations) + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "vm_sequential_delete", + "protocol": protocol_key, "checkpoint": remaining, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", + }) + log.info( + "checkpoint remaining=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + remaining, protocol_key, total, avg, deleted_ok, idx - deleted_ok, + ) + if delay: + time.sleep(delay) + + +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_vm_name_prefix(cfg["vm_bench"]["vm_name_prefix"]) + output_dir = cfg["vm_bench"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) + + if cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = cfg["vm_bench"]["vm_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) + return + + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: sequential") + log.info("Dry run: %s", dry_run) + + client = None if dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info( + "Next: python3 render_report.py --run-id %s --output-dir %s --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm", + run_id, output_dir, + ) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned VMs remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy benchmark VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/cloudstack_client.py b/private-cicd/benchmark/ontap/cloudstack_client.py new file mode 100644 index 000000000000..0baed80b9c1e --- /dev/null +++ b/private-cicd/benchmark/ontap/cloudstack_client.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Minimal CloudStack HTTP/REST API client used by the ONTAP plugin benchmark +scripts (private-cicd/benchmark/ontap/). + +Only session-key based auth (login -> sessionkey + JSESSIONID cookie) is +implemented. API key/secret signed requests are not. + +Every call is timed end-to-end (including async job polling, since that is +part of the wall-clock cost an operator/automation actually pays) and the +elapsed time is handed back to the caller so benchmark scripts can log it. +""" + +import logging +import time + +import requests + +log = logging.getLogger("ontap.benchmark.client") + + +class CloudStackAPIError(Exception): + """Raised when CloudStack returns an errorcode/errortext, or the HTTP + call itself fails validation.""" + + def __init__(self, command, detail): + self.command = command + self.detail = detail + super().__init__(f"{command} failed: {detail}") + + +class CloudStackClient: + """Thin wrapper around the CloudStack query API. + + A single instance (and its underlying requests.Session) is safe to share + across threads for concurrency benchmarks: we never mutate session state + after login, and urllib3's connection pool is designed for concurrent + use. + """ + + def __init__( + self, + api_url, + username, + password, + verify_ssl=True, + http_timeout_sec=30, + job_timeout_sec=300, + job_poll_interval_sec=1.5, + ): + self.api_url = api_url.rstrip("/") + self.http_timeout_sec = http_timeout_sec + self.job_timeout_sec = job_timeout_sec + self.job_poll_interval_sec = job_poll_interval_sec + + self.session = requests.Session() + self.session.verify = verify_ssl + if not verify_ssl: + requests.packages.urllib3.disable_warnings( + requests.packages.urllib3.exceptions.InsecureRequestWarning + ) + + self.sessionkey = None + self._login(username, password) + + def _login(self, username, password): + params = { + "command": "login", + "username": username, + "password": password, + "response": "json", + } + resp = self.session.post(self.api_url, data=params, timeout=self.http_timeout_sec) + resp.raise_for_status() + data = resp.json() + login_resp = data.get("loginresponse") + if not login_resp or "sessionkey" not in login_resp: + raise CloudStackAPIError("login", data) + self.sessionkey = login_resp["sessionkey"] + log.info("Logged in as %s, session established", username) + + def call(self, command, params=None, poll_async=True): + """Issue one CloudStack API command. + + Returns a tuple (payload, elapsed_sec). `elapsed_sec` covers the + initial HTTP round trip AND any async job polling (i.e. the full + wall-clock time a caller would observe waiting for the operation to + finish). + """ + req_params = dict(params or {}) + req_params["command"] = command + req_params["response"] = "json" + req_params["sessionkey"] = self.sessionkey + + start = time.perf_counter() + resp = self.session.post(self.api_url, data=req_params, timeout=self.http_timeout_sec) + resp.raise_for_status() + data = resp.json() + + # Most commands wrap their payload as {"response": {...}}, but a + # few (e.g. enableStorageMaintenance -> prepareprimarystorageformaintenanceresponse) + # use a legacy internal command name instead. Since CloudStack always wraps + # the payload in exactly one top-level key, fall back to "the sole value" + # rather than assuming the key name matches the command. + resp_key = f"{command.lower()}response" + if resp_key in data: + payload = data[resp_key] + elif len(data) == 1: + payload = next(iter(data.values())) + else: + payload = data + + if isinstance(payload, dict) and "errorcode" in payload and "jobid" not in payload: + raise CloudStackAPIError(command, payload.get("errortext", payload)) + + if poll_async and isinstance(payload, dict) and "jobid" in payload: + payload = self._poll_job(payload["jobid"]) + + elapsed = time.perf_counter() - start + return payload, elapsed + + def _poll_job(self, jobid): + deadline = time.time() + self.job_timeout_sec + while time.time() < deadline: + params = { + "command": "queryAsyncJobResult", + "jobid": jobid, + "response": "json", + "sessionkey": self.sessionkey, + } + resp = self.session.post(self.api_url, data=params, timeout=self.http_timeout_sec) + resp.raise_for_status() + job = resp.json().get("queryasyncjobresultresponse", {}) + status = job.get("jobstatus", 0) + if status == 1: + return job.get("jobresult", job) + if status == 2: + raise CloudStackAPIError( + "queryAsyncJobResult", job.get("jobresult", job) + ) + time.sleep(self.job_poll_interval_sec) + raise TimeoutError(f"Async job {jobid} did not complete within {self.job_timeout_sec}s") diff --git a/private-cicd/benchmark/ontap/config.example.yaml b/private-cicd/benchmark/ontap/config.example.yaml new file mode 100644 index 000000000000..b75738b26f91 --- /dev/null +++ b/private-cicd/benchmark/ontap/config.example.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Copy this file to config.yaml and fill in your lab's real values. +# config.yaml is gitignored - never commit real credentials/IPs. + +cloudstack: + api_url: "http://10.193.56.12:8080/client/api" + username: "admin" + password: "password" + verify_ssl: false # CloudStack mgmt server usually plain http/self-signed + # createStoragePool is synchronous from the client's POV and internally polls + # ONTAP jobs for up to 60s (StorageStrategy.jobPollForSuccess: 30 retries * + # 2000ms) before returning. Under C=20-30 concurrency, createStoragePool can + # legitimately take close to that full 60s. A 30s client timeout was firing + # "Read timed out" on calls that actually succeeded moments later on the + # server (confirmed via real CloudStack pool records found + deleted during + # cleanup) - so keep meaningful headroom above the plugin's poll budget. + http_timeout_sec: 90 # per HTTP request + job_timeout_sec: 300 # max wait for an async job (enable/cancel maintenance) + job_poll_interval_sec: 1.5 + +# Zone/pod/cluster the pools will be created under. scope=cluster needs +# podid+clusterid; scope=zone only needs zoneid. +infrastructure: + zoneid: "91659187-b00b-4997-8ae3-8955c05a7d48" + podid: "004494cc-8a24-4519-9cd4-5e68b2eff03a" + clusterid: "23788acb-1a63-45a9-8c61-5ed8f5374bc6" + scope: "cluster" + +# One entry per protocol you want to benchmark. Keys must be "nfs3"/"iscsi". +# Remove a block to skip that protocol. +# NOTE: the ONTAP plugin (StorageStrategy.java) builds its REST API base URL +# from storageIP for EVERY call (aggregates/volumes/svm/jobs/network/san/nas) - +# it never falls back to `url`. storageIP MUST answer HTTPS/443 with the ONTAP +# REST service (i.e. it needs the same management-https access as the mgmt +# LIF). A pure data-only LIF (NFS/iSCSI only, no REST) will fail with +# "errorcode":530 / "Connection refused" on createStoragePool. If your data +# LIFs don't have REST enabled, point storageIP at the mgmt LIF instead (as +# below) rather than the actual data LIF. +ontap: + nfs3: + provider: "NetApp ONTAP" + url: "https://10.0.0.10" # ONTAP mgmt LIF URL + storageIP: "10.0.0.10" # must answer HTTPS/443 (see NOTE above) + svmName: "vs0" + protocol: "NFS3" + username: "admin" + password_b64: "cGFzc3dvcmQ=" # base64-encoded ONTAP password (this is "password" - replace it) + capacitybytes: 10737418240 # 10 GiB per pool + tags: "bench_nfs3" + iscsi: + provider: "NetApp ONTAP" + url: "https://10.0.0.11" + storageIP: "10.0.0.11" # must answer HTTPS/443 (see NOTE above) + svmName: "vs0" + protocol: "ISCSI" + username: "admin" + password_b64: "cGFzc3dvcmQ=" + capacitybytes: 10737418240 + tags: "bench_iscsi" + +benchmark: + pool_name_prefix: "bench_ontap" # ONTAP volume names disallow "-"; use "_" only + # Select any configured protocols without deleting their ontap blocks. + enabled_protocols: [nfs3, iscsi] + # Checkpoints and concurrency levels are the combinations for a run. + # Change them here. + sequential_checkpoints: [1, 5, 10, 20, 30] + concurrency_levels: [2, 5, 10, 20, 30] + # Pause between deleteStoragePool calls to give the mgmt server GC/ONTAP a + # beat; set to 0 to hammer as fast as possible. + inter_op_delay_sec: 0 + output_dir: "results" + +# VM instance benchmark (benchmark_vm_instance_sequential.py / +# benchmark_vm_instance_concurrency.py / benchmark_vm_instance_combined.py). Requires: +# - a template (templateid) +# - a network (networkid) +# - per protocol: a service offering (root disk) + disk offering (data disk) +# whose storage `tags` both match a persistent bench_vm_ pool, so +# root and data disk land on the same pool. These are one-time setup +# (createServiceOffering / createDiskOffering / createStoragePool) - not +# created/destroyed by the benchmark itself, unlike the storage-pool +# benchmark's transient pools. +# NOTE: watch aggregate capacity - StorageStrategy.java's aggregate-selection +# pre-check requires available space >= requested size even though volumes +# are created with guarantee=none (thin). On small/vsim aggregates this can +# reject valid requests; keep pool/disk sizes modest until that's addressed. +vm_bench: + templateid: "1af26eaa-8985-4b1c-ae8e-d39b7984b8f3" + networkid: "5e6f4c59-dc01-4245-8c0f-2c1c9639b37d" + # CloudStack VM names are used as hostnames (RFC1123): letters/digits/"-" only, + # no "_" (the opposite of ONTAP's volume-name rule used by pool_name_prefix above). + vm_name_prefix: "bench-vm" + # Unattached data disks are deleted only from pools with this name prefix, + # and only when that pool's provider and zone match this config. + cleanup_pool_name_prefix: "bench_vm" + # Select any configured protocols without deleting their offering definitions. + enabled_protocols: [nfs3, iscsi] + sequential_checkpoints: [1, 5, 10, 20] + concurrency_levels: [2, 5, 10] + inter_op_delay_sec: 0 + output_dir: "results" + protocols: + nfs3: + # rootdisksize (GB, set on the offering) must be >= the template's + # actual qemu-img virtual size, not its download/file size - e.g. the + # stock Ubuntu 22.04 cloud image download is ~660MB on disk but has a + # 2.2 GiB virtual size. Undersizing this "works" on NFS (the backing + # file just grows past its nominal size) but hard-fails on iSCSI with + # "qemu-img: Cannot grow device files" since a LUN is a fixed-size + # block device that cannot be grown during the template copy. + serviceofferingid: "ac54f38c-5fd3-401b-b784-2c2163b08274" + diskofferingid: "e27813c0-6e8b-4375-8603-14c8ade75246" + iscsi: + serviceofferingid: "481feb00-16d8-4127-9937-cba3bec8a88b" + diskofferingid: "bf057b78-c5db-4ce4-820b-d7bd78f8ae69" diff --git a/private-cicd/benchmark/ontap/render_report.py b/private-cicd/benchmark/ontap/render_report.py new file mode 100644 index 000000000000..d76e4c6b1c0f --- /dev/null +++ b/private-cicd/benchmark/ontap/render_report.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Turn a benchmark run's raw and summary CSVs into markdown tables for the +storage-pool and VM-instance sequential and parallel runs, plus a results log. + +Usage: + python3 render_report.py --run-id RUN_20260722_120000_ab12 \ + --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 +""" + +import argparse +import csv +import math +import os +from collections import defaultdict + +PROTOCOL_ORDER = ["nfs3", "iscsi"] +PROTOCOL_LABEL = {"nfs3": "NFS3", "iscsi": "iSCSI"} + +# test_id -> (title, sequential N-column label, category, phase) +SECTION_DEFS = { + "pool_sequential_create": ( + "Sequential create (1 pool at a time, cumulative to N)", + "N (pools)", "Storage Pool", "sequential_create"), + "pool_sequential_delete": ( + "Sequential delete (1 pool at a time, from N remaining)", + "N (pools remaining)", "Storage Pool", "sequential_delete"), + "pool_concurrent_create": ("Parallel pool creation", None, "Storage Pool", "concurrent_create"), + "pool_concurrent_delete": ("Parallel pool deletion", None, "Storage Pool", "concurrent_delete"), + "vm_sequential_create": ( + "Sequential create (1 VM at a time, cumulative to N)", + "N (VMs)", "VM Instance", "sequential_create"), + "vm_sequential_delete": ( + "Sequential delete (1 VM at a time, from N remaining)", + "N (VMs remaining)", "VM Instance", "sequential_delete"), + "vm_concurrent_create": ("Parallel VM creation", None, "VM Instance", "concurrent_create"), + "vm_concurrent_delete": ("Parallel VM deletion", None, "VM Instance", "concurrent_delete"), +} + + +def percentile(sorted_values, pct): + if not sorted_values: + return None + if len(sorted_values) == 1: + return sorted_values[0] + k = (len(sorted_values) - 1) * (pct / 100.0) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_values[int(k)] + d0 = sorted_values[f] * (c - k) + d1 = sorted_values[c] * (k - f) + return d0 + d1 + + +def load_csv(path): + if not os.path.exists(path): + return [] + with open(path, newline="") as f: + return list(csv.DictReader(f)) + + +def stats_for(durations): + if not durations: + return None, None, None, None + durations = sorted(durations) + return ( + round(durations[0], 3), + round(sum(durations) / len(durations), 3), + round(percentile(durations, 95), 3), + round(durations[-1], 3), + ) + + +def raw_durations(raw_rows, phase, protocol, predicate): + return [ + float(r["duration_sec"]) for r in raw_rows + if r["phase"] == phase and r["protocol"] == protocol + and r["success"] == "True" and predicate(r) + ] + + +def render_sequential_table(summary_rows, raw_rows, phase, test_id, n_label): + checkpoints = sorted({int(r["checkpoint"]) for r in summary_rows if r["test_id"] == test_id}) + by_key = {(r["protocol"], int(r["checkpoint"])): r for r in summary_rows if r["test_id"] == test_id} + + lines = [f"| {n_label} | NFS3 Total (s) | NFS3 Avg/op (s) | iSCSI Total (s) | iSCSI Avg/op (s) | Notes |", + "|---|---|---|---|---|---|"] + for cp in checkpoints: + cells = [str(cp)] + notes = [] + for proto in PROTOCOL_ORDER: + row = by_key.get((proto, cp)) + if row: + cells.append(row["total_time_sec"]) + cells.append(row["avg_time_sec"]) + if row.get("notes") and row["notes"] not in notes: + notes.append(row["notes"]) + else: + cells.append("") + cells.append("") + cells.append("; ".join(notes)) + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def render_concurrency_table(summary_rows, test_id): + rows = [r for r in summary_rows if r["test_id"] == test_id] + rows.sort(key=lambda r: (int(r["checkpoint"]), PROTOCOL_ORDER.index(r["protocol"]) + if r["protocol"] in PROTOCOL_ORDER else 99)) + lines = ["| Concurrency (C) | Protocol | Total Wall-clock (s) | Success | Failure | Avg Time/op (s) | Notes |", + "|---|---|---|---|---|---|---|"] + for r in rows: + lines.append( + f"| {r['checkpoint']} | {PROTOCOL_LABEL.get(r['protocol'], r['protocol'])} | " + f"{r['total_time_sec']} | {r['success_count']} | {r['failure_count']} | " + f"{r['avg_time_sec']} | {r.get('notes', '')} |" + ) + return "\n".join(lines) + + +def build_results_log_rows(summary_rows, raw_rows, run_id, cs_build, ontap_version, run_date): + header = ("| Run Date | Run ID | CloudStack Build | ONTAP Version | Category | Test ID | " + "Scale (N) / Concurrency (C) | Protocol | Min (s) | Avg (s) | P95 (s) | Max (s) | " + "Success Rate | Notes |") + sep = "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|" + lines = [header, sep] + + delete_total_created = {} + for r in raw_rows: + if r["phase"] == "sequential_delete": + delete_total_created[r["protocol"]] = int(r["scale_or_concurrency"]) + + for row in summary_rows: + phase, test_id, protocol, checkpoint = row["phase"], row["test_id"], row["protocol"], int(row["checkpoint"]) + if phase == "sequential_create": + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["index"]) <= checkpoint) + scale_label = f"N={checkpoint}" + elif phase == "sequential_delete": + total_created = delete_total_created.get(protocol, checkpoint) + deleted_so_far = total_created - checkpoint + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["index"]) <= deleted_so_far) + scale_label = f"N={checkpoint}" + else: # concurrent_create / concurrent_delete + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["scale_or_concurrency"]) == checkpoint) + scale_label = f"C={checkpoint}" + + mn, avg, p95, mx = stats_for(durations) + total_ops = int(row["success_count"]) + int(row["failure_count"]) + success_rate = f"{(int(row['success_count']) / total_ops * 100):.0f}%" if total_ops else "-" + category = SECTION_DEFS.get(test_id, (None, None, "Storage Pool", None))[2] + lines.append( + f"| {run_date} | {run_id} | {cs_build} | {ontap_version} | {category} | {test_id} | " + f"{scale_label} | {PROTOCOL_LABEL.get(protocol, protocol)} | " + f"{mn if mn is not None else '-'} | {avg if avg is not None else '-'} | " + f"{p95 if p95 is not None else '-'} | {mx if mx is not None else '-'} | " + f"{success_rate} | {row.get('notes', '')} |" + ) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--run-id", required=True) + parser.add_argument("--output-dir", default="results") + parser.add_argument("--cloudstack-build", default="unknown") + parser.add_argument("--ontap-version", default="unknown") + parser.add_argument("--run-date", default=None, help="Defaults to today's date (YYYY-MM-DD)") + parser.add_argument("--raw-prefix", default="raw_ops", + help="Filename prefix for the raw ops CSV, e.g. raw_ops_vm for the benchmark_vm_instance_* scripts") + parser.add_argument("--summary-prefix", default="summary", + help="Filename prefix for the summary CSV, e.g. summary_vm for the benchmark_vm_instance_* scripts") + parser.add_argument("--report-suffix", default="", + help="Optional suffix to disambiguate the output report filename, e.g. _vm") + args = parser.parse_args() + + import datetime + run_date = args.run_date or datetime.date.today().isoformat() + + summary_path = os.path.join(args.output_dir, f"{args.summary_prefix}_{args.run_id}.csv") + raw_path = os.path.join(args.output_dir, f"{args.raw_prefix}_{args.run_id}.csv") + summary_rows = load_csv(summary_path) + raw_rows = load_csv(raw_path) + if not summary_rows: + raise SystemExit(f"No summary rows found at {summary_path}") + + # Render whichever tests actually have data in this run, in a stable order. + present_test_ids = {r["test_id"] for r in summary_rows} + sections = [] + for test_id, (title, n_label, _category, phase) in SECTION_DEFS.items(): + if test_id not in present_test_ids: + continue + if n_label is not None: + sections.append(f"### {title}\n\n" + + render_sequential_table(summary_rows, raw_rows, phase, test_id, n_label)) + else: + sections.append(f"### {title}\n\n" + render_concurrency_table(summary_rows, test_id)) + sections.append("### Results log\n\n" + + build_results_log_rows(summary_rows, raw_rows, args.run_id, args.cloudstack_build, + args.ontap_version, run_date)) + + report = f"# Benchmark report - {args.run_id}\n\n" + "\n\n".join(sections) + "\n" + print(report) + + report_path = os.path.join(args.output_dir, f"report{args.report_suffix}_{args.run_id}.md") + with open(report_path, "w") as f: + f.write(report) + print(f"\nSaved to {report_path}") + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/requirements.txt b/private-cicd/benchmark/ontap/requirements.txt new file mode 100644 index 000000000000..ee9fb649d666 --- /dev/null +++ b/private-cicd/benchmark/ontap/requirements.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +requests>=2.31.0 +PyYAML>=6.0 diff --git a/private-cicd/benchmark/ontap/results/.gitkeep b/private-cicd/benchmark/ontap/results/.gitkeep new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/private-cicd/benchmark/ontap/results/.gitkeep @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/private-cicd/benchmark/ontap/run.py b/private-cicd/benchmark/ontap/run.py new file mode 100644 index 000000000000..691150da5b6c --- /dev/null +++ b/private-cicd/benchmark/ontap/run.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Single entry point for the ONTAP scale benchmarks. + +Protocols, checkpoints, and concurrency levels come from config.yaml. Drop a +protocol block to skip it, and edit sequential_checkpoints or concurrency_levels +to change the matrix. The selected test reads those fields as written. + +Usage: + python3 run.py --test storage-pool-sequential --config config.yaml + python3 run.py --test storage-pool-concurrency --config config.yaml --dry-run + python3 run.py --test vm-instance-combined --config config.yaml + python3 run.py --test vm-instance-sequential --config config.yaml --cleanup-only +""" + +import argparse + +from benchmark_storage_pool_concurrency import execute as run_storage_pool_concurrency +from benchmark_storage_pool_sequential import execute as run_storage_pool_sequential +from benchmark_support import configure_logging +from benchmark_vm_instance_combined import execute as run_vm_instance_combined +from benchmark_vm_instance_concurrency import execute as run_vm_instance_concurrency +from benchmark_vm_instance_sequential import execute as run_vm_instance_sequential +from storage_pool_common import load_config + +TESTS = { + "storage-pool-sequential": run_storage_pool_sequential, + "storage-pool-concurrency": run_storage_pool_concurrency, + "vm-instance-sequential": run_vm_instance_sequential, + "vm-instance-concurrency": run_vm_instance_concurrency, + "vm-instance-combined": run_vm_instance_combined, +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--test", required=True, choices=sorted(TESTS), help="Benchmark to run") + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Reuse a run id (auto-generated timestamp + random suffix otherwise)") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave leftovers from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete benchmark resources whose name contains FILTER (default: configured name prefix) and exit", + ) + args = parser.parse_args() + + configure_logging() + cfg = load_config(args.config) + runner = TESTS[args.test] + runner( + cfg, + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/storage_pool_common.py b/private-cicd/benchmark/ontap/storage_pool_common.py new file mode 100644 index 000000000000..21d626ee23d8 --- /dev/null +++ b/private-cicd/benchmark/ontap/storage_pool_common.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Shared helpers for the storage-pool benchmark scripts +(benchmark_storage_pool_sequential.py / benchmark_storage_pool_concurrency.py). + +Kept in one module (rather than duplicated per script) so the createStoragePool/ +deleteStoragePool call shape, CSV formats, and cleanup logic can't drift between +the two - only the sequential-vs-concurrent driving logic differs per script. +""" + +import csv +import dataclasses +import datetime +import os +import random +import re +import sys +import time + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install -r requirements.txt") + +from benchmark_support import get_logger +from cloudstack_client import CloudStackAPIError, CloudStackClient + +RAW_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "scale_or_concurrency", + "index", "pool_name", "pool_id", "success", "duration_sec", + "start_ts", "end_ts", "error", +] + +SUMMARY_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "checkpoint", + "total_time_sec", "avg_time_sec", "success_count", "failure_count", "notes", +] + + +@dataclasses.dataclass +class OpResult: + success: bool + duration_sec: float + start_ts: str + end_ts: str + pool_name: str + pool_id: str = None + error: str = None + + +def now_iso(): + return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" + + +def build_create_pool_params(name, infra_cfg, ontap_cfg): + params = { + "name": name, + "zoneid": infra_cfg["zoneid"], + "scope": infra_cfg.get("scope", "cluster"), + "provider": ontap_cfg["provider"], + "url": ontap_cfg["url"], + "managed": "true", + "details[0].username": ontap_cfg["username"], + "details[0].password": ontap_cfg["password_b64"], + "details[0].svmName": ontap_cfg["svmName"], + "details[0].protocol": ontap_cfg["protocol"], + "details[0].storageIP": ontap_cfg["storageIP"], + } + if infra_cfg.get("scope", "cluster") == "cluster": + params["podid"] = infra_cfg["podid"] + params["clusterid"] = infra_cfg["clusterid"] + if ontap_cfg.get("capacitybytes"): + params["capacitybytes"] = str(ontap_cfg["capacitybytes"]) + if ontap_cfg.get("tags"): + params["tags"] = ontap_cfg["tags"] + return params + + +def create_pool(client, name, infra_cfg, ontap_cfg): + start_ts = now_iso() + t0 = time.perf_counter() + try: + payload, elapsed = client.call( + "createStoragePool", build_create_pool_params(name, infra_cfg, ontap_cfg) + ) + pool = payload.get("storagepool", payload) if isinstance(payload, dict) else {} + pool_id = pool.get("id") if isinstance(pool, dict) else None + return OpResult(True, elapsed, start_ts, now_iso(), name, pool_id=pool_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, error=str(exc)) + + +def delete_pool(client, pool_id, name, forced=True): + """deleteStoragePool requires the pool to already be in Maintenance state + (CloudStack error 431 otherwise), so this enables maintenance first and + waits for that async job before issuing the actual delete. Both steps are + included in the reported duration since that is the real wall-clock cost + of retiring a pool.""" + start_ts = now_iso() + t0 = time.perf_counter() + try: + client.call("enableStorageMaintenance", {"id": pool_id}) + payload, _ = client.call( + "deleteStoragePool", {"id": pool_id, "forced": "true" if forced else "false"} + ) + elapsed = time.perf_counter() - t0 + success = payload.get("success", True) if isinstance(payload, dict) else True + return OpResult(bool(success), elapsed, start_ts, now_iso(), name, pool_id=pool_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, pool_id=pool_id, error=str(exc)) + + +def fake_create(name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(0.5, 3.0), now_iso(), now_iso(), name, pool_id=f"dryrun-{name}") + + +def fake_delete(pool_id, name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(0.3, 2.0), now_iso(), now_iso(), name, pool_id=pool_id) + + +class RawLogger: + def __init__(self, path): + self.file = open(path, "a", newline="") + self.writer = csv.DictWriter(self.file, fieldnames=RAW_FIELDNAMES) + if self.file.tell() == 0: + self.writer.writeheader() + + def log(self, run_id, phase, test_id, protocol, scale_or_concurrency, index, result: OpResult): + self.writer.writerow({ + "run_id": run_id, "phase": phase, "test_id": test_id, "protocol": protocol, + "scale_or_concurrency": scale_or_concurrency, "index": index, + "pool_name": result.pool_name, "pool_id": result.pool_id or "", + "success": result.success, "duration_sec": round(result.duration_sec, 4), + "start_ts": result.start_ts, "end_ts": result.end_ts, "error": result.error or "", + }) + self.file.flush() + + def close(self): + self.file.close() + + +def append_summary_csv(path, rows): + """Appends checkpoint rows to the run's summary CSV, writing the header only + if the file is new/empty. Append (not overwrite) so that running the + sequential and concurrency scripts separately under the *same* --run-id + accumulates into one combined summary_.csv that render_report.py + can render as a single report, instead of the second script clobbering the + first script's rows.""" + is_new = not os.path.exists(path) or os.path.getsize(path) == 0 + with open(path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDNAMES) + if is_new: + writer.writeheader() + writer.writerows(rows) + + +def _pool_name_pattern(prefix): + return re.compile( + rf"^{re.escape(prefix)}_(?:seq|c\d+)_[A-Za-z0-9]+_[A-Za-z0-9_]+_\d{{3}}$" + ) + + +def cleanup_by_filter(client, name_filter, cfg): + """Delete benchmark pools only. + + A shared zone can contain pools this run did not create. A match has to use + this config's name prefix and naming pattern, sit in the configured zone and + cluster, and use one of the configured storage providers. + """ + log = get_logger() + if not name_filter or len(name_filter) < 4: + sys.exit("Refusing cleanup: name filter is missing or shorter than 4 characters") + + prefix = cfg["benchmark"]["pool_name_prefix"] + infra = cfg.get("infrastructure", {}) + zoneid = infra.get("zoneid") + clusterid = infra.get("clusterid") if infra.get("scope", "cluster") == "cluster" else None + providers = { + str(entry.get("provider", "")).strip().lower() + for entry in cfg.get("ontap", {}).values() + if entry.get("provider") + } + pattern = _pool_name_pattern(prefix) + + payload, _ = client.call("listStoragePools", {}, poll_async=False) + pools = payload.get("storagepool", []) if isinstance(payload, dict) else [] + name_hits = [p for p in pools if name_filter in (p.get("name") or "")] + matches = [] + for pool in name_hits: + name = pool.get("name") or "" + provider = str(pool.get("provider") or "").strip().lower() + if not pattern.match(name): + continue + if providers and provider not in providers: + continue + if zoneid and pool.get("zoneid") and pool.get("zoneid") != zoneid: + continue + if clusterid and pool.get("clusterid") and pool.get("clusterid") != clusterid: + continue + matches.append(pool) + + skipped = len(name_hits) - len(matches) + log.info("Found %s pool(s) eligible for cleanup (filter=%r)", len(matches), name_filter) + if skipped: + log.warning( + "Skipped %s pool(s) whose name contains %r but failed the prefix, zone, cluster, or provider check", + skipped, name_filter, + ) + for pool in matches: + try: + if pool.get("state") != "Maintenance": + client.call("enableStorageMaintenance", {"id": pool["id"]}) + client.call("deleteStoragePool", {"id": pool["id"], "forced": "true"}) + log.info("deleted %s (%s)", pool["name"], pool["id"]) + except Exception as exc: # noqa: BLE001 + log.error("FAILED to delete %s: %s", pool["name"], exc) + + +def load_config(path): + with open(path) as f: + return yaml.safe_load(f) + + +def resolve_protocols(cfg, requested): + available = list(cfg.get("ontap", {}).keys()) + if requested == "both": + enabled = cfg.get("benchmark", {}).get("enabled_protocols", available) + unknown = [protocol for protocol in enabled if protocol not in available] + if unknown: + sys.exit( + f"Unknown config.benchmark.enabled_protocols entries: {unknown} " + f"(available: {available})" + ) + if not enabled: + sys.exit("config.benchmark.enabled_protocols must contain at least one protocol") + return enabled + if requested not in available: + sys.exit(f"Protocol '{requested}' not found in config.ontap (available: {available})") + return [requested] + + +def make_cloudstack_client(cfg): + cs_cfg = cfg["cloudstack"] + return CloudStackClient( + cs_cfg["api_url"], cs_cfg["username"], cs_cfg["password"], + verify_ssl=cs_cfg.get("verify_ssl", True), + http_timeout_sec=cs_cfg.get("http_timeout_sec", 30), + job_timeout_sec=cs_cfg.get("job_timeout_sec", 300), + job_poll_interval_sec=cs_cfg.get("job_poll_interval_sec", 1.5), + ) diff --git a/private-cicd/benchmark/ontap/vm_instance_common.py b/private-cicd/benchmark/ontap/vm_instance_common.py new file mode 100644 index 000000000000..0e3c52de76b4 --- /dev/null +++ b/private-cicd/benchmark/ontap/vm_instance_common.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Shared helpers for the VM-instance benchmark scripts +(benchmark_vm_instance_sequential.py / benchmark_vm_instance_concurrency.py / +benchmark_vm_instance_combined.py). + +Kept in one module (rather than duplicated per script) so the +deployVirtualMachine/destroyVirtualMachine call shape, CSV formats, and +cleanup logic can't drift between the three - only the sequential-vs- +concurrent-vs-both driving logic differs per script. +""" + +import csv +import dataclasses +import datetime +import os +import random +import re +import sys +import time + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install -r requirements.txt") + +from benchmark_support import get_logger, vm_name_token +from cloudstack_client import CloudStackAPIError, CloudStackClient + +RAW_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "scale_or_concurrency", + "index", "vm_name", "vm_id", "success", "duration_sec", + "start_ts", "end_ts", "error", +] + +SUMMARY_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "checkpoint", + "total_time_sec", "avg_time_sec", "success_count", "failure_count", "notes", +] + + +@dataclasses.dataclass +class OpResult: + success: bool + duration_sec: float + start_ts: str + end_ts: str + vm_name: str + vm_id: str = None + error: str = None + + +def now_iso(): + return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" + + +def build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg): + return { + "name": name, + "displayname": name, + "zoneid": infra_cfg["zoneid"], + "templateid": vm_cfg["templateid"], + "networkids": vm_cfg["networkid"], + "serviceofferingid": proto_cfg["serviceofferingid"], + "diskofferingid": proto_cfg["diskofferingid"], + "startvm": "true", + } + + +_UUID_RE = re.compile(r'"uuid"\s*:\s*"([0-9a-fA-F-]{36})"') + + +def _extract_vm_id_from_error(error_text): + """When deployVirtualMachine's async job fails deep in orchestration (for example + an "Unable to orchestrate the start of VM instance" failure), its errortext + can embed the attempted VM's UUID. Preserve that UUID in the raw operation + result even if CloudStack subsequently removes the failed VM.""" + if not error_text: + return None + m = _UUID_RE.search(error_text) + return m.group(1) if m else None + + +def deploy_vm(client, name, infra_cfg, vm_cfg, proto_cfg): + start_ts = now_iso() + t0 = time.perf_counter() + try: + payload, elapsed = client.call( + "deployVirtualMachine", build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg) + ) + vm = payload.get("virtualmachine", payload) if isinstance(payload, dict) else {} + vm_id = vm.get("id") if isinstance(vm, dict) else None + return OpResult(True, elapsed, start_ts, now_iso(), name, vm_id=vm_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + error = str(exc) + return OpResult(False, elapsed, start_ts, now_iso(), name, + vm_id=_extract_vm_id_from_error(error), error=error) + + +def _force_purge_destroy_state_volumes(client, volume_ids, max_wait_sec=10): + """destroyVirtualMachine(expunge=true) only guarantees the VM/volume DB records + move to "Destroy" state - actual physical removal from the backing storage + (ONTAP LUN/file deletion) is deferred to CloudStack's periodic storage-cleanup + background thread, which by default only *considers* volumes for real deletion + after `storage.cleanup.delay` (86400s / 24h) has elapsed, even though the + cleanup thread itself polls every `storage.cleanup.interval` (40s by default). + Confirmed directly against our management server via listConfigurations. + + That means every "successfully destroyed" VM's ROOT/DATA disks still + physically occupy space on the ONTAP pool for up to 24h unless something + calls deleteVolume() on them explicitly - which force-deletes a Destroy-state + volume immediately, bypassing the delay. Without this, repeated benchmark + runs progressively fill up the dedicated bench_vm_ pool with + "deleted" disks, compounding ENOSPC failures on the dedicated bench pool. + """ + deadline = time.time() + max_wait_sec + remaining = set(volume_ids) + while remaining and time.time() < deadline: + still_pending = set() + for vol_id in remaining: + try: + vol_payload, _ = client.call("listVolumes", {"id": vol_id}, poll_async=False) + vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + if not vols: + continue # already fully gone + state = vols[0].get("state") + if state == "Destroy": + client.call("deleteVolume", {"id": vol_id}) + elif state not in ("Expunged",): + still_pending.add(vol_id) # not yet transitioned to Destroy - retry shortly + except (CloudStackAPIError, TimeoutError, Exception): # noqa: BLE001 + still_pending.add(vol_id) + remaining = still_pending + if remaining: + time.sleep(1) + + +def destroy_vm(client, vm_id, name, expunge=True): + """destroyVirtualMachine with expunge=true only reclaims the ROOT disk; + data disks are merely DETACHED (left behind as free-floating DATADISK + volumes) unless their ids are explicitly passed via `volumeids`. So this + looks up any attached data disks first and includes them, otherwise every + VM with a data disk leaks a LUN/file on the backing storage pool. + + It also force-purges any ROOT/DATA volumes still left in "Destroy" state + afterwards (see _force_purge_destroy_state_volumes) so this VM's disks are + truly gone from ONTAP before the next benchmark op runs, instead of + lingering for CloudStack's 24h background cleanup delay. + """ + start_ts = now_iso() + t0 = time.perf_counter() + try: + vol_payload, _ = client.call( + "listVolumes", {"virtualmachineid": vm_id}, poll_async=False + ) + all_vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + all_volume_ids = [v["id"] for v in all_vols] + data_disk_ids = [v["id"] for v in all_vols if v.get("type") == "DATADISK"] + + params = {"id": vm_id, "expunge": "true" if expunge else "false"} + if data_disk_ids: + params["volumeids"] = ",".join(data_disk_ids) + payload, _ = client.call("destroyVirtualMachine", params) + + if expunge and all_volume_ids: + _force_purge_destroy_state_volumes(client, all_volume_ids) + + elapsed = time.perf_counter() - t0 + success = payload.get("success", True) if isinstance(payload, dict) else True + return OpResult(bool(success), elapsed, start_ts, now_iso(), name, vm_id=vm_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, vm_id=vm_id, error=str(exc)) + + +def fake_create(name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(3.0, 15.0), now_iso(), now_iso(), name, vm_id=f"dryrun-{name}") + + +def fake_delete(vm_id, name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(2.0, 8.0), now_iso(), now_iso(), name, vm_id=vm_id) + + +class RawLogger: + def __init__(self, path): + self.file = open(path, "a", newline="") + self.writer = csv.DictWriter(self.file, fieldnames=RAW_FIELDNAMES) + if self.file.tell() == 0: + self.writer.writeheader() + + def log(self, run_id, phase, test_id, protocol, scale_or_concurrency, index, result: OpResult): + self.writer.writerow({ + "run_id": run_id, "phase": phase, "test_id": test_id, "protocol": protocol, + "scale_or_concurrency": scale_or_concurrency, "index": index, + "vm_name": result.vm_name, "vm_id": result.vm_id or "", + "success": result.success, "duration_sec": round(result.duration_sec, 4), + "start_ts": result.start_ts, "end_ts": result.end_ts, "error": result.error or "", + }) + self.file.flush() + + def close(self): + self.file.close() + + +def append_summary_csv(path, rows): + """Appends checkpoint rows to the run's summary CSV, writing the header only + if the file is new/empty. Append (not overwrite) so that running the + sequential and concurrency scripts separately under the *same* --run-id + accumulates into one combined summary_vm_.csv that render_report.py + can render as a single report, instead of the second script clobbering the + first script's rows.""" + is_new = not os.path.exists(path) or os.path.getsize(path) == 0 + with open(path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDNAMES) + if is_new: + writer.writeheader() + writer.writerows(rows) + + +def _vm_name_pattern(prefix): + return re.compile( + rf"^{re.escape(prefix)}-(?:seq|c\d+)-[A-Za-z0-9]+-[A-Za-z0-9-]+-\d{{3}}$" + ) + + +def _allowed_orphan_pool_names(client, cfg): + """Pools this config is allowed to reclaim unattached data disks from.""" + vm_cfg = cfg.get("vm_bench", {}) + pool_prefix = vm_cfg.get("cleanup_pool_name_prefix", "bench_vm") + infra = cfg.get("infrastructure", {}) + zoneid = infra.get("zoneid") + providers = { + str(entry.get("provider", "")).strip().lower() + for entry in cfg.get("ontap", {}).values() + if entry.get("provider") + } + payload, _ = client.call("listStoragePools", {}, poll_async=False) + pools = payload.get("storagepool", []) if isinstance(payload, dict) else [] + allowed = set() + for pool in pools: + name = pool.get("name") or "" + provider = str(pool.get("provider") or "").strip().lower() + if not name.startswith(pool_prefix): + continue + if providers and provider not in providers: + continue + if zoneid and pool.get("zoneid") and pool.get("zoneid") != zoneid: + continue + allowed.add(name) + return allowed + + +def cleanup_by_filter(client, name_filter, cfg): + """Destroy benchmark VMs only. + + VM names rewrite '_' to '-' (hostnames). The filter is rewritten the same + way, otherwise a run id such as RUN_20260927_081500 never matches the VM + that was actually created. A shared zone is protected by the name prefix, + the benchmark name pattern, and the configured zone. + """ + log = get_logger() + token = vm_name_token(name_filter) + if not token or len(token) < 4: + sys.exit("Refusing cleanup: name filter is missing or shorter than 4 characters") + + prefix = cfg["vm_bench"]["vm_name_prefix"] + zoneid = cfg.get("infrastructure", {}).get("zoneid") + pattern = _vm_name_pattern(prefix) + + payload, _ = client.call("listVirtualMachines", {}, poll_async=False) + vms = payload.get("virtualmachine", []) if isinstance(payload, dict) else [] + name_hits = [v for v in vms if token in (v.get("name") or "")] + matches = [] + for vm in name_hits: + name = vm.get("name") or "" + if not pattern.match(name): + continue + if zoneid and vm.get("zoneid") and vm.get("zoneid") != zoneid: + continue + matches.append(vm) + + skipped = len(name_hits) - len(matches) + log.info("Found %s VM(s) eligible for cleanup (filter=%r, match token=%r)", len(matches), name_filter, token) + if skipped: + log.warning( + "Skipped %s VM(s) whose name contains %r but failed the prefix, name pattern, or zone check", + skipped, token, + ) + for vm in matches: + try: + result = destroy_vm(client, vm["id"], vm["name"]) + if result.success: + log.info("destroyed %s (%s) including any data disks", vm["name"], vm["id"]) + else: + log.error("FAILED to destroy %s: %s", vm["name"], result.error) + except Exception as exc: # noqa: BLE001 + log.error("FAILED to destroy %s: %s", vm["name"], exc) + + # Data disks can also end up orphaned (unattached) from prior runs whose + # VM was already destroyed without the volumeids argument. Volume names + # (e.g. "DATA-25") don't carry the run id, so only unattached data disks + # on pools that match this config's provider, zone, and pool-name prefix + # are deleted. + allowed_pools = _allowed_orphan_pool_names(client, cfg) + vol_payload, _ = client.call("listVolumes", {"type": "DATADISK"}, poll_async=False) + vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + orphan_vols = [ + v for v in vols + if not v.get("virtualmachineid") and (v.get("storage") or "") in allowed_pools + ] + if orphan_vols: + log.info("Found %s orphaned unattached DATADISK volume(s) on configured pools", len(orphan_vols)) + for vol in orphan_vols: + try: + client.call("deleteVolume", {"id": vol["id"]}) + log.info("deleted orphan volume %s (%s)", vol["name"], vol["id"]) + except Exception as exc: # noqa: BLE001 + log.error("FAILED to delete orphan volume %s: %s", vol["name"], exc) + + +def load_config(path): + with open(path) as f: + return yaml.safe_load(f) + + +def resolve_protocols(cfg, requested): + available = list(cfg.get("vm_bench", {}).get("protocols", {}).keys()) + if requested == "both": + enabled = cfg.get("vm_bench", {}).get("enabled_protocols", available) + unknown = [protocol for protocol in enabled if protocol not in available] + if unknown: + sys.exit( + f"Unknown config.vm_bench.enabled_protocols entries: {unknown} " + f"(available: {available})" + ) + if not enabled: + sys.exit("config.vm_bench.enabled_protocols must contain at least one protocol") + return enabled + if requested not in available: + sys.exit(f"Protocol '{requested}' not found in config.vm_bench.protocols (available: {available})") + return [requested] + + +def make_cloudstack_client(cfg): + cs_cfg = cfg["cloudstack"] + return CloudStackClient( + cs_cfg["api_url"], cs_cfg["username"], cs_cfg["password"], + verify_ssl=cs_cfg.get("verify_ssl", True), + http_timeout_sec=cs_cfg.get("http_timeout_sec", 30), + job_timeout_sec=cs_cfg.get("job_timeout_sec", 300), + job_poll_interval_sec=cs_cfg.get("job_poll_interval_sec", 1.5), + ) diff --git a/private-cicd/config/vm-inventory.yaml.example b/private-cicd/config/vm-inventory.yaml.example new file mode 100644 index 000000000000..1c705dddbf16 --- /dev/null +++ b/private-cicd/config/vm-inventory.yaml.example @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Copy this schema to a protected file outside Git and replace every placeholder +# with non-secret lab inventory data. Upload the populated file to Jenkins as a +# Secret file credential; never commit it. +# Create one Jenkins Lockable Resource per entry. Its resource name must equal +# lock_resource and it must carry the label: cloudstack-presubmit-vm. + +vms: + - id: presubmit-vm-01 + enabled: true + lock_resource: PLACEHOLDER-cloudstack-presubmit-vm-01 + capabilities: [ubuntu22, kvm, ontap] + + # vCenter inventory names. + vcenter_vm: PLACEHOLDER-vcenter-vm-name + snapshot: PLACEHOLDER-clean-snapshot-name + + # Passwordless SSH from the Jenkins presubmit pod. + ssh_host: PLACEHOLDER-vm-fqdn-or-ip + ssh_user: root + bridge: cloudbr0 + + # Non-secret network allocation dedicated to this VM. + dedicated_ips: + management_server: PLACEHOLDER-management-ip + kvm_host: PLACEHOLDER-kvm-ip + public: + gateway: PLACEHOLDER-public-gateway + netmask: PLACEHOLDER-public-netmask + start: PLACEHOLDER-public-start-ip + end: PLACEHOLDER-public-end-ip + pod: + gateway: PLACEHOLDER-pod-gateway + netmask: PLACEHOLDER-pod-netmask + start: PLACEHOLDER-pod-start-ip + end: PLACEHOLDER-pod-end-ip + + zone: + name: CloudStack-ONTAP-CI + network_type: Advanced + dns1: PLACEHOLDER-dns1 + dns2: PLACEHOLDER-dns2 + internal_dns1: PLACEHOLDER-internal-dns1 + internal_dns2: PLACEHOLDER-internal-dns2 + guest_cidr: 10.1.1.0/24 + guest_vlan_range: PLACEHOLDER-guest-vlan-range + pod_name: Pod1 + cluster_name: Cluster1 + primary_storage_url: PLACEHOLDER-nfs-primary-url + secondary_storage_url: PLACEHOLDER-nfs-secondary-url + systemvm_template_url: PLACEHOLDER-systemvm-template-url + template_name: PLACEHOLDER-kvm-template-name + + ontap: + storage_ip: PLACEHOLDER-ontap-data-lif + svm_name: PLACEHOLDER-ontap-svm-name diff --git a/private-cicd/docker/Dockerfile.driver b/private-cicd/docker/Dockerfile.driver new file mode 100644 index 000000000000..165701f5f07b --- /dev/null +++ b/private-cicd/docker/Dockerfile.driver @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG BASE_IMAGE=ubuntu:22.04 +FROM ${BASE_IMAGE} + +ARG NODE_VERSION=16.20.2 + +ENV DEBIAN_FRONTEND=noninteractive +ENV JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64" +# npm is a script with shebang `#!/usr/bin/env node`, so /opt/node/bin must be +# on PATH while the build verifies npm, not only in the finished image. +ENV PATH="/opt/cloudstack-driver/bin:/opt/node/bin:${PATH}" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository -y universe \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + bzip2 \ + build-essential \ + ca-certificates \ + coreutils \ + curl \ + debhelper \ + default-mysql-client \ + devscripts \ + dpkg-dev \ + fakeroot \ + file \ + findutils \ + gawk \ + genisoimage \ + git \ + gzip \ + iproute2 \ + iptables \ + libffi-dev \ + libvirt-clients \ + lsb-release \ + maven \ + nfs-common \ + openjdk-17-jdk \ + openssh-client \ + procps \ + python3 \ + python3-dev \ + python3-distutils \ + python3-mysql.connector \ + python3-netaddr \ + python3-pip \ + python3-setuptools \ + python3-venv \ + rsync \ + sshpass \ + tar \ + uuid-runtime \ + wget \ + xz-utils \ + && update-alternatives --set java /usr/lib/jvm/java-17-openjdk-amd64/bin/java \ + && update-alternatives --set javac /usr/lib/jvm/java-17-openjdk-amd64/bin/javac \ + && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + && mkdir -p /opt/node \ + && tar -xJf "node-v${NODE_VERSION}-linux-x64.tar.xz" -C /opt/node --strip-components=1 \ + && rm "node-v${NODE_VERSION}-linux-x64.tar.xz" \ + && python3 -m venv /opt/cloudstack-driver \ + && /opt/cloudstack-driver/bin/pip install --no-cache-dir --upgrade pip pyvmomi pyyaml \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && bash --version >/dev/null \ + && dpkg-buildpackage --version >/dev/null \ + && git --version >/dev/null \ + && java -version >/dev/null \ + && mvn --version >/dev/null \ + && /opt/node/bin/node --version | grep -q "^v${NODE_VERSION}$" \ + && /opt/node/bin/npm --version | grep -q '^8\.' \ + && tar --version >/dev/null \ + && command -v ssh >/dev/null \ + && command -v sshpass >/dev/null \ + && command -v ssh-keygen >/dev/null \ + && command -v scp >/dev/null \ + && command -v ssh-keyscan >/dev/null \ + && command -v gzip >/dev/null \ + && command -v gawk >/dev/null \ + && command -v mysql >/dev/null \ + && command -v virsh >/dev/null \ + && command -v dpkg-scanpackages >/dev/null \ + && /opt/cloudstack-driver/bin/python -c 'import pyVim, pyVmomi, yaml' \ + && command -v node >/dev/null \ + && command -v npm >/dev/null diff --git a/private-cicd/docs/CREATE-PRESUBMIT-JOB.md b/private-cicd/docs/CREATE-PRESUBMIT-JOB.md new file mode 100644 index 000000000000..18fc9053d38a --- /dev/null +++ b/private-cicd/docs/CREATE-PRESUBMIT-JOB.md @@ -0,0 +1,1346 @@ + + +# Create the auto-triggered ONTAP presubmit job + +This guide creates **one** Pipeline job, `cloudstack-ontap-presubmit`. Timer +builds poll GitHub every five minutes and self-queue one worker build for each +new eligible PR head SHA. Worker builds compile and package concurrently, then +wait for an available `cloudstack-presubmit-vm` Lockable Resource before +integration. Until `feature/CSTACKEX-223` is merged, the job loads trusted CI +from that branch. After merge, change only **Branches to build** to `*/main`. + +Do not create a GitHub repository webhook. OpenLab Jenkins is not reachable +from GitHub.com, so inbound `pull_request` deliveries fail with +`failed to connect to host`. That is expected. NetApp's +[CI/CD in Github](https://netapp.atlassian.net/wiki/spaces/NGAGE/pages/121930531/CI+CD+in+Github) +guide says to leave the GitHub App webhook inactive and to **poll** instead. +Do **not** create a Jenkins **GitHub Organization** / **Organization Folder** +item for this presubmit: that would run `Jenkinsfile` from the PR branch. This +design loads trusted CI from a fixed branch and checks the PR SHA out +separately. + +Do not create a separate poller, Multibranch Pipeline, or second production +copy of this job. + +This guide starts before the Jenkins item exists and ends after polling and +required Check have been verified. Follow sections 1 through 21 in order. Do +not skip a section merely because `cloudstack-presubmit-manual` already works; +each relevant section says whether to confirm existing controller state or +create new state. + +### Where to start if you already followed an earlier revision + +If the job, GitHub App credential, and parameterized smoke test already +work, do not recreate them. Start here: + +1. On GitHub, deactivate or delete any `NetApp/cloudstack` webhook whose payload + URL points at Jenkins. GitHub cannot deliver it. +2. On the GitHub App, keep **Webhook active** unchecked and add + **Pull requests: Read-only** if that permission is missing (section 9.1). + Also confirm **Checks: Read and write** is both requested *and* approved on + the installation; the NetApp guide omits it, so an App built from that guide + cannot publish the Check (section 9.0). +3. Push this branch so the polling-enabled `private-cicd/Jenkinsfile` is on + `origin/feature/CSTACKEX-223`. +4. Create the named Lockable Resource `cloudstack-presubmit-discovery` + (section 8). +5. **Build Now** once on `cloudstack-ontap-presubmit`. This first `discover` + build seeds the watermark and queues no existing PRs. +6. Push a new commit to the PR, then continue at section 16. + +The executable pipeline is [`../Jenkinsfile`](../Jenkinsfile). For pipeline +behavior and troubleshooting beyond job creation, see +[`PRIVATE-CICD-GUIDE.md`](PRIVATE-CICD-GUIDE.md). + +## Before you begin + +The manual job and this job load the same Jenkinsfile, use the same lab VM, and +use the same lab credentials. Reuse that controller state; do not recreate it. +You will still open each Jenkins screen in this guide and confirm the existing +value before continuing. + +Do not create duplicate credentials, a second Lockable Resource, or a second +inventory Secret file. Duplicated lab state is the most common cause of a +presubmit that hangs on the lock or configures the wrong VM. + +The following items should already exist from `cloudstack-presubmit-manual`: + +| Section | Existing state | What to do now | +| --- | --- | --- | +| 4.1 | Most plugins already installed | Confirm every plugin; install any missing plugin | +| 4.3 | Extended E-mail SMTP configured | Confirm sender still resolves | +| 4.4 | Kubernetes cloud proven by manual runs | Confirm the same cloud remains available | +| 5.1 | Driver image already pulled by the pod | Confirm the tag is unchanged | +| 5.2 | Clean VM snapshot exists | Confirm the last manual run left it clean | +| 5.3 | Inventory YAML populated | Confirm it has no placeholders; keep it outside Git | +| 6.1 | `cloudstack-presubmit-inventory` Secret file | Confirm the ID resolves | +| 6.2 | The 7 username/password credentials | Confirm the IDs resolve | +| 6.3 | Optional Git credential decision | Reuse the existing decision | +| 8 | VM Lockable Resources labeled `cloudstack-presubmit-vm` | Reuse them; add the discovery lock | +| 13 | Groovy signatures approved during manual runs | Approve only new pending ones | + +The 7 reused username/password credentials are `cloudstack-vcenter`, +`cloudstack-presubmit-ssh`, `cloudstack-mysql-root`, `cloudstack-db`, +`cloudstack-kvm-host`, `cloudstack-ontap`, and `cloudstack-admin`. + +In-process Script Approval is controller-wide, not per job. Signatures approved +while testing the manual job stay approved for this job, so section 13 is +normally a short confirmation rather than a full pass. + +If the manual job ran with a folder-scoped credential store, this job must live +in the same folder or a descendant of it. Otherwise the credential IDs will not +resolve and the build fails during **Validate source request**. + +The following work is new for the automatic job: + +| Section | New because | +| --- | --- | +| 4.1 | GitHub Branch Source is required for the GitHub App credential | +| 4.2 | Jenkins URL is still required for Check **Details** links, not for inbound GitHub webhooks | +| 7 | Self-queued workers need committed non-interactive defaults | +| 9 | The manual job never authenticated to GitHub; discovery needs Pull requests read | +| 10 | The manual job must stay disabled so it does not publish the same Check | +| 11, 12 | This is a new Pipeline item and needs a bootstrap/discovery run | +| 14 | The manual job only exercised `SOURCE_MODE=branch` | +| 15, 16 | The polling mode and its verification are new | +| 17 | The GitHub Check must be required on `main` to block merges | +| 20 | The trusted Pipeline SCM must move to `*/main` after merge | + +Terms used in the Jenkins UI: + +- **Jenkins controller:** the Jenkins web site where you create jobs and + credentials. +- **Pipeline job:** the `cloudstack-ontap-presubmit` page you create in + section 11. +- **Pipeline SCM:** the trusted branch from which Jenkins reads the Jenkinsfile; + it is not necessarily the PR branch being tested. +- **Credential ID:** a non-secret name Jenkins uses to locate a stored secret. + Enter the ID exactly; never enter the password itself in a Jenkinsfile field. +- **Discovery build:** a short `SOURCE_MODE=discover` run started by Jenkins + cron. It calls GitHub and self-queues worker builds; GitHub never connects to + Jenkins. + +If a named menu is missing, stop and ask the Jenkins administrator. Jenkins +permissions and plugin versions can hide or rename administration pages. + +## 1. Understand what will run + +The job has two different source locations: + +1. **Trusted pipeline source:** Jenkins loads `private-cicd/Jenkinsfile` and + its helper scripts from the job's SCM branch (`feature/CSTACKEX-223` now, + `main` after cutover in section 20). +2. **Source under test:** A self-queued worker fetches the exact PR head SHA. + +The PR cannot replace the Jenkinsfile used by this job. The Jenkinsfile checks +out the PR separately under `cloudstack-src/`, verifies that `HEAD` equals +`PR_HEAD_SHA`, and requires that SHA to be reachable from +`refs/pull//head`. + +Discovery queues a worker build when all of these are true: + +- the repository is `NetApp/cloudstack`; +- the PR is open, not a draft, and targets `main`; +- the PR was updated after the latest successful discovery watermark; +- the job has no queued or recorded worker for that PR number and head SHA; +- fewer than `MAX_ACTIVE_WORKERS` (default 5) `pull_request` workers are already + queued or running. + +Discovery queues oldest-updated eligible PRs first. Extra SHAs are deferred to +the next poll and the watermark is held so they are not skipped. + +Discovery sets `PR_ACTION` to `opened` for the first SHA of a PR and +`synchronize` for later SHAs. Worker eligibility still accepts +`reopened` and `ready_for_review` for manual parameterized runs. + +A push to a PR branch is visible to GitHub immediately. Jenkins notices it on +the next poll, typically within five minutes, not at the instant of `git push`. + +### Security boundary + +Only trusted maintainers may push to `feature/CSTACKEX-223` while it is the +job's SCM branch. That branch supplies executable Groovy and shell scripts to a +job that can access Jenkins credentials and a lab VM. After cutover, protect +`main` the same way and stop treating the feature branch as trusted CI. + +Discovery currently does not inspect whether the PR head repository is a fork. +Do not enable it for this lab unless PR authors and PR code are trusted to run +in the credentialed ONTAP lab. If fork PRs are accepted, add and validate a +head-repository restriction before using this design. + +## 2. Values to collect + +Record these values before opening Jenkins: + +| Value | Example or required form | +| --- | --- | +| Jenkins external URL | `https://jenkins.example.net/` | +| Jenkins job name | `cloudstack-ontap-presubmit` | +| Trusted pipeline branch | `feature/CSTACKEX-223` | +| Git repository | `https://github.com/NetApp/cloudstack.git` | +| Script Path | `private-cicd/Jenkinsfile` | +| Driver image | The immutable image in the Jenkinsfile pod template | +| vCenter host | Hostname only; no scheme or path | +| Inventory file | Populated copy of `vm-inventory.yaml.example` outside Git | +| Inventory VM | VM name, clean snapshot, SSH host/user, and lock resource | +| Test PR | Non-draft PR to `NetApp/cloudstack:main` | +| Test PR head SHA | Full 40-character SHA | + +You also need the credential IDs listed in section 6. + +## 3. Confirm the trusted branch + +Run these commands from the CloudStack repository: + +```bash +git fetch origin feature/CSTACKEX-223 +git status --short +git rev-parse HEAD +git rev-parse origin/feature/CSTACKEX-223 +``` + +Requirements: + +- the two SHAs are identical; +- all intended Jenkinsfile and script changes are committed and pushed; +- the working tree contains no change that the Jenkins job needs; +- only trusted maintainers can update this branch. + +In the GitHub UI, protect or restrict the branch while it is used as pipeline +source: + +1. Open `NetApp/cloudstack`. +2. Select **Settings**. +3. Open **Branches** or **Rules > Rulesets**, depending on the repository UI. +4. Create or verify a rule for `feature/CSTACKEX-223`. +5. Restrict pushes to the maintainers operating this test. +6. Do not allow force pushes while this branch is trusted CI. + +The job always reads the remote branch. Local uncommitted files are invisible +to Jenkins. + +## 4. Confirm Jenkins prerequisites + +You need Jenkins administrator help for controller, plugin, credential, and +script-approval operations. + +### 4.1 Required plugins + +In Jenkins: + +1. Select **Manage Jenkins**. +2. Select **Plugins**. +3. Open **Installed plugins**. +4. Search for and confirm each plugin: + +| Plugin | Pipeline use | +| --- | --- | +| Pipeline | Declarative pipeline execution | +| Pipeline Utility Steps | `readYaml` for VM inventory | +| Kubernetes | Jenkins agent pod | +| Git | trusted and PR source checkout | +| Lockable Resources | exclusive integration VM | +| Credentials | credential storage | +| Credentials Binding | temporary secret bindings | +| Email Extension | start and final email | +| GitHub Branch Source 2.7.1+ | GitHub App credential kind and token exchange | + +Generic Webhook Trigger is **not** required. Do not install it for this +presubmit. GitHub.com cannot POST to OpenLab Jenkins. + +If any plugin is missing: + +1. Open **Available plugins**. +2. Select the plugin. +3. Choose the site's approved installation option. +4. Restart Jenkins only when the controller requires it. +5. Return to **Installed plugins** and confirm the plugin is enabled. + +### 4.2 Jenkins URL + +The GitHub Check links back to `BUILD_URL`, so Jenkins needs a public HTTPS URL +that GitHub users can reach. + +1. Select **Manage Jenkins > System**. +2. Find **Jenkins Location**. +3. Set **Jenkins URL** to the externally reachable HTTPS root URL. +4. Ensure it ends in `/`. +5. Save. + +Do not use a pod-local hostname, private controller service name, or +`localhost`. + +The URL is used for GitHub Check **Details** links that people open in a +browser, often over VPN. GitHub.com never needs to connect to this host. Do +not create a repository webhook pointing at Jenkins; GitHub deliveries will +fail with `failed to connect to host`. + +### 4.3 Email + +The pipeline tolerates SMTP failure, but email should still be configured: + +1. Select **Manage Jenkins > System**. +2. Find **Extended E-mail Notification**. +3. Confirm the SMTP server, port, TLS mode, credential, and default sender. +4. Use **Test configuration by sending test e-mail**, if available. +5. Save. + +A missing PR email or SMTP failure does not replace the build result. + +### 4.4 Kubernetes agent + +Confirm the configured Jenkins Kubernetes cloud can: + +- start a pod in the intended namespace; +- pull the `jnlp` and `cloudstack-driver` images in the Jenkinsfile; +- provide the image pull secret, if required; +- reach GitHub, Maven/npm/package sources, vCenter, and the inventory VM; +- satisfy the pod CPU, memory, and ephemeral-storage requests. + +The current driver container requests 1 GiB memory and limits memory to 8 GiB. +The `jnlp` container is required for Jenkins agent attachment. + +## 5. Prepare the driver image and lab VM + +### 5.1 Driver image + +Open the Kubernetes pod YAML in +[`../Jenkinsfile`](../Jenkinsfile) and record the exact +`cloudstack-driver` image. + +From a machine with registry access: + +```bash +docker pull +docker image inspect --format '{{index .RepoDigests 0}}' +``` + +Use an immutable tag. If the Dockerfile changed, publish a new tag and update +the trusted branch before creating the job. Do not overwrite an existing +immutable tag because Kubernetes may retain an `IfNotPresent` cached image. + +### 5.2 Clean snapshot + +The inventory VM must satisfy the clean snapshot contract in the comprehensive +guide. At minimum, confirm: + +- Ubuntu 22.04; +- `/dev/kvm` exists; +- the configured bridge, normally `cloudbr0`, exists and is up; +- the management IP is configured; +- password SSH is available to the deployment user; +- DNS, apt repositories, ONTAP HTTPS, NFS, and template URLs are reachable; +- no previous CloudStack packages, databases, Marvin state, or credentials + remain. + +Take one clean vCenter snapshot and record its exact, case-sensitive VM and +snapshot names. + +### 5.3 Inventory file + +Create populated inventory outside Git: + +```bash +cp private-cicd/config/vm-inventory.yaml.example \ + /tmp/cloudstack-presubmit-vm-inventory.yaml +chmod 600 /tmp/cloudstack-presubmit-vm-inventory.yaml +``` + +Fill every `PLACEHOLDER-*` value. Do not put passwords, tokens, or private keys +in this file. Check it: + +```bash +python3 - <<'PY' +from pathlib import Path +import yaml + +path = Path("/tmp/cloudstack-presubmit-vm-inventory.yaml") +data = yaml.safe_load(path.read_text()) +assert data and data.get("vms"), "vms must be non-empty" +assert "PLACEHOLDER-" not in path.read_text(), "placeholder remains" +print("Inventory YAML parsed and has no placeholders.") +PY +``` + +If PyYAML is unavailable, use an approved YAML parser. Do not print the +populated file into a shared terminal or Jenkins log. + +For each enabled VM, the inventory `lock_resource` must exactly match a Jenkins +Lockable Resource name and the resource must carry label +`cloudstack-presubmit-vm`. + +## 6. Create Jenkins credentials + +Open the credential store: + +1. Select **Manage Jenkins**. +2. Select **Credentials**. +3. Select the appropriate store, normally **System**. +4. Select **Global credentials (unrestricted)**, or the approved domain. +5. Select **Add Credentials**. + +If the job is inside a Jenkins folder, folder-scoped credentials are safer. +They must be created in the same folder or an ancestor visible to the job. + +Create or confirm these credentials: + +| Suggested ID | Jenkins Kind | Value | +| --- | --- | --- | +| `cloudstack-presubmit-inventory` | Secret file | populated inventory YAML | +| `cloudstack-vcenter` | Username with password | vCenter account | +| `cloudstack-presubmit-ssh` | Username with password | initial VM login | +| `cloudstack-mysql-root` | Username with password | MySQL deployment administrator | +| `cloudstack-db` | Username with password | CloudStack database account | +| `cloudstack-kvm-host` | Username with password | KVM host account for `addHost` | +| `cloudstack-ontap` | Username with password | ONTAP SVM account | +| `cloudstack-admin` | Username with password | CloudStack API administrator | +| `cloudstack-presubmit-github-app` | GitHub App | App ID and PKCS#8 private key | +| site-defined ID | Username with password | optional read-only Git credential | + +Credential IDs are configuration names, not secret values. Password fields +must not be empty. The database passwords must work in the +`user:password@host` syntax accepted by `cloudstack-setup-databases`; avoid +unescaped `:`, `@`, and whitespace. + +### 6.1 Inventory Secret file + +1. Select **Add Credentials**. +2. Set **Kind** to **Secret file**. +3. Upload `/tmp/cloudstack-presubmit-vm-inventory.yaml`. +4. Set **ID** to `cloudstack-presubmit-inventory`, or your selected ID. +5. Add a description such as `ONTAP presubmit VM inventory`. +6. Select **Create**. +7. Delete the local temporary file after setup is verified. + +### 6.2 Username/password credentials + +Repeat these steps for vCenter, VM SSH, MySQL root, CloudStack DB, KVM host, +ONTAP, and CloudStack API credentials: + +1. Select **Add Credentials**. +2. Set **Kind** to **Username with password**. +3. Enter the service account username and password. +4. Enter the exact ID from the table. +5. Enter a description that names the service and lab. +6. Select **Create**. + +`VM_SSH_CREDENTIALS_ID` must point to Username with password, not an SSH key. +The pipeline uses that password once to install an ephemeral Ed25519 key. + +### 6.3 Optional Git credential + +Public HTTPS checkout normally needs no credential. If policy requires one: + +1. Create a least-privilege read-only GitHub credential. +2. Record its Jenkins ID. +3. Use the same credential in Pipeline SCM and `GIT_CREDENTIALS_ID`. + +Do not use a personal credential with repository write permission. + +## 7. Set non-interactive pipeline defaults + +Self-queued worker builds do not display **Build with Parameters**. Discovery +contributes only the PR fields. All other values come from the Jenkinsfile +parameter defaults. + +This change is already made in the Jenkinsfile on +`feature/CSTACKEX-223`. Before creating the job, open +[`../Jenkinsfile`](../Jenkinsfile), find the `parameters` block, and confirm +these exact non-secret defaults: + +```text +VM_INVENTORY_CREDENTIALS_ID=cloudstack-presubmit-inventory +VCENTER_HOST=cstack-netapp-lab.rtp.openenglab.netapp.com +VCENTER_CREDENTIALS_ID=cloudstack-vcenter +VM_SSH_CREDENTIALS_ID=cloudstack-presubmit-ssh +MYSQL_ROOT_CREDENTIALS_ID=cloudstack-mysql-root +CLOUD_DB_CREDENTIALS_ID=cloudstack-db +KVM_HOST_CREDENTIALS_ID=cloudstack-kvm-host +ONTAP_CREDENTIALS_ID=cloudstack-ontap +CLOUDSTACK_ADMIN_CREDENTIALS_ID=cloudstack-admin +GITHUB_APP_CREDENTIALS_ID=cloudstack-presubmit-github-app +MAX_ACTIVE_WORKERS=5 +``` + +These are credential lookup names, not usernames or passwords. The vCenter +hostname is configuration, not a credential. + +From the repository root, verify no parameter default still contains a +placeholder: + +```bash +git grep -n "defaultValue: 'PLACEHOLDER-" -- private-cicd/Jenkinsfile +``` + +The command must print nothing and exit with status 1, meaning no match. + +Review the change: + +```bash +git diff -- private-cicd/Jenkinsfile +``` + +Commit and push this Jenkinsfile change together with the rest of the +presubmit implementation on `feature/CSTACKEX-223` before creating the Jenkins +job. Do not commit usernames, passwords, private keys, or populated inventory. +Re-run section 3 afterward and require the local and remote SHAs to match. + +Do not rely on editing generated parameter defaults in the Jenkins job UI. +Declarative Pipeline properties can replace those values from the Jenkinsfile +on a later run. + +## 8. Create the Lockable Resources + +For each enabled inventory VM: + +1. Select **Manage Jenkins**. +2. Open **System** or **Configure System**, depending on Jenkins version. +3. Find **Lockable Resources Manager**. +4. Select **Add Lockable Resource**. +5. Set **Name** to the exact inventory `lock_resource`. +6. Set **Labels** to `cloudstack-presubmit-vm`. +7. Leave **Reserved by** empty. +8. Add a description naming the vCenter VM and snapshot. +9. Save. + +Open the Lockable Resources page and confirm the resource is free. A spelling +or case mismatch prevents the pipeline from mapping the lock back to inventory. +Only enabled inventory VMs may carry label `cloudstack-presubmit-vm`. The +pipeline locks a resource by label and validates its name afterward, so a stale +or unrelated resource with this label can be selected and then rejected. + +Create one additional resource that serializes only the short discovery runs: + +1. Select **Add Lockable Resource**. +2. Set **Name** to `cloudstack-presubmit-discovery`. +3. Leave **Labels** and **Reserved by** empty. +4. Add description `Serializes CloudStack GitHub PR discovery`. +5. Save and confirm the resource is free. + +Do not use `cloudstack-presubmit-vm` for discovery. Worker builds remain +concurrent through compilation and queue on that label only at integration. + +A discovery run that finds the lock held skips its work and finishes as +`NOT_BUILT` instead of waiting, so a slow poll cannot build a backlog of +discovery runs. The skipped run leaves the watermark alone, so the next run +still queues everything that changed. + +## 9. Create and install the GitHub App + +The App publishes the `cloudstack-ontap-presubmit` Check, looks up a commit +author email, and lets discovery list open pull requests. GitHub never sends +events to Jenkins. + +### 9.0 Where this differs from the NetApp CI/CD in Github guide + +NetApp's [CI/CD in Github](https://netapp.atlassian.net/wiki/spaces/NGAGE/pages/121930531/CI+CD+in+Github) +guide grants **Commit statuses: Read and write** and never mentions Checks. +That is correct for the standard GitHub Branch Source integration, which +publishes commit statuses. This presubmit publishes a Check Run instead, so it +additionally requires **Checks: Read and write**. + +An App created strictly from that guide therefore returns HTTP 403 from +`POST /repos/NetApp/cloudstack/check-runs` while discovery, pull-request +listing, and commit-email lookup all keep working, because those need only the +read permissions the guide does grant. A 403 on the write with successful reads +is the signature of this gap. + +Two further steps in that guide do not apply here: + +| Guide step | This presubmit | +| --- | --- | +| Subscribe to all events | Subscribe to no events and leave the webhook inactive | +| Create a Jenkins GitHub Organization item | Create one Pipeline job; an Organization folder would run `Jenkinsfile` from the PR branch | + +Adding a permission to an App that is **already installed** only requests it. +An organization owner must then approve the updated permission on the +`NetApp/cloudstack` installation before GitHub issues tokens that carry it. +Until that approval the App settings page shows Checks read and write while +every Check write still returns 403. Installing a newly created App grants +everything it requests, so a fresh App needs no separate approval step. + +### 9.1 Create the App + +Use an organization-owned App: + +1. Sign in to GitHub with permission to manage Apps for the NetApp + organization. +2. Open the organization **Settings**. +3. Open **Developer settings > GitHub Apps**. If the organization UI links to a + personal developer-settings page, confirm the owner is the organization. +4. Select **New GitHub App**. +5. Enter a unique name, such as `Jenkins - CloudStack ONTAP Presubmit`. +6. Set **Homepage URL** to the Jenkins external URL or repository URL. +7. Uncheck **Webhook active**. Leave **Webhook URL** empty and do not subscribe + to events. OpenLab Jenkins cannot receive GitHub deliveries. +8. Under **Repository permissions**, set: + - **Checks:** Read and write; + - **Contents:** Read-only; + - **Metadata:** Read-only; + - **Pull requests:** Read-only. +9. Leave every other repository, organization, and account permission at + **No access**. +10. Under event subscriptions, select no events. +11. Limit installation to the owning organization. +12. Select **Create GitHub App**. +13. Record the numeric **App ID**. Do not use Client ID. + +### 9.2 Install the App + +1. On the App settings page, select **Install App**. +2. Select the NetApp organization. +3. Choose **Only select repositories**. +4. Select only `cloudstack`. +5. Complete installation. + +If the App is not installed on `NetApp/cloudstack`, Check API calls normally +return HTTP 404 even when the App ID and key are valid. + +### 9.3 Generate and convert the private key + +1. Return to the App settings page. +2. Under **Private keys**, select **Generate a private key**. +3. Save the downloaded PEM file to a protected local directory. +4. Convert it to unencrypted PKCS#8: + +```bash +umask 077 +openssl pkcs8 -topk8 -inform PEM -outform PEM \ + -in .private-key.pem \ + -out converted-github-app.pem \ + -nocrypt +head -1 converted-github-app.pem +``` + +The first line must be the unencrypted PKCS#8 PEM header +(five hyphens, BEGIN, a space, PRIVATE KEY, five hyphens). + +A PKCS#1 RSA PEM header is not the converted PKCS#8 form expected by the +Jenkins GitHub App credential. + +### 9.4 Add the App credential to Jenkins + +1. Open the Jenkins credential store used by the job. +2. Select **Add Credentials**. +3. Set **Kind** to **GitHub App**. +4. Set **ID** to `cloudstack-presubmit-github-app`. +5. Enter the numeric **App ID**. +6. Set **Owner** to `NetApp` if the credential form provides that field. This + selects the organization installation used for token exchange. +7. Paste or upload `converted-github-app.pem` in the private-key field. +8. Add a description naming the repository and this Jenkins job. +9. Select **Create**. +10. Use **Test Connection** if the plugin provides it. +11. Securely delete both local private-key files after Jenkins stores the key. + +If **GitHub App** is not available under **Kind**, update or enable GitHub +Branch Source before continuing. + +## 10. Disable competing jobs and inbound webhooks + +Do not create `cloudstack-presubmit-webhook-token`. The production Jenkinsfile no +longer declares Generic Webhook Trigger. + +### Delete any GitHub webhook aimed at Jenkins + +If you already created a repository webhook while following an earlier revision: + +1. Open `https://github.com/NetApp/cloudstack/settings/hooks`. +2. Open the webhook whose payload URL contains Jenkins or + `generic-webhook-trigger`. +3. Select **Delete**, or clear **Active** and save. +4. `failed to connect to host` on Recent Deliveries is expected and is not a + Jenkins misconfiguration. + +### Disable `cloudstack-presubmit-manual` + +Keep the manual job disabled so it cannot publish Check +`cloudstack-ontap-presubmit` on the same SHA as a production worker. + +1. Open `cloudstack-presubmit-manual`. +2. Select **Disable Project**, or tick **Disable this project** in + **Configure** and save. +3. Confirm the job page shows the disabled banner. + +To run a branch-mode test later, enable it for that run only, or use +`SOURCE_MODE=branch` on `cloudstack-ontap-presubmit` through **Build with +Parameters**. That path needs no second job. + +Do not select **GitHub Organization** or **Organization Folder** as a +substitute trigger. The Confluence page uses that item type for repos with a +root `Jenkinsfile`. This presubmit is `private-cicd/Jenkinsfile` and must not +execute untrusted PR Groovy. + +## 11. Create the Jenkins Pipeline item + +Create a regular Pipeline job: + +1. Return to the Jenkins dashboard. +2. Select **New Item**. +3. Enter `cloudstack-ontap-presubmit`. +4. Select **Pipeline**. +5. Select **OK**. + +Do not select Freestyle, Multi-configuration, Multibranch Pipeline, or +Organization Folder. + +### 11.1 General + +In **General**: + +1. Add a description: + + ```text + Loads trusted CI from feature/CSTACKEX-223 until that branch is merged to + main, then change Branches to build to */main. Tests exact + NetApp/cloudstack PR head SHAs. + ``` + +2. Keep **This project is parameterized** as generated by the Jenkinsfile; do + not manually create duplicate parameters before the first load. +3. Keep **Disable concurrent builds** unchecked. +4. If the UI offers **Abort previous builds**, keep it disabled. + +The pipeline implements source-aware cancellation itself. Job-level +abort-previous would incorrectly abort unrelated PRs. + +### 11.2 Build Triggers + +Do not configure triggers manually: + +- do not select **Poll SCM**; +- do not select **GitHub hook trigger for GITScm polling**; +- do not add a Generic Webhook Trigger. + +After the first load, the Jenkinsfile applies **Build periodically** with +`H/5 * * * *`. Do not add a second schedule in the UI. + +### 11.3 Pipeline + +In the **Pipeline** section: + +1. Set **Definition** to **Pipeline script from SCM**. +2. Set **SCM** to **Git**. +3. Set **Repository URL** to: + + ```text + https://github.com/NetApp/cloudstack.git + ``` + +4. Set **Credentials** to `- none -` for public HTTPS, or select the approved + read-only Git credential. +5. Under **Branches to build > Branch Specifier**, enter: + + ```text + */feature/CSTACKEX-223 + ``` + +6. Set **Script Path** to: + + ```text + private-cicd/Jenkinsfile + ``` + +7. Uncheck **Lightweight checkout**. +8. Save. + +Do not enter the PR branch in Branch Specifier. This field chooses the trusted +pipeline definition. Discovery parameters choose the source under test. + +## 12. Bootstrap the job + +Jenkins applies Declarative Pipeline parameters and the cron after it loads the +Jenkinsfile at least once. + +1. Open `cloudstack-ontap-presubmit`. +2. Select **Build Now**. +3. Open the new build. +4. Select **Console Output**. + +The first run may use old parameter defaults and fail validation. If it starts +with `SOURCE_MODE=discover`, it calls GitHub using the current build start as +its initial watermark, queues no older PR revisions, and finishes `NOT_BUILT`. +Both outcomes are safe. Its purpose is to load: + +- the Declarative Pipeline parameter definitions; +- the five-minute timer; +- the build retention and timeout settings. + +After the run: + +1. Return to the job page. +2. Confirm **Build with Parameters** is available. +3. Select **Configure**. +4. Confirm the generated parameters appear once. +5. Confirm `SOURCE_MODE` choices are `discover`, `pull_request`, `branch`. +6. Under **Build Triggers**, confirm **Build periodically** uses + `H/5 * * * *` and Generic Webhook Trigger is absent. +7. Run **Build with Parameters**, select `SOURCE_MODE=discover`, and confirm + the build title reads `discover: queued 0, deferred 0` and the description + starts `seeded at` if no prior successful discovery exists. + +If the job still shows Generic Webhook Trigger, the SCM branch still has the +old Jenkinsfile. Confirm `origin/feature/CSTACKEX-223` contains a Jenkinsfile +with no `triggers { GenericTrigger(...) }` block, then **Build Now** again. + +## 13. Approve only required Groovy signatures + +Discovery and **Abort superseded run** inspect this job's queue and older +builds through Jenkins internal APIs. Script Security may stop the first runs. + +Approvals are stored per controller, not per job. Signatures already approved +while running `cloudstack-presubmit-manual` remain approved here, so expect few +or no pending entries. Approve anything that does appear rather than assuming +the list is complete. + +When Jenkins reports `Scripts not permitted to use method ...`: + +1. Copy the exact rejected signature from Console Output. +2. Select **Manage Jenkins**. +3. Select **In-process Script Approval**. +4. Find the exact pending signature from this trusted + `feature/CSTACKEX-223` Jenkinsfile. +5. Review what it allows. +6. Select the normal **Approve** action. +7. Do not select **Approve assuming permission check**. +8. Re-run the parameterized smoke test. +9. Repeat only for signatures reported by this pipeline. + +Expected operations include obtaining this Jenkins job, enumerating its builds, +reading build parameters/environment, checking whether a build is running, and +stopping an older matching build. Discovery also reads the job's configured Git +branch and remote so it can clone one commit of `private-cicd/scripts` instead +of the whole repository; until those two signatures are approved it logs +`Shallow discovery checkout unavailable` and falls back to a full checkout that +takes minutes. Exact signatures vary by Jenkins and plugin version, so do not +approve a copied generic list. + +Never approve unrelated file, process, credential, network, reflection, or +Jenkins-administration access. + +## 14. Run a parameterized smoke test before enabling discovery + +Use an existing non-draft PR targeting `main`. In GitHub: + +1. Open the PR. +2. Copy its number from the URL. +3. Copy the head branch name. +4. Open the latest commit and copy its full 40-character SHA. +5. Copy the PR URL, title, and author login. + +In Jenkins: + +1. Open `cloudstack-ontap-presubmit`. +2. Select **Build with Parameters**. +3. Enter: + +| Parameter | Value | +| --- | --- | +| `SOURCE_MODE` | `pull_request` | +| `SOURCE_BRANCH` | blank | +| `SOURCE_SHA` | blank | +| `PR_ID` | PR number | +| `PR_ACTION` | `opened` | +| `PR_DRAFT` | `false` | +| `PR_REPOSITORY` | `NetApp/cloudstack` | +| `PR_BASE_BRANCH` | `main` | +| `PR_HEAD_BRANCH` | PR head branch | +| `PR_HEAD_SHA` | full 40-character head SHA | +| `PR_AUTHOR_LOGIN` | PR author login | +| `PR_AUTHOR_EMAIL` | author email or blank | +| `PR_URL` | full PR URL | +| `PR_TITLE` | PR title | +| `EXPECTED_REPOSITORY` | `NetApp/cloudstack` | +| `CLOUDSTACK_GIT_URL` | `https://github.com/NetApp/cloudstack.git` | +| `GIT_CREDENTIALS_ID` | blank or approved read-only ID | +| `GITHUB_APP_CREDENTIALS_ID` | `cloudstack-presubmit-github-app` | +| inventory/vCenter/runtime fields | exact section 6 IDs and vCenter host | +| `PAUSE_BETWEEN_STAGES` | `true` for the first observed run | + +4. Select **Build**. +5. Open **Console Output**. +6. At each pause, inspect the completed stage, then select **Proceed**. + +Expected stage order: + +1. Validate source request +2. Check eligibility +3. Abort superseded run +4. Checkout CI scripts +5. Start PR reporting +6. Validate builder +7. Checkout source +8. Build and unit tests +9. Build Debian packages +10. Deploy and run ONTAP integration + +Confirm: + +- **Checkout CI scripts** shows `feature/CSTACKEX-223`; +- `presubmit-results/source.properties` records the test PR and exact SHA; +- the source checkout `HEAD` equals `PR_HEAD_SHA`; +- Maven tests and required Debian packages pass; +- the selected inventory VM is locked through result retrieval; +- the exact snapshot is reverted; +- `setup_zone`, `iscsi`, and `nfs3` run in that order; +- the PR commit receives Check `cloudstack-ontap-presubmit`; +- start and final mail are attempted; +- the start mail shows a `STARTED` banner and a bordered table of source, title, + diff number, SHA, and start time; +- both subjects end with `diff # ()`, where the diff number is `1` for + the first commit presubmitted on that PR and increments on each new push, so a + second push reports `diff #2` while a re-run of the same commit stays `diff #1`; +- the final mail contains the diff number, stage durations, and an ONTAP test + table with failures first; +- `presubmit-results/report.html` and each test's linked HTML log open from + Jenkins artifacts; +- archived artifacts contain no inventory, `ontap.cfg`, `secrets.json`, + passwords, private keys, or webhook tokens. + +For normal automatic runs, `PAUSE_BETWEEN_STAGES` must be `false`. A +self-queued worker cannot click **Proceed**, and a pause after snapshot revert +holds the VM lock. + +You may add Check `cloudstack-ontap-presubmit` as a required status on `main` +after a successful smoke test. This is the same job that will load CI from +`main` after section 20, so you do not wait for a second Jenkins item. + +## 15. Enable polling on the existing job + +No second Jenkins item is needed. The same `cloudstack-ontap-presubmit` job +uses two kinds of builds: + +- short `discover` builds started every five minutes; +- `pull_request` worker builds self-queued by discovery. + +Concurrent builds must stay enabled. The named +`cloudstack-presubmit-discovery` lock serializes only discovery; each worker +later queues at the existing `cloudstack-presubmit-vm` lock. + +1. Open `cloudstack-ontap-presubmit`. +2. Select **Configure**. +3. Confirm **Disable concurrent builds** remains unchecked. +4. Confirm **Abort previous builds** remains disabled. +5. Under **Build Triggers**, confirm **Build periodically** is present with + `H/5 * * * *`. +6. Confirm **Poll SCM**, **GitHub hook trigger**, and Generic Webhook Trigger + are absent. +7. Save only if Jenkins requires it. +8. Select **Build with Parameters**, choose `SOURCE_MODE=discover`, and build. +9. Confirm the first successful discovery is titled `discover: queued 0, + deferred 0` with description `seeded at ...`. This prevents historical open + PRs from being started. + +A successful discovery build finishes as `NOT_BUILT` by design. That status is +its watermark marker and prevents worker stages, mail, artifacts, and GitHub +Checks from running. A discovery build that fails reports `FAILURE`, so a red +discovery build always means discovery itself is broken. + +Build titles distinguish the three kinds of run in the history: + +| Title | Meaning | +| --- | --- | +| `#N discover: queued 1, deferred 0` | a poll that queued one worker | +| `#N discover: skipped, lock held` | a poll that overlapped a slower one | +| `#N PR-98 diff #3 synchronize` | the presubmit for PR-98's third commit | + +## 16. Verify automatic polling + +### 16.1 Confirm discovery can call GitHub + +Open the latest discovery **Console Output**. Confirm it reached +`api.github.com` and wrote `eligible-prs.json`. A 401/403/404 here is an App +credential or **Pull requests** permission problem, not a webhook problem. + +### 16.2 Push a new PR revision + +Use the test PR from section 14. Push an intended small change to its branch. +An empty commit is enough to change the head SHA: + +```bash +git checkout +git pull --ff-only +git commit --allow-empty -m "Test ONTAP presubmit discovery" +git push origin HEAD +``` + +Wait up to five minutes, or run `SOURCE_MODE=discover` manually. + +### 16.3 Confirm discovery queued a worker + +The discovery console should contain a line similar to: + +```text +Queueing PR- synchronize at . +``` + +A later discovery run for the same SHA should say the presubmit is already +queued or recorded. + +### 16.4 Confirm Jenkins parameters and checkout + +Open the new worker build in the same job. Confirm in **Parameters** or +Console Output: + +```text +SOURCE_MODE=pull_request +PR_ACTION=synchronize +PR_REPOSITORY=NetApp/cloudstack +PR_BASE_BRANCH=main +PR_DRAFT=false +PR_HEAD_SHA= +``` + +Confirm **Checkout source** checks out that exact SHA, not the moving branch +name or merge commit. + +### 16.5 Confirm same-PR cancellation + +While one revision of the test PR is still in Maven or packaging, push another +revision and wait for the next poll. The newer build should abort the older +build because both have source key `pull_request:`. + +Repeat only if needed after the older run acquires the VM. Once +`PRESUBMIT_INTEGRATION_STARTED=true`, the newer run must not abort it. It waits +for a compatible VM while the older integration completes. + +Different PR IDs must not abort each other. + +### 16.6 Confirm the five-worker cap + +This is optional if you have only one test PR. When several eligible SHAs exist: + +1. Confirm five `pull_request` workers are already queued or running, or queue + that many with **Build with Parameters**. +2. Push two more eligible SHAs. +3. Run `SOURCE_MODE=discover`. +4. Confirm the console reports two `Deferred PR-... at cap 5` lines and the + description includes `deferred 2`. +5. Confirm the next discovery watermark in the description is unchanged from + this run. +6. After a worker finishes, run discover again and confirm the deferred SHAs + are queued. +7. Confirm extra workers wait on `cloudstack-presubmit-vm` rather than failing. + +### 16.7 Confirm GitHub Check + +On the new PR commit: + +1. Open the PR **Checks** area or the commit status. +2. Find `cloudstack-ontap-presubmit`. +3. Confirm it starts **In progress**. +4. Confirm **Details** opens this Jenkins build. +5. After completion, confirm conclusion is **Success**, **Failure**, or + **Cancelled** as expected. + +You may require this Check on `main` after the smoke test. + +## 17. Make the GitHub Check required on `main` + +Do this only after section 16 has created at least one Check named +`cloudstack-ontap-presubmit`. GitHub may not offer a Check name until it has +been reported recently. + +Changing branch protection requires repository administrator permission. If +you cannot see **Settings**, or cannot edit branch rules, send this section to +a `NetApp/cloudstack` repository administrator. Do not weaken or replace an +existing rule to gain access. + +### 17.1 Open the rule for `main` + +On GitHub: + +1. Open `https://github.com/NetApp/cloudstack`. +2. Select **Settings**. This is the repository's Settings tab, not your + personal settings and not the GitHub App settings. +3. In the left sidebar, select **Branches**. +4. Under **Branch protection rules**, find the rule whose branch name pattern + covers `main`. +5. Select **Edit**. + +If the repository uses rulesets instead of the Branches page: + +1. Select **Settings > Rules > Rulesets**. +2. Open the active ruleset whose target branches include `main`. +3. Select **Edit**. +4. Find the rule that requires status checks before merging. + +Ask the repository administrator which existing rule owns `main` if both +systems are present. Do not create a second overlapping rule without their +approval. + +### 17.2 Require the presubmit Check + +In the existing rule: + +1. Enable **Require status checks to pass before merging**. +2. Select **Add checks**, or use the status-check search field. +3. Search for this exact, case-sensitive name: + + ```text + cloudstack-ontap-presubmit + ``` + +4. Select only that Check name for this presubmit. If GitHub displays the + source App, select the result produced by the new GitHub App. +5. Leave all existing required checks selected. +6. Leave other review, signed-commit, force-push, deletion, and administrator + settings unchanged unless the repository administrator explicitly asks for + a change. +7. Select **Save changes**. +8. Complete any organization confirmation or approval prompt. + +The required Check applies to every PR targeting `main`, including the first PR +from `feature/CSTACKEX-223` and later PRs from other branches. It does not +filter on the source branch. + +### 17.3 Verify merge blocking + +Return to the test PR: + +1. Confirm `cloudstack-ontap-presubmit` appears in the required checks list. +2. While the Check is queued or running, confirm GitHub says merging is + blocked. +3. If the Check fails, confirm merging remains blocked. +4. After a successful Check for the current head SHA, confirm this requirement + is satisfied. +5. Push a harmless new commit during the controlled smoke test. +6. Confirm the prior success no longer satisfies the PR and the Check becomes + pending for the new SHA. + +Do not merge solely because a Jenkins email says success. The GitHub Check on +the current PR head SHA is the merge gate. + +## 18. Expected trigger cases + +| Change | Expected result | +| --- | --- | +| Non-draft PR to `main` updated after the watermark | discovery queues a worker | +| New commit pushed to an eligible PR | discovery queues `synchronize` for the new SHA | +| Same SHA already queued or recorded | discovery skips it | +| Draft PR | no queue | +| PR targeting a branch other than `main` | no queue | +| PR closed | no queue | +| Open PR last updated before initial activation | never queued by the initial seed | +| More than five eligible SHAs while five workers are already active | discovery queues none extra and holds the watermark | +| Push with no open PR to `main` | no queue | +| GitHub repository webhook ping | ignored; do not create that webhook | + +The Jenkins **Check eligibility** stage repeats the action, draft, and base +checks. A manually parameterized ineligible request becomes `NOT_BUILT`. + +## 19. Troubleshooting + +### Discovery does not queue a new push + +- Confirm the job is enabled and a `discover` build ran after the push. +- Confirm the PR is not a draft and targets `main`. +- Confirm the job has no existing worker for that `PR_ID` and `PR_HEAD_SHA`. +- Confirm GitHub App **Pull requests** is Read-only and the App is installed + on `cloudstack`. +- Confirm the latest successful discovery description has an earlier watermark + than the PR's `updated_at`. +- If the console says `Deferred PR-... at cap`, wait for an active worker to + finish and for the next discovery run. Manual `Build with Parameters` with + `SOURCE_MODE=pull_request` bypasses the cap. + +### Discovery builds pile up behind the discovery lock + +A queue of `discover` builds all reporting `The resource +[cloudstack-presubmit-discovery] is locked by build ...` means one discovery run +is taking longer than the five-minute timer. Cancel the queued discovery builds +from the job page, then confirm the running one finishes in seconds: + +- Confirm the loaded Jenkinsfile passes `skipIfLocked: true` to the discovery + lock, so a run that finds the lock held finishes as `NOT_BUILT`. +- Confirm the console shows a shallow clone of `private-cicd/scripts` and not a + full CloudStack checkout. `Shallow discovery checkout unavailable` means the + two Git signatures in section 13 still need approval. + +### `failed to connect to host` on a GitHub webhook + +That webhook cannot work against OpenLab Jenkins. Delete or deactivate it. +Use `SOURCE_MODE=discover`. Do not ask GitHub to redeliver. + +### Duplicate builds + +- Confirm `cloudstack-presubmit-discovery` exists so discovery runs serialize. +- Confirm `cloudstack-presubmit-manual` is disabled. +- Do not click **Build with Parameters** for a SHA discovery already queued. + +### Build fails on `PLACEHOLDER-*` + +- Self-queued workers use Jenkinsfile defaults for lab parameter values. +- Replace placeholder host/credential IDs on the fixed trusted branch. +- Push the branch and bootstrap the job again. +- Confirm the credential IDs exist in the job's scope. + +### Pipeline cannot load + +- Confirm SCM branch is exactly `*/feature/CSTACKEX-223` before cutover, or + `*/main` after section 20. +- Confirm Script Path is exactly `private-cicd/Jenkinsfile`. +- Confirm the remote branch contains that file. +- Check Git credential and network access. +- Keep Lightweight Checkout off. + +### Script approval failure + +- Approve only the exact pending signature produced by this trusted job. +- Re-run after each approved signature. +- Do not approve broad or unrelated signatures. + +### Pod never starts + +- Inspect the Kubernetes pod event. +- `ErrImagePull` or `ImagePullBackOff`: verify image, tag, registry credential, + and namespace pull secret. +- `FailedScheduling`: verify CPU, memory, and ephemeral-storage capacity. +- No Jenkins agent connection: verify the `jnlp` container and controller + connectivity. + +### GitHub Check API failure + +| Response | Likely cause | +| --- | --- | +| 401 | invalid App ID/key or installation token | +| 403 | missing Checks write permission, policy, or rate limit | +| 404 | App not installed on `NetApp/cloudstack` or wrong repository | + +Also confirm the private key is unencrypted PKCS#8 PEM (not PKCS#1 RSA), the +credential Kind is GitHub App, and the ID matches `GITHUB_APP_CREDENTIALS_ID`. + +GitHub reporting errors do not replace the build result, but a missing success +cannot satisfy a required Check. + +### Check has the wrong Details link + +Set **Manage Jenkins > System > Jenkins Location > Jenkins URL** to the external +HTTPS URL, then start a new build. Existing Check links are not rewritten. + +### Build waits for VM + +- Open Lockable Resources. +- Confirm at least one enabled inventory VM has label + `cloudstack-presubmit-vm`. +- Confirm resource name exactly equals inventory `lock_resource`. +- Remove this label from stale or unrelated resources that are absent from the + inventory. +- Check whether an older integration build owns the VM. +- Never manually unlock a resource while its build is active. + +### ONTAP integration failed but the console ends with later passing tests + +The pipeline deliberately runs NFS3 after an iSCSI failure so both protocol +results are available. The final stage remains failed if either protocol +failed. + +1. Open the final mail and find the **ONTAP tests** table. +2. Failed and exception rows are listed first. +3. Select **HTML log** on the failing test to jump directly to that test in + the archived suite log. +4. Alternatively, open Jenkins **Artifacts > + presubmit-results/report.html**. + +Do not infer that the run passed because the last NFS3 summary is green. Do not +debug an ONTAP test failure from `maven.log`, `configure-cloudstack.log`, or +`health-check.log` when their stage rows are green. + +### Two Check results conflict + +Disable any older job that still publishes `cloudstack-ontap-presubmit`. The +worker builds in this job are the only intended publisher of that Check name. + +## 20. Cutover to `main` + +Do this after `feature/CSTACKEX-223` is merged to `main`. Do **not** create a +second job. Do **not** delete this job, the GitHub App, credentials, or +Lockable Resources. + +1. Confirm `origin/main` contains `private-cicd/Jenkinsfile` at the merged + revision: + + ```bash + git fetch origin main + git show origin/main:private-cicd/Jenkinsfile | head + ``` + +2. In Jenkins, open `cloudstack-ontap-presubmit` → **Configure**. +3. Under **Pipeline > Branches to build > Branch Specifier**, replace + `*/feature/CSTACKEX-223` with: + + ```text + */main + ``` + +4. Leave Script Path `private-cicd/Jenkinsfile`, Lightweight checkout off, and + concurrent builds enabled. +5. Save. +6. Run **Build Now** once so Jenkins reloads the Jenkinsfile from `main`. +7. Confirm the timer remains `H/5 * * * *` and Generic Webhook Trigger is absent. +8. Update the job description so it no longer says the trusted branch is + `feature/CSTACKEX-223`. +9. Restrict or delete `feature/CSTACKEX-223` so it is no longer a trusted CI + source. + +Leave the Check name and branch-protection rule unchanged. Later PRs from any +feature branch into `main` use the same job. + +If you ever retire the presubmit entirely, disable the timer, let any in-flight +integration finish, then disable the job. + +## 21. Completion checklist + +- [ ] Trusted CI branch is pushed, restricted to maintainers, and used as SCM + until merge. +- [ ] PR code is trusted for the credentialed integration environment. +- [ ] Required plugins are installed and enabled. +- [ ] Jenkins external URL and SMTP are configured. +- [ ] Kubernetes can pull both pod images and reach dependencies. +- [ ] Driver image uses an immutable, pullable tag. +- [ ] Clean VM snapshot satisfies the baseline contract. +- [ ] Populated inventory remains outside Git and has no placeholders. +- [ ] Every enabled VM maps to a free labeled Lockable Resource. +- [ ] All credentials exist with the correct Kind and ID. +- [ ] Jenkinsfile defaults contain usable host and credential IDs. +- [ ] GitHub App is installed only on `NetApp/cloudstack`. +- [ ] App key is PKCS#8 and local copies have been deleted. +- [ ] Pipeline job name is `cloudstack-ontap-presubmit`. +- [ ] Before merge, SCM branch is `*/feature/CSTACKEX-223`. +- [ ] Script Path is `private-cicd/Jenkinsfile`. +- [ ] Lightweight Checkout is off and concurrent builds are enabled. +- [ ] Timer is `H/5 * * * *`; no competing trigger is enabled. +- [ ] `MAX_ACTIVE_WORKERS` defaults to `5`. +- [ ] `cloudstack-presubmit-discovery` exists and is free. +- [ ] Bootstrap registered `discover`, `pull_request`, and `branch` parameters. +- [ ] Only required Groovy signatures were approved. +- [ ] Parameterized smoke test passed with exact-SHA checkout. +- [ ] `PAUSE_BETWEEN_STAGES` is off for automatic worker runs. +- [ ] No old job publishes the same Check name. +- [ ] `cloudstack-presubmit-manual` is disabled. +- [ ] No GitHub repository webhook points at Jenkins. +- [ ] Initial discovery seeded without queueing historical PRs. +- [ ] Discovery queued one worker for a new PR SHA within five minutes. +- [ ] Jenkins parameters and source checkout match the new PR SHA. +- [ ] GitHub Check links to and concludes with the correct build. +- [ ] Same-PR cancellation and post-lock protection behave as designed. +- [ ] Archives contain no secrets or generated runtime configuration. +- [ ] After merge, SCM branch is `*/main` and the feature branch is no longer + trusted CI. diff --git a/private-cicd/docs/PRIVATE-CICD-GUIDE.md b/private-cicd/docs/PRIVATE-CICD-GUIDE.md new file mode 100644 index 000000000000..54d4dae56a6f --- /dev/null +++ b/private-cicd/docs/PRIVATE-CICD-GUIDE.md @@ -0,0 +1,1079 @@ + + +# Private CloudStack ONTAP CI/CD guide + +## 1. Purpose, scope, and authority + +This directory implements the NetApp-fork-only presubmit for Apache CloudStack ONTAP primary storage changes. It +builds an exact source revision, runs CloudStack unit tests, creates Debian packages, deploys those packages to a +reverted lab VM, and runs ONTAP iSCSI and NFS3 integration suites. + +This is the consolidated operator and developer reference. The current [`Jenkinsfile`](../Jenkinsfile), scripts under +[`scripts/`](../scripts/), and ONTAP test code are authoritative if prose and executable code differ. + +The guide covers: + +- the trusted production Pipeline-from-SCM job with five-minute GitHub API + discovery, including first-PR cutover from `feature/CSTACKEX-223` to `main` + ([`CREATE-PRESUBMIT-JOB.md`](CREATE-PRESUBMIT-JOB.md)); +- the separate triggerless manual branch job; +- local CI-file validation and direct disposable-VM validation; +- Stage 1 build, unit-test, and Debian package handoff; +- Phase 2 snapshot, deployment, health, zone, and ONTAP tests; +- concurrency, reporting, artifacts, troubleshooting, and upstream hygiene. + +`private-cicd/` is downstream infrastructure. Do not include it in a pull request to `apache/cloudstack`. + +## 2. Current status + +Implemented: + +- trusted CI checkout and separate exact-SHA source checkout; +- full, no-tags Git checkout with SHA and branch/PR-ref reachability checks; +- builder, credential, inventory, and eligibility validation; +- full Maven build, unit tests, preserved Surefire XML, and Jenkins JUnit publication; +- Debian packages reusing Maven output, with manifests, checksums, and Marvin handoff; +- compatible-VM locking, exact vCenter snapshot revert, and one-run SSH-key bootstrap; +- package deployment, CloudStack configuration, health checks, zone setup, then iSCSI and NFS3 suites; +- runtime log collection, result retrieval, and literal/base64 password redaction; +- GitHub Check creation and final conclusion, start mail, final collated mail, and archived artifacts; +- source-aware cancellation of superseded runs only before VM lock acquisition. + +Phase 3 is partially implemented. JUnit publication, GitHub Check conclusion, final collated mail, and Jenkins +artifacts exist. A stable external per-PR artifact store and an explicit retention/cleanup policy for that store do +not exist. Jenkins currently retains 500 builds so five-minute discovery runs do not flush worker +history; that is still not a stable external retention policy. + +`PAUSE_BETWEEN_STAGES` is a boolean parameter for manual verification, off by default and intended to stay off for +automatic worker runs. When enabled, each pause times out after 60 minutes; the pause after snapshot revert holds the +VM lock. + +## 3. Architecture and data flow + +```text +Jenkins timer -> discover mode (every 5 minutes) + -> GitHub API -> unseen PR head SHAs + -> self-queue at most MAX_ACTIVE_WORKERS pull_request builds + -> extra SHAs deferred with watermark held + -> self-queued worker builds in the same job + -> trusted Jenkinsfile/scripts from protected main + -> request, eligibility, credential, inventory, and builder checks + -> exact PR or branch SHA checkout into cloudstack-src + -> Maven clean install and unit tests + -> Debian packages plus Marvin/API handoff + -> lock one compatible VM + -> exact vCenter snapshot revert and power-on + -> render temporary ontap.cfg and secrets.json + -> copy packages, Marvin, ONTAP tests, and Phase 2 scripts + -> configure VM -> health gates -> setup_zone -> iscsi -> nfs3 + -> collect, redact, retrieve, publish, archive, notify +``` + +Three domains remain separate: + +1. **Trusted CI source:** production loads `Jenkinsfile` and helpers from protected `main`. +2. **Source under test:** PR/manual source is fetched into `cloudstack-src` at an exact 40-character SHA. +3. **Lab configuration:** populated inventory is a Jenkins Secret file outside Git; secrets use separate credentials. + +The Kubernetes pod compiles and packages CloudStack; it does not run CloudStack services. The locked VM runs MySQL, +management, agent, KVM/libvirt, NFS, iSCSI, and Marvin. + +The full source tree is not copied to the VM. Phase 2 transfers the required `.deb` files and manifest, the +version-matched Marvin archive, `test/integration/plugins/ontap/`, VM-side scripts, trusted +`check_ontap_prereqs.py`, and generated runtime files. + +## 4. Current folder layout + +```text +private-cicd/ +├── Jenkinsfile +├── README.md +├── config/ +│ └── vm-inventory.yaml.example +├── docker/ +│ └── Dockerfile.driver +├── docs/ +│ ├── CREATE-PRESUBMIT-JOB.md +│ └── PRIVATE-CICD-GUIDE.md +└── scripts/ + ├── build-debs.sh + ├── check-build-prereqs.sh + ├── check-phase2-health.sh + ├── collect-phase2-logs.sh + ├── configure-cloudstack-vm.sh + ├── github-check-run.sh + ├── list-eligible-prs.py + ├── marvin-run.sh + ├── mvn-full.sh + ├── redact-phase2-results.py + ├── render-phase2-config.py + ├── run-phase2-remote.sh + ├── run-phase2-vm.sh + ├── test_list_eligible_prs.py + ├── validate-local.sh + └── vcenter-revert.py +``` + +The scripts respectively build packages; check the builder; check Phase 2 health; collect logs; configure the +VM; manage GitHub Checks; list eligible PRs for discovery; run Marvin; run Maven; redact results; render runtime +config; drive remote Phase 2; drive on-VM Phase 2; validate locally; and revert vCenter. + +Product code stays under `plugins/storage/volume/ontap/`. Tests stay under +[`test/integration/plugins/ontap/`](../../test/integration/plugins/ontap/) and are documented by +[`test/integration/plugins/ontap/README.md`](../../test/integration/plugins/ontap/README.md). + +## 5. Branch and security model + +### Production trust boundary + +Production loads executable CI code only from protected NetApp `main`: + +```text +Definition: Pipeline script from SCM +Repository: https://github.com/NetApp/cloudstack.git +Branch: */main +Script Path: private-cicd/Jenkinsfile +Lightweight checkout: off +``` + +PR code cannot replace trusted CI helpers before credentials are used. The source under test is checked out +separately. + +Until `feature/CSTACKEX-223` is merged, that Jenkinsfile is not on `main`. Create one job named +`cloudstack-ontap-presubmit` that loads CI from `*/feature/CSTACKEX-223`, using +[`CREATE-PRESUBMIT-JOB.md`](CREATE-PRESUBMIT-JOB.md). Its timer builds discover work and self-queue workers in the +same item. After merge, change only **Branches to build** to `*/main`. Keep the same job, GitHub App, credentials, +resources, and Check. Do not create a GitHub repository webhook: OpenLab Jenkins is not reachable from GitHub.com. +Do not create a GitHub Organization folder for this presubmit. + +### Manual-job trust boundary + +A manual job may load the Jenkinsfile from an unreviewed remote feature branch. That branch can request every +credential visible to the job. Create a separate triggerless job, restrict Configure/Build permissions, expose only +least-privilege lab credentials, review the diff before every run, and never convert it into the production job. + +Keep `cloudstack-presubmit-manual` disabled and enable it only for the duration of a branch run, or use +`SOURCE_MODE=branch` on the production job instead. + +### Secret boundaries + +Never commit populated inventory, real `ontap.cfg`, `secrets.json`, credentials, tokens, or private keys. Only +[`config/vm-inventory.yaml.example`](../config/vm-inventory.yaml.example) is tracked. Keep populated inventory outside +Git and upload it as a Secret file; keep passwords and keys in dedicated Jenkins credentials. + +Generated runtime files are mode `0600`, excluded from retrieved results, and deleted by the VM wrapper. Use only +approved lab networks, disposable CloudStack state, and an ONTAP SVM approved for destructive tests. + +## 6. Entry modes + +### Discovery and PR workers + +The timer starts `SOURCE_MODE=discover`. Discovery lists open non-draft PRs against `main`, compares `updated_at` +with the latest successful discovery watermark, and self-queues `SOURCE_MODE=pull_request` workers with: + +| Parameter | Source | +| --- | --- | +| `PR_ID` | pull request number | +| `PR_ACTION` | `opened` for the first SHA of that PR, otherwise `synchronize` | +| `PR_DRAFT` | `false` | +| `PR_REPOSITORY` | `NetApp/cloudstack` | +| `PR_BASE_BRANCH` | `main` | +| `PR_HEAD_BRANCH` | pull request head ref | +| `PR_HEAD_SHA` | pull request head SHA | +| `PR_AUTHOR_LOGIN` | pull request user login | +| `PR_AUTHOR_EMAIL` | pull request user email, if present | +| `PR_URL` | pull request HTML URL | +| `PR_TITLE` | pull request title | + +Discovery skips a SHA that already has a queued or recorded worker. It queues at most `MAX_ACTIVE_WORKERS` (default +5) `pull_request` workers at a time, oldest-updated first. Extra eligible SHAs are deferred and the watermark is +held so the next poll retries them. Manual **Build with Parameters** runs bypass the cap. Its first successful run +seeds the current time without queueing historical PRs. The eligibility stage repeats safety checks. Ineligible +parameterized requests become `NOT_BUILT`. + +### Manual exact-SHA branch + +```text +SOURCE_MODE=branch +SOURCE_BRANCH= +SOURCE_SHA= +``` + +Jenkins validates syntax, fetches only that branch ref, checks out the exact SHA, verifies reachability from the +branch, and requires a clean worktree. A moving branch name is never the build identity. + +### Local validation + +```bash +./private-cicd/scripts/validate-local.sh +./private-cicd/scripts/validate-local.sh --with-docker +``` + +The first command checks Bash syntax, compiles Python, and parses YAML when Ruby/Psych, PyYAML, or `yq` exists. +Read the output because YAML may be skipped. The Docker option builds the driver image. Neither runs Maven, package +creation, deployment, or ONTAP tests. + +### Direct throwaway VM + +Use a disposable Ubuntu 22.04 nested-KVM VM for script-by-script validation. Keep temporary inventory and secrets +under `/tmp`. This path does not test Jenkins, Lockable Resources, or vCenter revert. Never reuse it as the +production inventory VM. + +## 7. Prerequisites and pod resources + +Required Jenkins plugins: + +- Pipeline and Pipeline Utility Steps (`readYaml`); +- Kubernetes and Git; +- Lockable Resources; +- Credentials and Credentials Binding; +- Email Extension; +- GitHub Branch Source 2.7.1 or newer. + +Generic Webhook Trigger is not required. GitHub.com cannot POST to OpenLab Jenkins. + +Jenkins must create pods that can pull both images, reach GitHub/dependency sources, vCenter, and the VM, and satisfy +the Jenkinsfile resources. + +Current pod: + +```text +jnlp: + image: docker.repo.eng.netapp.com/global/devts-daas-prod/jnlp/jnlp-slave:latest + requests: 512Mi memory, 100m CPU + limits: 1Gi memory, 500m CPU + +cloudstack-driver: + image: docker.repo.eng.netapp.com/slocharl/cloudstack-presubmit-driver:v202608261233 + requests: 1Gi memory, 500m CPU, 5Gi ephemeral storage + limits: 8Gi memory, 2 CPU, 20Gi ephemeral storage +``` + +Pipeline steps run in `cloudstack-driver`. The `jnlp` sidecar is required for agent attachment; do not override its +command or arguments. + +Lab prerequisites: vCenter revert/power access; dedicated network/IP/VLAN allocations; reachable NFS and KVM +template URLs; ONTAP SVM with NFS3/iSCSI and matching data LIFs; dedicated lab accounts; and an externally reachable +HTTPS Jenkins URL for Check Details links. GitHub does not need inbound access to Jenkins. + +## 8. Builder image + +The current Jenkinsfile uses: + +```text +docker.repo.eng.netapp.com/slocharl/cloudstack-presubmit-driver:v202608261233 +``` + +Do not overwrite this immutable tag. Confirm workstation and Kubernetes pulls before production. + +[`docker/Dockerfile.driver`](../docker/Dockerfile.driver) builds Ubuntu 22.04 with JDK 17, Maven, Node 16.20.2, npm 8, +Debian/CloudStack packaging dependencies, Git/SSH, Python, PyYAML, and pyvmomi. + +Reproduce the current image reference: + +```bash +export IMAGE='docker.repo.eng.netapp.com/slocharl/cloudstack-presubmit-driver:v202608261233' +DOCKER_BUILDKIT=1 docker build --no-cache --provenance=false --sbom=false \ + -f private-cicd/docker/Dockerfile.driver -t "$IMAGE" private-cicd +docker push "$IMAGE" +docker pull "$IMAGE" +docker run --rm --entrypoint /bin/bash "$IMAGE" -lc \ + 'command -v sshpass java mvn node npm python3 dpkg-buildpackage dpkg-deb \ + dpkg-scanpackages sha256sum ssh scp ssh-keygen ssh-keyscan gzip gawk mysql virsh; \ + python3 -c "import setuptools, yaml, pyVim; print(\"ok\")"' +``` + +Only push that exact tag if publishing it for the first time. After a Dockerfile change, choose a new immutable tag, +push and pull it, then update the trusted Jenkinsfile image reference. Reused non-`latest` tags can remain cached +under Kubernetes `IfNotPresent`. This registry may reject BuildKit attestations, hence the two disabled options. +A local build does not prove a successful push; require a digest, zero status, and successful pull. + +The image contains build/client tools. MySQL server, KVM daemons, nginx, NFS server, and CloudStack services are +installed on the VM, where systemd is available. + +## 9. Jenkins credentials + +Create credentials in a scope visible to the job: + +| Suggested ID | Kind | Use | +| --- | --- | --- | +| `cloudstack-presubmit-inventory` | Secret file | Populated inventory outside Git | +| `cloudstack-presubmit-ssh` | Username with password | Initial VM login | +| `cloudstack-vcenter` | Username with password | Snapshot revert/power-on | +| `cloudstack-mysql-root` | Username with password | DB deployment administrator | +| `cloudstack-db` | Username with password | CloudStack DB | +| `cloudstack-kvm-host` | Username with password | Marvin `addHost` | +| `cloudstack-ontap` | Username with password | ONTAP test SVM | +| `cloudstack-admin` | Username with password | Fresh CloudStack API | +| `cloudstack-presubmit-github-app` | GitHub App | Check Runs, commit email, discovery PR list | +| site-defined optional ID | Username with password | Read-only Git access | + +Parameter mapping: + +| Parameter | Credential | +| --- | --- | +| `VM_INVENTORY_CREDENTIALS_ID` | Inventory Secret file | +| `VM_SSH_CREDENTIALS_ID` | VM username/password | +| `VCENTER_CREDENTIALS_ID` | vCenter username/password | +| `MYSQL_ROOT_CREDENTIALS_ID` | MySQL administrator | +| `CLOUD_DB_CREDENTIALS_ID` | CloudStack DB | +| `KVM_HOST_CREDENTIALS_ID` | KVM host | +| `ONTAP_CREDENTIALS_ID` | ONTAP | +| `CLOUDSTACK_ADMIN_CREDENTIALS_ID` | CloudStack API | +| `GITHUB_APP_CREDENTIALS_ID` | GitHub App | +| `GIT_CREDENTIALS_ID` | Optional read-only Git | + +`VM_SSH_CREDENTIALS_ID` is not a private-key credential. The pipeline uses its password once, generates an +ephemeral Ed25519 key, installs the public key, uses key authentication, then deletes local key material. + +All Jenkins password credentials must be non-empty; empty credential values do not arrive as usable environment +variables. Database passwords must work in `user:password@host`; avoid unescaped `:`, `@`, and whitespace. + +Typical UI path: + +**Manage Jenkins > Credentials > System/Jenkins > Global credentials (unrestricted) > Add Credentials** + +The Add action appears only after opening a store and domain. Folder credentials are visible only to jobs in that +folder. + +## 10. Inventory contract + +```bash +cp private-cicd/config/vm-inventory.yaml.example /tmp/cloudstack-presubmit-vm-inventory.yaml +chmod 600 /tmp/cloudstack-presubmit-vm-inventory.yaml +``` + +Fill every placeholder, validate YAML, upload it as a Secret file, and never copy it into Git. Include no passwords, +tokens, or keys. + +Each enabled VM needs: + +| Field | Contract | +| --- | --- | +| `id` | Unique readable ID | +| `enabled` | Omitted or `true` | +| `lock_resource` | Unique Jenkins resource name | +| `capabilities` | Includes `ubuntu22`, `kvm`, `ontap` | +| `vcenter_vm`, `snapshot` | Exact case-sensitive names | +| `ssh_host`, `ssh_user` | Pod-reachable host and deployment user | +| `bridge` | Existing bridge, normally `cloudbr0` | +| `dedicated_ips.management_server` | IP already on a VM interface | +| `dedicated_ips.kvm_host` | Marvin `addHost` address | +| `dedicated_ips.public.*` | Gateway, netmask, start, end | +| `dedicated_ips.pod.*` | Gateway, netmask, start, end | +| `zone.name`, `zone.network_type` | Zone identity/type | +| `zone.dns1`, `zone.internal_dns1` | Reachable DNS | +| `zone.guest_cidr`, `zone.guest_vlan_range` | Approved non-conflicting allocation | +| `zone.pod_name`, `zone.cluster_name` | Pod/cluster names | +| `zone.primary_storage_url`, `zone.secondary_storage_url` | Reachable NFS URLs | +| `zone.systemvm_template_url`, `zone.template_name` | KVM template data | +| `ontap.storage_ip`, `ontap.svm_name` | ONTAP data LIF and SVM | + +`dns2` and `internal_dns2` are optional when absent, but any present placeholder fails recursive validation. Confirm +concrete example defaults such as bridge and guest CIDR against the lab. + +The Pipeline requires a non-empty VM list, no placeholders anywhere, unique non-empty resource names, at least one +enabled compatible VM, and all current required fields. Runtime rendering selects exactly one enabled VM by `id` and +checks placeholders again. + +For each enabled VM create: + +```text +Lockable Resource Name: +Label: cloudstack-presubmit-vm +Reserved by: empty +``` + +The Pipeline locks by label, receives the resource name as `LOCKED_VM`, and maps it back to inventory. + +Also create: + +```text +Lockable Resource Name: cloudstack-presubmit-discovery +Label: empty +Reserved by: empty +``` + +This serializes only short discovery runs. It does not prevent worker builds from compiling concurrently. A +discovery run that finds the lock held skips its body and ends `NOT_BUILT` rather than queueing, so a slow poll +cannot accumulate a backlog of discovery builds. + +## 11. Clean snapshot contract + +The reusable baseline must: + +- run Ubuntu 22.04 with unique hostname and reserved networking; +- expose `/dev/kvm` and an up configured bridge; +- already have the management IP on an interface; +- permit password SSH for the deployment account; +- have working DNS, routes, apt, ONTAP 443, NFS, template, and dependency access; +- have enough CPU, RAM, and disk for nested KVM and CloudStack; +- contain no CloudStack packages/databases, Marvin state, secrets, or previous test resources. + +Take one uniquely named snapshot and record exact VM/snapshot names. The pipeline then disables apt timers, removes +third-party CloudStack sources, installs VM services, creates a local package repository, force-recreates databases, +sets API port 8096 before first management start, configures management with `--no-start`, configures libvirt/agent, +disables Docker and `ufw` in the isolated lab, and starts management once. + +The next snapshot revert provides isolation. `cleanup_zone` is not automatic. + +## 12. Production job setup + +### Controller and plugins + +1. Install section 7 plugins. +2. Set **Manage Jenkins > System > Jenkins Location > Jenkins URL** to externally reachable HTTPS; this controls + GitHub Check Details links. +3. Configure **Extended E-mail Notification** SMTP and an accepted sender. +4. Add section 9 credentials and section 10 resources. + +SMTP failure is non-fatal but should be tested. + +### GitHub App + +Create an organization-owned app, install it only on `NetApp/cloudstack`, disable its own webhook, and subscribe to +no events. Grant: + +| Permission | Level | +| --- | --- | +| Checks | Read and write | +| Contents | Read-only | +| Metadata | Read-only | +| Pull requests | Read-only | + +Grant no organization/account permissions. Convert the downloaded key to PKCS#8: + +```bash +openssl pkcs8 -topk8 -inform PEM -outform PEM \ + -in .private-key.pem -out converted-github-app.pem -nocrypt +``` + +The result is unencrypted PKCS#8 PEM. Add Kind **GitHub App**, ID +`cloudstack-presubmit-github-app`, with App ID and converted key. Test it when possible, then delete both PEM files. +An app not installed on `cloudstack` receives API 404. The pipeline rebinds the credential per API call because +installation tokens are short-lived. + +### Discovery in the same job + +Do not create a GitHub repository webhook. OpenLab is not publicly accessible; GitHub deliveries fail with +`failed to connect to host`. The production Jenkinsfile declares a five-minute timer. A `discover` build calls +GitHub, then self-queues one `pull_request` worker per unseen PR head SHA. Create the named Lockable Resource +`cloudstack-presubmit-discovery` to prevent overlapping discovery runs. Do not create a separate poller job. + +A `discover` build clones a single commit of `private-cicd/scripts` rather than the full repository, which keeps +each poll well inside the five-minute timer. That shallow clone reads the job's configured Git branch and remote, +so approve those two signatures when Script Security reports them; otherwise the build logs +`Shallow discovery checkout unavailable` and falls back to the slow full checkout. + +### Pipeline item + +Create a regular Pipeline, not Freestyle, Multibranch, or Organization: + +```text +Name: cloudstack-ontap-presubmit +Definition: Pipeline script from SCM +SCM: Git +Repository: https://github.com/NetApp/cloudstack.git +Credentials: none or read-only Git credential +Branch: */main +Script Path: private-cicd/Jenkinsfile +Lightweight checkout: off +Concurrent builds: enabled +Job-level abort previous: disabled +``` + +Do not add UI triggers manually. The Jenkinsfile declares `cron('H/5 * * * *')`. + +### First load and script approvals + +Click **Build Now** once. It may load the new defaults or fail once using old defaults. Refresh, confirm +`SOURCE_MODE` offers `discover`, `pull_request`, and `branch`, then run `discover` once to seed the watermark without +queueing historical PRs. Confirm **Configure > Build Triggers** shows the five-minute timer and no Generic Webhook +Trigger. + +`Abort superseded run` uses Jenkins internal APIs. Under **Manage Jenkins > In-process Script Approval**, approve +only signatures actually requested by this trusted `WorkflowScript` for obtaining the job, enumerating builds, +reading parameters/environment, stopping a matching build, and reading the job's Git branch and remote for the +shallow discovery clone. Never approve unrelated calls or use **Approve assuming permission check**. Exact +signatures vary by controller/plugin versions. + +### Parameterized smoke test + +Before relying on discovery, run a real non-draft PR manually: + +```text +SOURCE_MODE=pull_request +PR_ID= +PR_ACTION=opened +PR_DRAFT=false +PR_REPOSITORY=NetApp/cloudstack +PR_BASE_BRANCH=main +PR_HEAD_BRANCH= +PR_HEAD_SHA= +EXPECTED_REPOSITORY=NetApp/cloudstack +CLOUDSTACK_GIT_URL=https://github.com/NetApp/cloudstack.git +GITHUB_APP_CREDENTIALS_ID=cloudstack-presubmit-github-app +VM_INVENTORY_CREDENTIALS_ID=cloudstack-presubmit-inventory +VCENTER_HOST= +=
+PAUSE_BETWEEN_STAGES=true +``` + +Confirm exact-SHA Check, start mail attempt, clean checkout, Stage 1/2 pass, matching Check conclusion, one final +mail attempt, and secret-free archives. Turn pauses off for automatic workers. + +### Required Check + +After the Check exists, edit **Repository Settings > Branches > main rule**, enable required status checks, and +select exactly: + +```text +cloudstack-ontap-presubmit +``` + +Without protection it is informational. Verify pending/failed results block merge and each new SHA needs a new Check. + +## 13. Manual branch job and exact SHA + +Create a separate Pipeline-from-SCM job: + +```text +Repository: https://github.com/NetApp/cloudstack.git +Branch: */ +Script Path: private-cicd/Jenkinsfile +Lightweight checkout: off +All triggers: off +Concurrent builds: enabled +``` + +The SCM branch selects executable CI; source parameters independently select product source. + +```bash +git push origin +git fetch origin +BRANCH='' +LOCAL_SHA="$(git rev-parse HEAD)" +REMOTE_SHA="$(git rev-parse "origin/$BRANCH")" +test "$LOCAL_SHA" = "$REMOTE_SHA" +printf '%s\n' "$REMOTE_SHA" +``` + +Run: + +```text +SOURCE_MODE=branch +SOURCE_BRANCH= +SOURCE_SHA= +PR_* fields=blank except PR_DRAFT=false and optional author email/title +EXPECTED_REPOSITORY=NetApp/cloudstack +CLOUDSTACK_GIT_URL=https://github.com/NetApp/cloudstack.git +VM_INVENTORY_CREDENTIALS_ID=cloudstack-presubmit-inventory +VCENTER_HOST= +=
+``` + +Branch mode creates no GitHub Check. It sends mail only when an address is available. The first unparameterized run +may fail while loading parameters. Keep the job triggerless and review approvals against the untrusted branch. + +## 14. Jenkins flow by stage + +The Pipeline has an eight-hour timeout and retains 500 builds so discovery runs do not flush worker history. + +Every stage body lives in a top-level method above the pipeline directive, and each `steps` block only calls one of +them. This is not a style preference. The JVM caps a single method at 64KB of bytecode and Jenkins compiles the +whole body of the pipeline directive into one method, so inlining a stage body again reintroduces +`Method too large: WorkflowScript.___cps___NNNN` at parse time and no build can start +([JENKINS-37984](https://issues.jenkins.io/browse/JENKINS-37984)). Those methods call steps, so none of them may +carry `@NonCPS`. Keep new stage logic in a method and keep the shared `when` condition in `isWorkerStageEnabled()`. + +### Discover PR revisions + +Runs only for `SOURCE_MODE=discover`. Lists eligible PRs, self-queues workers under the cap, holds the watermark +when work is deferred, then finishes `NOT_BUILT` so mail, Checks, and tests are skipped. The worker stages are +skipped on `SOURCE_MODE`, not on `NOT_BUILT`, so a discovery run that breaks still reports `FAILURE`: a red +discovery build always means discovery itself is broken. + +Discovery titles its build `discover: queued N, deferred M`, or `discover: skipped, lock held` when it overlapped a +slower run, and puts the watermark in the description. + +### Validate source request + +Records start time/integration false; validates mode, branch/SHA or PR/repository, HTTPS `.git` URL, credential IDs, +and vCenter hostname; sets display name/description. PR repository must match `EXPECTED_REPOSITORY`. Worker titles +read `PR-98 diff #3 synchronize` so an author can find their run among the five-minute discovery builds, with the +head SHA and PR title in the description. + +### Check eligibility + +Branch mode continues. PR mode requires an eligible action, non-draft state, and base `main`; otherwise it becomes +`NOT_BUILT`. + +### Abort superseded run + +Source key is `pull_request:` or `branch:`. Only an older running build of this job with the +same key is considered. It is stopped only while `PRESUBMIT_INTEGRATION_STARTED` is not true. That flag becomes true +immediately after VM lock acquisition. Maven, packaging, and lock wait are abortable; snapshot revert onward is not. + +### Checkout CI scripts + +Deletes workspace and runs `checkout(scm)`. Production SCM therefore must be protected `main`. + +### Start PR reporting + +PR mode creates `cloudstack-ontap-presubmit` on exact head SHA, links `BUILD_URL`, resolves payload or commit email, +ignores GitHub noreply addresses, and attempts one start mail. GitHub/SMTP errors do not change the build result. + +### Validate builder + +Rejects empty/placeholder configuration, validates inventory before a lock, runs `check-build-prereqs.sh` in +`cloudstack-driver`, and optionally pauses. + +### Checkout source + +Fetches `refs/pull//head` or `refs/heads/`, then: + +```text +honorRefspec: true +noTags: true +shallow: false +``` + +Checkout is not shallow. Jenkins requires exact `HEAD`, reachability from fetched ref, clean status, and writes +`presubmit-results/source.properties`. + +### Build and unit tests + +Calls `mvn-full.sh` with simulator false, `MAVEN_THREADS=1C`, `MAVEN_OPTS=-Xms1g -Xmx8g`, mode-independent source ID, +and exact SHA. + +### Build Debian packages + +Calls `build-debs.sh` with `REUSE_MAVEN_BUILD=true`, the same Maven options, and output +`cloudstack-src/dist/deb-all`. + +### Deploy and run ONTAP integration + +Locks one `cloudstack-presubmit-vm`; marks integration started inside the lock; maps inventory; reverts exact +snapshot with `--insecure` internal-lab TLS mode; powers on; optionally pauses while holding lock; binds runtime +credentials; runs remote Phase 2; and holds the lock through result retrieval/cleanup. + +### Post actions + +Always publishes Maven Surefire XML as JUnit (empty allowed), maps `SUCCESS`/`ABORTED`/other to +`success`/`cancelled`/`failure`, updates the collated log, completes an existing Check, attempts one final mail, and +archives/fingerprints `presubmit-results/**/*` plus `cloudstack-src/dist/deb-all/**/*`. `NOT_BUILT` gets no final PR +reporting. + +## 15. Stage 1 contracts + +### Builder + +`check-build-prereqs.sh` requires Java 17, Maven, Node 16.x, npm 8.x, Git, Python with setuptools/PyYAML/pyvmomi, +Debian tools, checksum tools, and SSH/scp/sshpass/key tools. + +### Maven + +Effective Jenkins command: + +```bash +mvn -B -P developer,systemvm clean install -T1C \ + -Dsystemvm -Dcs.replace.properties=replace.properties.tmp +``` + +Simulator/noredist are off and tests remain enabled. Packaging `replace.properties` plus project version allows +target reuse. + +```text +presubmit-results/unit-tests/ +├── maven.log +├── stage1-handoff.properties +├── surefire//target/surefire-reports/*.xml +└── surefire-reports.tar.gz +``` + +The handoff records source ID/SHA, mirrored legacy PR keys, Maven result, and report count. Failure prevents package +creation while preserving available results outside the source tree. + +### Debian packages + +The script edits only workspace `debian/rules` to skip its second Maven build, then runs: + +```bash +dpkg-buildpackage -uc -us -b -d -nc +``` + +`-nc` preserves target; Debian configuration, npm UI, install, and assembly remain. `-d` skips dependency checks, so +the immutable image must supply dependencies. + +Required packages: `cloudstack-common`, `cloudstack-agent`, `cloudstack-management`. Required handoff: + +```text +cloudstack-src/dist/deb-all/ +├── *.deb, *.changes, *.buildinfo +├── deb-build.log +├── cloudstack-{common,agent,management}.contents +├── package-manifest.tsv +├── SHA256SUMS +└── marvin/ + ├── Marvin-*.tar.gz + ├── commands.xml + ├── manifest.tsv + └── SHA256SUMS +``` + +Missing/duplicate Marvin, API metadata, package, expected contents, or checksum fails before VM acquisition. + +## 16. Phase 2 contracts + +### Render and transfer + +`render-phase2-config.py` combines inventory with DB, KVM, ONTAP, and CloudStack API credentials; enables both +protocols; sets API port 8096; and writes mode-`0600` `ontap.cfg` and `secrets.json`. + +`run-phase2-remote.sh` verifies Marvin checksum, generates an Ed25519 key, waits up to 900 seconds for password SSH, +uses strict checking against a freshly scanned temporary known-hosts file, and enables keepalives. Remote root: + +```text +/tmp/cloudstack-phase2-- +``` + +### Configure gate + +`configure-cloudstack-vm.sh`: + +1. requires root; +2. masks apt timers, waits on locks, and signals only apt-family holders; +3. installs VM dependencies and configures MySQL; +4. exports local primary/secondary NFS; +5. disables active Docker and `ufw` in the isolated lab; +6. serves exact built packages from local nginx apt; +7. installs exact common/management versions; +8. force-recreates databases and inserts API port 8096 before first start; +9. runs `cloudstack-setup-management --no-start`; +10. installs exact agent, configures unauthenticated lab libvirt TCP and bridge; +11. starts management once, then restarts agent. + +If setup lacks `--systemvm-templates`, the script fallback installs the published 4.22.0 KVM template URL directly; +do not assume that fallback comes from inventory. + +### Health gate + +Requires active MySQL/libvirtd/iscsid/NFS/agent/management; manifest-matching package versions; `/dev/kvm`; bridge; +local management IP; virsh; iSCSI IQN; both NFS exports; ONTAP module in the shaded management JAR; DB access; API +8096 within 900 seconds; ONTAP 443; and SVM protocol/data-LIF prerequisites for both protocols. + +The plugin is shaded into the management uber-JAR; no standalone ONTAP JAR is expected. + +### Marvin order + +Marvin comes from Stage 1 and is installed into an isolated environment. Tests use `nose_compat.py`. + +1. `run_tests.sh setup_zone` +2. `run_tests.sh iscsi` +3. `run_tests.sh nfs3` + +Zone failure prevents protocols. NFS3 is still attempted after iSCSI failure, then combined result fails. Suites are +sequential because they share one KVM host. `cleanup_zone` is intentionally absent. + +See [`test/integration/plugins/ontap/README.md`](../../test/integration/plugins/ontap/README.md) for suite semantics. + +### Exit + +Every VM-wrapper exit collects system/CloudStack/Marvin logs best-effort, redacts literal/base64 password values, +writes `phase2.properties`, and deletes generated secrets. Jenkins retrieves via SSH/tar and removes remote root. +Successful tests with failed retrieval still fail the stage. + +## 17. Reporting and artifacts + +PR mode creates one `cloudstack-ontap-presubmit` Check on exact SHA, starts it `in_progress`, links the build, then +completes the same Check as `success`, `failure`, or `cancelled`. API failures are non-fatal to Jenkins. A required +Check still remains merge-safe when API reporting fails because no required success exists. + +`presubmit-results/stage-events.tsv` contains structured UTC stage events. The final post action converts text logs +to HTML, writes `presubmit-results/report.html`, and archives those files before sending mail. Mail behavior: + +- one short HTML start mail styled like the final report, with a `STARTED` banner and a bordered table of source, + title, diff number, SHA, and start time, plus build and console links; +- one final HTML mail with result, diff number, VM, total duration, and a stage table with start time, duration, VM + wait, and links to the applicable HTML log; +- a complete ONTAP test table with failures first and per-test links into the archived suite HTML logs; +- no raw log tails in the message body; use the linked report or log to debug; +- payload email first, then commit email; noreply ignored; +- missing address or SMTP failure does not alter result; +- no per-stage threading because Email Extension provides no stable Message-ID. + +Both mails share the subject base `CloudStack presubmit PR- , diff #<n> (<sha12>)`, so mail clients thread +the pair. The diff number counts the distinct commits presubmitted for that PR: re-running the same commit keeps its +number and each new push increments it. Only builds still retained by `logRotator(numToKeepStr: '500')` are inspected, +so a PR with more than 500 intervening builds can report a lower number. When the lookup fails the subject falls back +to `diff <sha12>` and the diff row is omitted. + +Jenkins archives: + +```text +presubmit-results/**/* +cloudstack-src/dist/deb-all/**/* +``` + +Key paths: + +- browsable report: `presubmit-results/report.html`; +- stage events: `presubmit-results/stage-events.tsv`; +- Maven/JUnit: `presubmit-results/unit-tests/`; +- packages: `cloudstack-src/dist/deb-all/`; +- configure/health: `presubmit-results/phase2/{configure-cloudstack,health-check}.log`; +- Marvin: `presubmit-results/phase2/marvin/`; +- runtime: `presubmit-results/phase2/runtime-logs/`; +- marker: `presubmit-results/phase2/phase2.properties`. + +Phase 2 summaries are archived but not currently Jenkins JUnit. + +## 18. Direct VM/script validation + +Use root on a disposable clean VM. + +### Validate, build, package + +```bash +export SOURCE_DIR=/root/cloudstack +export CICD_DIR="$SOURCE_DIR/private-cicd" +cd "$SOURCE_DIR" +test -z "$(git status --porcelain)" +"$CICD_DIR/scripts/validate-local.sh" +"$CICD_DIR/scripts/check-build-prereqs.sh" + +export RESULT_DIR=/tmp/presubmit-results/unit-tests +export SOURCE_ID=local +export SOURCE_SHA="$(git rev-parse HEAD)" +export ENABLE_SIMULATOR=false MAVEN_THREADS=1C MAVEN_OPTS='-Xms1g -Xmx8g' +"$CICD_DIR/scripts/mvn-full.sh" + +export RESULT_DIR="$SOURCE_DIR/dist/deb-all" REUSE_MAVEN_BUILD=true +"$CICD_DIR/scripts/build-debs.sh" +``` + +Do not package after Maven failure. Verify three packages, manifests, Marvin, commands XML, and checksums. + +### Render temporary config + +Create `/tmp/throwaway-inventory.yaml` from the example, remove every placeholder, use `id: throwaway-vm`, and give +non-placeholder dummy values to Jenkins-only fields. Read secrets without shell history: + +```bash +read -rp 'MySQL admin user: ' MYSQL_ROOT_USER +read -rsp 'MySQL admin password: ' MYSQL_ROOT_PASSWORD; echo +read -rp 'Cloud DB user: ' CLOUD_DB_USER +read -rsp 'Cloud DB password: ' CLOUD_DB_PASSWORD; echo +read -rp 'KVM host user: ' KVM_HOST_USER +read -rsp 'KVM host password: ' KVM_HOST_PASSWORD; echo +read -rp 'ONTAP user: ' ONTAP_USER +read -rsp 'ONTAP password: ' ONTAP_PASSWORD; echo +read -rp 'CloudStack API user: ' CLOUDSTACK_ADMIN_USER +read -rsp 'CloudStack API password: ' CLOUDSTACK_ADMIN_PASSWORD; echo +export MYSQL_ROOT_USER MYSQL_ROOT_PASSWORD CLOUD_DB_USER CLOUD_DB_PASSWORD +export KVM_HOST_USER KVM_HOST_PASSWORD ONTAP_USER ONTAP_PASSWORD +export CLOUDSTACK_ADMIN_USER CLOUDSTACK_ADMIN_PASSWORD + +python3 "$CICD_DIR/scripts/render-phase2-config.py" \ + --inventory /tmp/throwaway-inventory.yaml --vm-id throwaway-vm \ + --ontap-config /tmp/ontap.cfg --secrets /tmp/secrets.json +``` + +Do not print those files. + +### Build workspace and run gates + +Create `/tmp/cloudstack-phase2-manual` with `scripts/`, `source/`, `debs/`, and `results/`. Copy the six VM-side +scripts plus `check_ontap_prereqs.py`; ONTAP tests and generated `ontap.cfg`; one Marvin archive; packages and +manifest; and `secrets.json`. Set secrets/config mode `0600`. + +```bash +export PHASE2_ROOT=/tmp/cloudstack-phase2-manual BRIDGE=cloudbr0 +set -o pipefail +DEB_DIR="$PHASE2_ROOT/debs" SECRETS_FILE="$PHASE2_ROOT/secrets.json" \ +ONTAP_CONFIG="$PHASE2_ROOT/source/test/integration/plugins/ontap/ontap.cfg" BRIDGE="$BRIDGE" \ + "$PHASE2_ROOT/scripts/configure-cloudstack-vm.sh" 2>&1 \ + | tee "$PHASE2_ROOT/results/configure-cloudstack.log" + +MANIFEST_FILE="$PHASE2_ROOT/debs/package-manifest.tsv" \ +ONTAP_CONFIG="$PHASE2_ROOT/source/test/integration/plugins/ontap/ontap.cfg" BRIDGE="$BRIDGE" \ + "$PHASE2_ROOT/scripts/check-phase2-health.sh" 2>&1 \ + | tee "$PHASE2_ROOT/results/health-check.log" + +cd "$PHASE2_ROOT/source" +bash test/integration/plugins/ontap/run_tests.sh setup_zone +bash test/integration/plugins/ontap/run_tests.sh iscsi +bash test/integration/plugins/ontap/run_tests.sh nfs3 + +OUTPUT_DIR="$PHASE2_ROOT/results" "$PHASE2_ROOT/scripts/collect-phase2-logs.sh" +python3 "$PHASE2_ROOT/scripts/redact-phase2-results.py" \ + --results "$PHASE2_ROOT/results" --secrets "$PHASE2_ROOT/secrets.json" \ + --ontap-config "$PHASE2_ROOT/source/test/integration/plugins/ontap/ontap.cfg" +``` + +Delete generated secrets after redaction. `pipefail` prevents `tee` hiding a failed gate. After partial +configuration, rebuild/revert rather than stacking another run on uncertain state. + +To test the wrapper, rebuild a clean workspace and run only: + +```bash +PHASE2_ROOT=/tmp/cloudstack-phase2-manual BRIDGE=cloudbr0 \ + /tmp/cloudstack-phase2-manual/scripts/run-phase2-vm.sh +``` + +Expect `PHASE2_RESULT=SUCCESS` and deleted generated secret files. + +## 19. Concurrency + +Keep concurrent builds enabled and job-level abort-previous disabled. + +- same PR ID: newer revision aborts older run only before lock; +- same manual source branch: newer exact SHA aborts older run only before lock; +- different PR IDs or branch names never abort each other; +- after lock acquisition, older integration finishes independently; +- with one VM, builds compile/package concurrently and serialize at Phase 2; +- aborted PR Check concludes `cancelled`; protected old/new revisions report separately. + +The boundary is VM lock acquisition, not stage entry or snapshot completion. Test same-source cancellation during +Maven and after `Deploy and run ONTAP integration STARTED`, plus different PRs concurrently. + +## 20. Troubleshooting + +### Pod or builder + +No stage output, `ErrImagePull`, `ImagePullBackOff`, or `FailedScheduling`: verify exact image existence, pull secret, +namespace, `jnlp`, pod events, and current requests. Pull occurs before scripts can diagnose it. + +Builder check failure: wrong Java/Node/npm or missing sshpass/PyYAML/pyvmomi/setuptools means stale/cached image. + +`OOMKilled` means the 8Gi cgroup limit was exceeded; `FailedScheduling` means requests could not be satisfied. The +authoritative memory request is 1Gi, limit 8Gi, with Maven heap options `-Xms1g -Xmx8g`. + +### Source, Groovy, Maven, package + +Source failure: verify full SHA, selected PR/branch ref, reachability, credential, HTTPS `.git` URL, and branch +syntax. A worker checkout is full, so do not diagnose it as shallow history. Only `discover` builds clone +shallowly, and they read nothing but `private-cicd/scripts`. + +Discovery builds queued behind `cloudstack-presubmit-discovery`: cancel the queued ones, then confirm the loaded +Jenkinsfile skips the lock when held and that the running discovery clones shallowly instead of taking minutes. + +`Method too large: WorkflowScript.___cps___NNNN` before any stage starts: a stage body was inlined into the +pipeline directive instead of a top-level method, or the directive itself grew too large. Move the new code into a +method as described in section 14; do not rely on the controller-wide `SCRIPT_SPLITTING_TRANSFORMATION` property. + +Groovy rejection: approve only the reported trusted WorkflowScript signature. Later skipped stages and empty-JUnit +messages are symptoms. + +Maven/package: inspect `maven.log` then `deb-build.log`. Exit 141 before Maven can be shell SIGPIPE under `pipefail`. +Target reuse requires `client/target`, API `commands.xml`, and exactly one Marvin archive. + +### Inventory, lock, snapshot, SSH + +Inventory errors: placeholder, missing field, no compatible enabled VM, or duplicate/empty resource. Lock wait: +missing label, name mismatch, reservation, or another build. + +Snapshot: verify vCenter host/permission/network and exact unique names. SSH: verify password login in snapshot, +Username-with-password Kind, correct user/password, pod-reachable host, and boot within 900 seconds. + +### VM configure and health + +Apt: current script disables timers, waits 300 seconds, signals only apt-family holders, repairs dpkg, and retries. +It deliberately leaves a non-apt lock holder alone and logs it. + +Duplicate schema key/column: likely interrupted first upgrade or stale script. Current flow force-recreates DBs, +sets 8096 before start, uses `--no-start`, and starts management once. Revert cleanly; do not repair ad hoc. + +API timeout: inspect management journal/log and DB config. Initial upgrade can take minutes; gate waits 900 seconds. + +Template 404: published names may use three version components, such as +`systemvmtemplate-4.22.0-x86_64-kvm.qcow2.bz2`; four-part project versions can be wrong. HTTP `000` is connectivity. + +ONTAP: inspect health, setup-env, setup_zone, iscsi/nfs3 logs, summaries, and copied Marvin logs. iSCSI hot-unplug +error 530 may be a guest/lab limitation but remains a failure. + +The final mail and `presubmit-results/report.html` list every Marvin test with failures first. Select a test's +**HTML log** link to open its archived suite log at that test. For an iSCSI VM workflow failure, use the link under +`phase2/marvin/ontap-results/.../suites/iscsi_vm_workflow/stdout.log.html`; do not diagnose it from Maven, +configure, or health logs that already passed. + +### GitHub and mail + +- rejected App key: require unencrypted PKCS#8 PEM; +- missing GitHub App Kind: update GitHub Branch Source; +- Check 401: bad credential; 403: permission/rate; 404: installation/repository; +- broken Details link: fix Jenkins URL; +- discovery 401/403 listing PRs: App missing Pull requests read or not installed on `cloudstack`; +- discovery skips a new push: SHA already recorded, PR is a draft, base is not `main`, `updated_at` is not after + the latest successful discovery watermark, or five `pull_request` workers are already active (console says + `Deferred`); +- discovery deferred a SHA: wait for a worker to finish; the next poll retries because the watermark was held; +- `failed to connect to host` on a GitHub webhook: expected; delete the webhook and use discovery; +- branch mode: no Check is expected. + +Mail failures appear as `Presubmit mail was not sent`; no address, noreply address, or SMTP failure does not change +the build. + +## 21. Production checklist + +- [ ] trusted `main` contains reviewed current CI code; +- [ ] immutable builder image pulls in Kubernetes; +- [ ] driver request is 1Gi and limit is 8Gi; +- [ ] plugins, external HTTPS Jenkins URL, and SMTP are configured; +- [ ] credentials have correct Kinds and non-empty passwords; +- [ ] populated inventory remains outside Git; +- [ ] each enabled VM maps to a labeled resource and clean snapshot; +- [ ] production SCM is `*/main` after CSTACKEX-223 is merged (until then `*/feature/CSTACKEX-223`), lightweight off, and source checkout full; +- [ ] first load applied parameters and only needed signatures are approved; +- [ ] parameterized PR smoke run passed; +- [ ] App installation, exact-SHA Check conclusion, and discovery queue of a new SHA are proven; +- [ ] Check is required on `main`; +- [ ] discovery defers extra SHAs once five workers are active and a later poll queues them; +- [ ] concurrency behavior is proven; +- [ ] archives contain no generated config, token, key, or secrets; +- [ ] `PAUSE_BETWEEN_STAGES` is off for automatic worker runs. + +## 22. Remaining gaps and future invariants + +Remaining delivery gaps: + +1. stable external per-PR result location; +2. retention, cleanup, ownership, and access policy for it. + +Current limitations: internal-lab vCenter `--insecure`; Phase 2 not Jenkins JUnit; snapshot-based rather than +automatic zone cleanup; and unauthenticated libvirt TCP suitable only for an isolated lab. + +Future changes must preserve trusted-main execution, exact-SHA identity/reachability, secret separation, package +gates before lock, source-key cancellation only before lock, lock ownership through result retrieval, and +notification failures not replacing test results. + +## 23. Upstream hygiene + +Before an Apache PR: + +```bash +git fetch apache +git log --oneline apache/main..HEAD -- private-cicd/ +git diff --name-only apache/main...HEAD +``` + +If private CI commits appear, branch from the intended `apache/main` and cherry-pick only upstream product/test +changes. The private-cicd log should be empty. ONTAP product/integration tests may belong upstream; NetApp Jenkins, +inventory operations, credentials, and private CI implementation do not. diff --git a/private-cicd/scripts/build-debs.sh b/private-cicd/scripts/build-debs.sh new file mode 100755 index 000000000000..bd2b149ef372 --- /dev/null +++ b/private-cicd/scripts/build-debs.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Build CloudStack Debian packages with dpkg-buildpackage -uc -us -b -d, +# adding -nc when REUSE_MAVEN_BUILD=true and otherwise building with +# ACS_BUILD_OPTS="-DskipTests". Copy .deb/.changes/.buildinfo and the +# generated Marvin archive into RESULT_DIR, require cloudstack-common, +# cloudstack-agent, and cloudstack-management, then write manifests and sums. +# Dependencies: +# SOURCE_DIR — CloudStack repo root with debian/ (required). +# RESULT_DIR — output dir (default: SOURCE_DIR/dist/deb-all). +# Optional REUSE_MAVEN_BUILD=true skips debian/rules' mvn clean package and +# reuses target/ from mvn-full.sh. Jenkins always sets this after unit tests. +# Commands: dpkg-buildpackage, dpkg-deb, python3 with setuptools, sha256sum; +# Maven/JDK as used by debian/rules unless Maven is reused. +# Does not call other private-cicd scripts. + +set -euo pipefail + +SOURCE_DIR="${SOURCE_DIR:?SOURCE_DIR is required}" +SOURCE_DIR="$(cd "$SOURCE_DIR" && pwd)" +RESULT_DIR="${RESULT_DIR:-$SOURCE_DIR/dist/deb-all}" + +for command_name in dpkg-buildpackage dpkg-deb python3 sha256sum; do + if ! command -v "$command_name" >/dev/null 2>&1; then + echo "Required command is missing: $command_name" >&2 + exit 1 + fi +done +if ! python3 -c 'import setuptools' >/dev/null 2>&1; then + echo "Python setuptools is required to build the Marvin archive" >&2 + echo "Install python3-setuptools before running the Debian package build" >&2 + exit 1 +fi + +mkdir -p "$RESULT_DIR" +RESULT_DIR="$(cd "$RESULT_DIR" && pwd)" +rm -rf "$RESULT_DIR" +mkdir -p "$RESULT_DIR" +temporary_log="$(mktemp)" +trap 'rm -f "$temporary_log"' EXIT + +cd "$SOURCE_DIR" + +REUSE_MAVEN_BUILD="${REUSE_MAVEN_BUILD:-false}" +if [[ "$REUSE_MAVEN_BUILD" == "true" ]]; then + echo "==> Reusing Maven target/ from mvn-full.sh" + missing=0 + if [[ ! -d "$SOURCE_DIR/client/target" ]]; then + echo "client/target is missing; run mvn-full.sh before packaging" >&2 + missing=1 + fi + if [[ ! -f "$SOURCE_DIR/tools/apidoc/target/commands.xml" ]]; then + echo "tools/apidoc/target/commands.xml is missing; run mvn-full.sh before packaging" >&2 + missing=1 + fi + shopt -s nullglob + marvin_precheck=("$SOURCE_DIR"/tools/marvin/dist/Marvin-*.tar.gz) + shopt -u nullglob + if [[ "${#marvin_precheck[@]}" -eq 0 ]]; then + echo "tools/marvin/dist/Marvin-*.tar.gz is missing; run mvn-full.sh before packaging" >&2 + missing=1 + fi + if [[ "$missing" -ne 0 ]]; then + exit 1 + fi + + python3 - "$SOURCE_DIR/debian/rules" <<'PY' +from pathlib import Path +import sys + +rules = Path(sys.argv[1]) +text = rules.read_text() +old = ( + "override_dh_auto_build:\n" + "\tmvn clean package -Psystemvm,developer -Dsystemvm \\\n" + "\t -Dcs.replace.properties=replace.properties.tmp \\\n" + "\t -Dmaven.repo.local=$(HOME)/.m2/repository \\\n" + "\t ${ACS_BUILD_OPTS}\n" +) +new = ( + "override_dh_auto_build:\n" + '\t@echo "Reusing Maven target/ from mvn-full.sh; skipping mvn clean package"\n' +) +if old not in text: + raise SystemExit("debian/rules override_dh_auto_build did not match the expected recipe") +rules.write_text(text.replace(old, new, 1)) +PY +fi + +echo "==> Building CloudStack Debian packages" +if [[ "$REUSE_MAVEN_BUILD" == "true" ]]; then + echo "==> Command: dpkg-buildpackage -uc -us -b -d -nc (Maven reused)" + set +e + dpkg-buildpackage -uc -us -b -d -nc \ + 2>&1 | tee "$temporary_log" + package_status="${PIPESTATUS[0]}" + set -e +else + echo "==> Command: ACS_BUILD_OPTS=\"-DskipTests\" dpkg-buildpackage -uc -us -b -d" + set +e + ACS_BUILD_OPTS="-DskipTests" dpkg-buildpackage -uc -us -b -d \ + 2>&1 | tee "$temporary_log" + package_status="${PIPESTATUS[0]}" + set -e +fi + +mkdir -p "$RESULT_DIR" +mv "$temporary_log" "$RESULT_DIR/deb-build.log" +trap - EXIT + +if [[ "$package_status" -ne 0 ]]; then + echo "Debian package build failed with status $package_status" >&2 + exit "$package_status" +fi + +commands_xml="$SOURCE_DIR/tools/apidoc/target/commands.xml" +if [[ ! -f "$commands_xml" ]]; then + echo "Marvin API metadata is missing after the Debian build: $commands_xml" >&2 + exit 1 +fi + +shopt -s nullglob +marvin_archives=("$SOURCE_DIR"/tools/marvin/dist/Marvin-*.tar.gz) +artifacts=(../*.deb ../*.changes ../*.buildinfo) +debs=(../*.deb) +shopt -u nullglob + +if [[ "${#marvin_archives[@]}" -eq 0 ]]; then + echo "No Marvin archive was produced under tools/marvin/dist" >&2 + echo "Verify that the build environment provides python3-setuptools, then inspect deb-build.log" >&2 + exit 1 +fi +if [[ "${#marvin_archives[@]}" -ne 1 ]]; then + echo "Expected one Marvin archive, found ${#marvin_archives[@]} under tools/marvin/dist" >&2 + printf ' %s\n' "${marvin_archives[@]}" >&2 + exit 1 +fi + +if [[ "${#debs[@]}" -eq 0 ]]; then + echo "No Debian packages were produced" >&2 + exit 1 +fi + +cp "${artifacts[@]}" "$RESULT_DIR/" +mkdir -p "$RESULT_DIR/marvin" +cp "${marvin_archives[0]}" "$RESULT_DIR/marvin/" +cp "$commands_xml" "$RESULT_DIR/marvin/commands.xml" + +required_packages=( + cloudstack-common + cloudstack-agent + cloudstack-management +) + +declare -A package_files +for package_name in "${required_packages[@]}"; do + matches=("$RESULT_DIR/${package_name}_"*.deb) + if [[ ! -e "${matches[0]}" ]]; then + echo "Required Debian package is missing: $package_name" >&2 + exit 1 + fi + + actual_name="$(dpkg-deb -f "${matches[0]}" Package)" + if [[ "$actual_name" != "$package_name" ]]; then + echo "Expected package '$package_name', found '$actual_name'" >&2 + exit 1 + fi + package_files["$package_name"]="${matches[0]}" +done + +dpkg-deb -c "${package_files[cloudstack-common]}" > "$RESULT_DIR/cloudstack-common.contents" +dpkg-deb -c "${package_files[cloudstack-agent]}" > "$RESULT_DIR/cloudstack-agent.contents" +dpkg-deb -c "${package_files[cloudstack-management]}" > "$RESULT_DIR/cloudstack-management.contents" + +grep -q '/usr/share/cloudstack-common/' "$RESULT_DIR/cloudstack-common.contents" +grep -q '/lib/systemd/system/cloudstack-agent.service' "$RESULT_DIR/cloudstack-agent.contents" +grep -q '/lib/systemd/system/cloudstack-management.service' "$RESULT_DIR/cloudstack-management.contents" + +( + cd "$RESULT_DIR" + sha256sum ./*.deb > SHA256SUMS + ( + cd marvin + sha256sum "$(basename "${marvin_archives[0]}")" > SHA256SUMS + ) + ls -lh +) + +{ + printf 'PACKAGE\tVERSION\tARCHITECTURE\tFILE\n' + for package_name in "${required_packages[@]}"; do + package_file="${package_files[$package_name]}" + printf '%s\t%s\t%s\t%s\n' \ + "$(dpkg-deb -f "$package_file" Package)" \ + "$(dpkg-deb -f "$package_file" Version)" \ + "$(dpkg-deb -f "$package_file" Architecture)" \ + "$(basename "$package_file")" + done +} > "$RESULT_DIR/package-manifest.tsv" + +{ + printf 'ARTIFACT\tFILE\tSHA256\n' + printf 'marvin\t%s\t%s\n' \ + "$(basename "${marvin_archives[0]}")" \ + "$(sha256sum "${marvin_archives[0]}" | awk '{print $1}')" + printf 'commands_xml\tcommands.xml\t%s\n' \ + "$(sha256sum "$commands_xml" | awk '{print $1}')" +} > "$RESULT_DIR/marvin/manifest.tsv" diff --git a/private-cicd/scripts/check-build-prereqs.sh b/private-cicd/scripts/check-build-prereqs.sh new file mode 100755 index 000000000000..8dc8258e78b4 --- /dev/null +++ b/private-cicd/scripts/check-build-prereqs.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Fail fast if the builder environment cannot run Maven, Debian packaging, +# Git/SSH, inventory YAML rendering, or vCenter revert. +# Checks required commands and major versions of Java, Node, and npm. +# Dependencies: +# Commands: git, java, mvn, node, npm, python3, dpkg-buildpackage, dpkg-deb, +# sha256sum, ssh, ssh-keygen, sshpass, scp, ssh-keyscan. python3 must import setuptools, yaml, and +# pyVim (pyvmomi). EXPECTED_JAVA_MAJOR (default 17), EXPECTED_NODE_MAJOR (16), +# EXPECTED_NPM_MAJOR (8). +# No CloudStack source tree and no other private-cicd scripts. + +set -euo pipefail + +EXPECTED_JAVA_MAJOR="${EXPECTED_JAVA_MAJOR:-17}" +EXPECTED_NODE_MAJOR="${EXPECTED_NODE_MAJOR:-16}" +EXPECTED_NPM_MAJOR="${EXPECTED_NPM_MAJOR:-8}" + +required_commands=( + dpkg-buildpackage + dpkg-deb + git + java + mvn + node + npm + python3 + scp + sha256sum + ssh + ssh-keygen + ssh-keyscan + sshpass +) + +for command_name in "${required_commands[@]}"; do + if ! command -v "$command_name" >/dev/null 2>&1; then + echo "Required build command is missing: $command_name" >&2 + exit 1 + fi +done + +if ! python3 -c 'import setuptools' >/dev/null 2>&1; then + echo "Python setuptools is required to build the Marvin archive" >&2 + echo "Install python3-setuptools before running the Debian package build" >&2 + exit 1 +fi +if ! python3 -c 'import yaml' >/dev/null 2>&1; then + echo "PyYAML is required to render Phase 2 inventory configuration" >&2 + exit 1 +fi +if ! python3 -c 'import pyVim' >/dev/null 2>&1; then + echo "pyvmomi (pyVim) is required for vCenter snapshot revert" >&2 + exit 1 +fi + +java_version="$(java -version 2>&1 | awk -F'[".]' '/version/ {print $2; exit}')" +node_version="$(node --version | sed 's/^v//')" +npm_version="$(npm --version)" + +if [[ "${java_version%%.*}" != "$EXPECTED_JAVA_MAJOR" ]]; then + echo "Java $EXPECTED_JAVA_MAJOR is required; found Java $java_version" >&2 + exit 1 +fi +if [[ "${node_version%%.*}" != "$EXPECTED_NODE_MAJOR" ]]; then + echo "Node $EXPECTED_NODE_MAJOR.x is required; found Node $node_version" >&2 + exit 1 +fi +if [[ "${npm_version%%.*}" != "$EXPECTED_NPM_MAJOR" ]]; then + echo "npm $EXPECTED_NPM_MAJOR.x is required; found npm $npm_version" >&2 + exit 1 +fi + +echo "Java: $(java -version 2>&1 | awk 'NR == 1 {print}')" +echo "Maven: $(mvn --version | awk 'NR == 1 {print}')" +echo "Node: $node_version" +echo "npm: $npm_version" +echo "dpkg-buildpackage: $(dpkg-buildpackage --version | awk 'NR == 1 {print}')" +echo "Python setuptools: $(python3 -c 'import setuptools; print(setuptools.__version__)')" +echo "PyYAML: $(python3 -c 'import yaml; print(getattr(yaml, "__version__", "present"))')" +echo "pyvmomi: $(python3 -c 'import pyVim; print("present")')" diff --git a/private-cicd/scripts/check-phase2-health.sh b/private-cicd/scripts/check-phase2-health.sh new file mode 100755 index 000000000000..4496060f6420 --- /dev/null +++ b/private-cicd/scripts/check-phase2-health.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Verify the VM is ready for Marvin: systemd units, dpkg versions vs manifest, +# /dev/kvm, bridge, NFS exports, ONTAP plugin, cloud DB, integration API +# on 8096, ONTAP HTTPS :443, and SVM protocol/LIF prerequisites. +# Dependencies: +# Run after configure-cloudstack-vm.sh. python3, curl, mysql, virsh, dpkg-query. +# MANIFEST_FILE — package-manifest.tsv from build-debs.sh. +# ONTAP_CONFIG — JSON from render-phase2-config.py. +# BRIDGE (default cloudbr0), HEALTH_TIMEOUT_SECONDS (default 900). +# check_ontap_prereqs.py — prefer the copy next to this script (trusted CI +# checkout); fall back to the file beside ontap.cfg. +# Called by run-phase2-vm.sh. + +set -euo pipefail + +MANIFEST_FILE="${MANIFEST_FILE:?MANIFEST_FILE is required}" +ONTAP_CONFIG="${ONTAP_CONFIG:?ONTAP_CONFIG is required}" +BRIDGE="${BRIDGE:-cloudbr0}" +HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-900}" + +required_capabilities=(mysql libvirtd iscsid nfs-kernel-server cloudstack-agent cloudstack-management) +for service_name in "${required_capabilities[@]}"; do + if ! systemctl is-active --quiet "$service_name"; then + systemctl status "$service_name" --no-pager || true + echo "Required service is not active: $service_name" >&2 + exit 1 + fi +done + +while IFS=$'\t' read -r package_name expected_version architecture file_name; do + if [[ "$package_name" == "PACKAGE" ]]; then + continue + fi + actual_version="$(dpkg-query -W -f='${Version}' "$package_name")" + if [[ "$actual_version" != "$expected_version" ]]; then + echo "$package_name version mismatch: expected $expected_version, found $actual_version" >&2 + exit 1 + fi + echo "$package_name $actual_version $architecture $file_name" +done < "$MANIFEST_FILE" + +test -c /dev/kvm +ip link show "$BRIDGE" >/dev/null +management_ip="$( + python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["mgtSvr"][0]["mgtSvrIp"])' \ + "$ONTAP_CONFIG" +)" +ip -4 address show | grep -Fq "$management_ip/" +virsh -c qemu:///system list --all >/dev/null +grep -q '^InitiatorName=iqn\.' /etc/iscsi/initiatorname.iscsi +exportfs -v | grep -q '/export/primary' +exportfs -v | grep -q '/export/secondary' + +# The plugin is shaded into the management uber-jar, not shipped standalone. +if ! python3 - <<'PY' +import glob +import sys +import zipfile + +resource = "META-INF/cloudstack/storage-volume-ontap/module.properties" +for jar in glob.glob("/usr/share/cloudstack-management/lib/cloudstack-*.jar"): + with zipfile.ZipFile(jar) as archive: + if resource in archive.namelist(): + print("ONTAP plugin found in", jar) + sys.exit(0) +sys.exit(1) +PY +then + echo "The NetApp ONTAP plugin is missing from cloudstack-management" >&2 + exit 1 +fi + +mapfile -t db_credentials < <( + python3 - "$ONTAP_CONFIG" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as stream: + config = json.load(stream)["dbSvr"] +print(config["user"]) +print(config["passwd"]) +PY +) +mysql "--user=${db_credentials[0]}" "--password=${db_credentials[1]}" \ + --batch --skip-column-names cloud -e 'SELECT 1' | grep -Fxq 1 + +deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) +until curl --fail --silent --show-error \ + 'http://127.0.0.1:8096/client/api?command=listCapabilities&response=json' \ + >/dev/null; do + if (( SECONDS >= deadline )); then + echo "CloudStack integration API did not become ready on port 8096" >&2 + exit 1 + fi + sleep 15 +done + +python3 - "$ONTAP_CONFIG" <<'PY' +import json +import socket +import sys + +with open(sys.argv[1], encoding="utf-8") as stream: + host = json.load(stream)["ontap"]["storageIP"] +with socket.create_connection((host, 443), timeout=10): + pass +print("ONTAP HTTPS reachable:", host) +PY + +prereq_script="$(dirname "${BASH_SOURCE[0]}")/check_ontap_prereqs.py" +if [[ ! -f "$prereq_script" ]]; then + prereq_script="$(dirname "$ONTAP_CONFIG")/check_ontap_prereqs.py" +fi +if [[ ! -f "$prereq_script" ]]; then + echo "check_ontap_prereqs.py was not found next to this script or ontap.cfg" >&2 + exit 1 +fi +python3 "$prereq_script" "$ONTAP_CONFIG" both diff --git a/private-cicd/scripts/collect-phase2-logs.sh b/private-cicd/scripts/collect-phase2-logs.sh new file mode 100755 index 000000000000..98462767ccf1 --- /dev/null +++ b/private-cicd/scripts/collect-phase2-logs.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Capture dpkg, systemd, journal, virsh, network, NFS, disk, and CloudStack / +# Marvin log trees into OUTPUT_DIR/runtime-logs. Best-effort (does not fail +# the job if a capture command fails). +# Dependencies: +# OUTPUT_DIR (required). Commands: dpkg-query, systemctl, journalctl, virsh, +# ip, iptables-save, exportfs, df, free. Run on the target VM. +# Called from run-phase2-vm.sh EXIT trap. Pair with redact-phase2-results.py. + +set -u + +OUTPUT_DIR="${OUTPUT_DIR:?OUTPUT_DIR is required}" +logs_dir="$OUTPUT_DIR/runtime-logs" +mkdir -p "$logs_dir" + +capture() { + local output="$1" + shift + "$@" > "$logs_dir/$output" 2>&1 || true +} + +capture dpkg-cloudstack.txt dpkg-query -W 'cloudstack-*' +capture systemctl-failed.txt systemctl --failed --no-pager +capture systemctl-cloudstack-management.txt systemctl status cloudstack-management --no-pager +capture systemctl-cloudstack-agent.txt systemctl status cloudstack-agent --no-pager +capture systemctl-libvirtd.txt systemctl status libvirtd --no-pager +capture systemctl-mysql.txt systemctl status mysql --no-pager +capture journal-cloudstack-management.txt journalctl -u cloudstack-management --no-pager +capture journal-cloudstack-agent.txt journalctl -u cloudstack-agent --no-pager +capture journal-libvirtd.txt journalctl -u libvirtd --no-pager +capture journal-mysql.txt journalctl -u mysql --no-pager +capture virsh.txt virsh -c qemu:///system list --all +capture ip-address.txt ip address show +capture ip-route.txt ip route show +capture iptables.txt iptables-save +capture exports.txt exportfs -v +capture disk-usage.txt df -h +capture memory.txt free -h + +for source_dir in \ + /var/log/cloudstack/management \ + /var/log/cloudstack/agent \ + /var/log/mysql \ + /tmp/MarvinLogs; do + if [[ -d "$source_dir" ]]; then + target_name="$(echo "$source_dir" | sed 's|^/||; s|/|-|g')" + cp -a "$source_dir" "$logs_dir/$target_name" 2>/dev/null || true + fi +done + +find "$OUTPUT_DIR" -type f -exec chmod 0644 {} + diff --git a/private-cicd/scripts/configure-cloudstack-vm.sh b/private-cicd/scripts/configure-cloudstack-vm.sh new file mode 100755 index 000000000000..24d9746568c8 --- /dev/null +++ b/private-cicd/scripts/configure-cloudstack-vm.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Follow the Apache Quick Installation Guide order: OS and NFS, MySQL with +# QIG tunables, local apt repo, cloudstack-management, setup-databases with +# --force-recreate so leftover snapshot schemas are not reused, +# cloudstack-setup-management --no-start (4.23 seeds the KVM system-VM +# template without starting the JVM), then cloudstack-agent. Set integration +# API port 8096 before the first management start. Management starts once at +# the end so the 4.0.0-to-4.23 schema upgrade is not interrupted. +# Package installs go through an apt wrapper: apt-daily and unattended-upgrades +# are masked up front, each apt call waits on the dpkg lock, and if the lock is +# still held only apt-family holders are killed before dpkg is repaired and the +# call retried once. +# Dependencies: +# Must run as root on the target VM. python3 (stdlib json). +# DEB_DIR — cloudstack-{common,agent,management}_*.deb from build-debs.sh. +# SECRETS_FILE — JSON from render-phase2-config.py (MySQL/cloud DB users). +# ONTAP_CONFIG — JSON from render-phase2-config.py (management IP). +# BRIDGE — KVM bridge name (default cloudbr0). Nested virt (/dev/kvm) expected. +# Apt packages: mysql-server, nfs-kernel-server, open-iscsi, qemu-kvm, +# libvirt-daemon-system, nginx, dpkg-dev, uuid-runtime, etc. +# The local repository under /var/www/html/repos/cloudstack is the only +# CloudStack package source; third-party CloudStack apt lists are removed. +# Called by run-phase2-vm.sh. Does not call other Phase 2 scripts. + +set -euo pipefail + +DEB_DIR="${DEB_DIR:?DEB_DIR is required}" +SECRETS_FILE="${SECRETS_FILE:?SECRETS_FILE is required}" +ONTAP_CONFIG="${ONTAP_CONFIG:?ONTAP_CONFIG is required}" +BRIDGE="${BRIDGE:-cloudbr0}" + +if [[ "$(id -u)" -ne 0 ]]; then + echo "CloudStack package deployment must run as root" >&2 + exit 1 +fi + +json_value() { + python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' \ + "$SECRETS_FILE" "$1" +} + +single_package() { + local package_name="$1" + local matches=("$DEB_DIR/${package_name}_"*.deb) + if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]}" ]]; then + echo "Expected exactly one package for $package_name in $DEB_DIR" >&2 + exit 1 + fi + printf '%s\n' "${matches[0]}" +} + +APT_LOCK_TIMEOUT_SECONDS=300 +# Matches the apt family only, so an unrelated process is never signalled. +apt_process_pattern='unattended-upgrade|apt\.systemd\.daily|apt-get|dpkg|(^| |/)apt( |$)' + +# fuser and lsof are not guaranteed on the VM, so read the lock owners from /proc. +apt_lock_holders() { + local pid_dir pid fd_link target + for pid_dir in /proc/[0-9]*; do + pid="${pid_dir##*/}" + for fd_link in "$pid_dir"/fd/*; do + target="$(readlink -f "$fd_link" 2>/dev/null || true)" + case "$target" in + /var/lib/dpkg/lock-frontend|/var/lib/dpkg/lock|/var/lib/apt/lists/lock|/var/cache/apt/archives/lock) + printf '%s\n' "$pid" + ;; + esac + done + done | sort -u +} + +# Returns 0 when at least one process was signalled. +signal_apt_lock_holders() { + local signal="$1" + local pid cmdline signalled=1 + for pid in $(apt_lock_holders); do + if [[ "$pid" == "$$" ]]; then + continue + fi + cmdline="$(tr '\0' ' ' <"/proc/${pid}/cmdline" 2>/dev/null || true)" + if [[ -z "$cmdline" ]]; then + continue + fi + if [[ ! "$cmdline" =~ $apt_process_pattern ]]; then + echo "PID ${pid} holds an apt lock but is not an apt process, leaving it: ${cmdline}" >&2 + continue + fi + echo "Sending SIG${signal} to apt lock holder PID ${pid}: ${cmdline}" >&2 + kill "-${signal}" "$pid" 2>/dev/null || true + signalled=0 + done + return "$signalled" +} + +free_apt_lock() { + if signal_apt_lock_holders TERM; then + sleep 10 + signal_apt_lock_holders KILL || true + sleep 2 + fi +} + +apt_get() { + local attempt status=0 + for attempt in 1 2; do + set +e + apt-get -o "DPkg::Lock::Timeout=${APT_LOCK_TIMEOUT_SECONDS}" "$@" + status=$? + set -e + if [[ "$status" -eq 0 ]]; then + return 0 + fi + if [[ "$attempt" -eq 2 ]]; then + break + fi + echo "apt-get exited with ${status}; looking for processes holding the dpkg lock" >&2 + free_apt_lock + echo "Repairing dpkg state before retrying apt-get" + dpkg --configure -a || true + apt-get -o "DPkg::Lock::Timeout=${APT_LOCK_TIMEOUT_SECONDS}" -f install -y || true + done + return "$status" +} + +# Schema-upgrade tracing. Each call records the cloud database version, table +# counts, and management-server state as one NDJSON line, so an interrupted +# 4.0.0-to-4.23 upgrade can be reconstructed from the stage log or the VM file. +DB_TRACE_FILE=/var/log/cloudstack-phase2-debug.ndjson +db_trace_log() { + local line + line="$(printf '{"runId":"phase2","checkpoint":"%s","location":"configure-cloudstack-vm.sh","data":%s,"timestamp":%s000}' \ + "$1" "$2" "$(date +%s)")" + echo "PHASE2_DB_TRACE ${line}" + echo "$line" >>"$DB_TRACE_FILE" 2>/dev/null || true +} +db_trace_state() { + local ver rel tables mgmt_active mgmt_pid mgmt_since upgrade_errors dup_errors + ver="$("${mysql_root[@]}" --batch --skip-column-names cloud \ + -e "SELECT IFNULL(GROUP_CONCAT(CONCAT(version,'/',step)),'none') FROM version" 2>/dev/null || echo NA)" + rel="$("${mysql_root[@]}" --batch --skip-column-names \ + -e "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='cloud' AND TABLE_NAME='async_job' AND COLUMN_NAME='related'" 2>/dev/null || echo NA)" + tables="$("${mysql_root[@]}" --batch --skip-column-names \ + -e "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='cloud'" 2>/dev/null || echo NA)" + mgmt_active="$(systemctl is-active cloudstack-management 2>/dev/null || true)" + mgmt_pid="$(systemctl show -p MainPID --value cloudstack-management 2>/dev/null || true)" + mgmt_since="$(systemctl show -p ActiveEnterTimestamp --value cloudstack-management 2>/dev/null || true)" + upgrade_errors="$(grep -c 'Unable to upgrade the database' /var/log/cloudstack/management/management-server.log 2>/dev/null || echo 0)" + dup_errors="$(grep -c 'Duplicate column name\|Duplicate key name' /var/log/cloudstack/management/management-server.log 2>/dev/null || echo 0)" + db_trace_log "$1" "$(printf '{"version":"%s","async_job_related":"%s","cloud_tables":"%s","mgmt_active":"%s","mgmt_pid":"%s","mgmt_since":"%s","upgrade_errors":"%s","duplicate_errors":"%s"}' \ + "$ver" "$rel" "$tables" "$mgmt_active" "$mgmt_pid" "$mgmt_since" "$upgrade_errors" "$dup_errors")" +} + +cloud_db_user="$(json_value cloudDbUser)" +cloud_db_password="$(json_value cloudDbPassword)" +mysql_root_user="$(json_value mysqlRootUser)" +mysql_root_password="$(json_value mysqlRootPassword)" + +single_package cloudstack-common >/dev/null +single_package cloudstack-agent >/dev/null +management_deb="$(single_package cloudstack-management)" + +package_version="$(dpkg-deb -f "$management_deb" Version)" +repo_version="$(cut -d. -f1,2 <<<"$package_version")" +repo_dir="/var/www/html/repos/cloudstack/${repo_version}" +management_ip="$( + python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["mgtSvr"][0]["mgtSvrIp"])' \ + "$ONTAP_CONFIG" +)" + +export DEBIAN_FRONTEND=noninteractive + +# Ubuntu runs apt-daily and unattended-upgrades from timers shortly after boot, +# which takes the dpkg lock while Phase 2 is installing. Minimal images may not +# ship every unit, so failures here are not fatal. +for apt_unit in apt-daily.timer apt-daily-upgrade.timer \ + apt-daily.service apt-daily-upgrade.service unattended-upgrades.service; do + echo "Disabling ${apt_unit} for the duration of this deployment" + systemctl disable --now "$apt_unit" >/dev/null 2>&1 || true + systemctl mask "$apt_unit" >/dev/null 2>&1 || true +done + +shopt -s nullglob +for apt_list in /etc/apt/sources.list.d/*cloudstack*.list \ + /etc/apt/sources.list.d/*cloudstack*.list.disabled; do + echo "Removing third-party CloudStack apt source: $apt_list" + rm -f "$apt_list" +done +shopt -u nullglob +apt_get update +apt_get install -y \ + curl \ + dpkg-dev \ + libvirt-daemon-system \ + mysql-server \ + nfs-common \ + nfs-kernel-server \ + nginx \ + open-iscsi \ + python3-pip \ + python3-venv \ + qemu-kvm \ + quota \ + rpcbind \ + uuid-runtime + +mysqld_config=/etc/mysql/mysql.conf.d/mysqld.cnf +set_mysqld_property() { + local key="$1" + local value="$2" + if grep -Eq "^#?${key}[[:space:]]*=" "$mysqld_config"; then + sed -i -E "s|^#?${key}[[:space:]]*=.*|${key}=${value}|" "$mysqld_config" + else + printf '%s=%s\n' "$key" "$value" >> "$mysqld_config" + fi +} +set_mysqld_property innodb_rollback_on_timeout 1 +set_mysqld_property innodb_lock_wait_timeout 600 +set_mysqld_property max_connections 350 +set_mysqld_property log_bin mysql-bin +set_mysqld_property binlog_format ROW + +systemctl enable mysql +systemctl restart mysql + +mysql_root=(mysql "--user=$mysql_root_user") +if [[ -n "$mysql_root_password" ]]; then + mysql_root+=("--password=$mysql_root_password") +fi + +mysql_cloud=(mysql "--user=$cloud_db_user") +if [[ -n "$cloud_db_password" ]]; then + mysql_cloud+=("--password=$cloud_db_password") +fi + +# A freshly installed mysql-server leaves the administrator on socket +# authentication with no password, so apply the configured one before anything +# connects with a password. Skipped when the account already accepts it. +if [[ -n "$mysql_root_password" ]] && ! "${mysql_root[@]}" -e 'SELECT 1' >/dev/null 2>&1; then + if ! mysql "--user=$mysql_root_user" -e 'SELECT 1' >/dev/null 2>&1; then + echo "Cannot reach MySQL as $mysql_root_user over the local socket to set its password" >&2 + exit 1 + fi + # mysql_native_password is absent from MySQL 8.4 and newer. + mysql_auth_plugin=caching_sha2_password + mysql_native_available="$( + mysql "--user=$mysql_root_user" --batch --skip-column-names -e \ + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PLUGINS + WHERE PLUGIN_NAME='mysql_native_password' AND PLUGIN_STATUS='ACTIVE'" + )" + if [[ "$mysql_native_available" == 1 ]]; then + mysql_auth_plugin=mysql_native_password + fi + echo "Setting the MySQL $mysql_root_user password for unattended lab setup" + mysql "--user=$mysql_root_user" -e \ + "ALTER USER '${mysql_root_user}'@'localhost' + IDENTIFIED WITH ${mysql_auth_plugin} BY '${mysql_root_password}'; FLUSH PRIVILEGES;" + "${mysql_root[@]}" -e 'SELECT 1' >/dev/null +fi + +systemctl enable --now iscsid +systemctl enable --now rpcbind +systemctl enable --now nfs-kernel-server + +mkdir -p /export/primary /export/secondary +if ! grep -Fqx '/export/primary *(rw,async,no_root_squash,no_subtree_check)' /etc/exports; then + echo '/export/primary *(rw,async,no_root_squash,no_subtree_check)' >> /etc/exports +fi +if ! grep -Fqx '/export/secondary *(rw,async,no_root_squash,no_subtree_check)' /etc/exports; then + echo '/export/secondary *(rw,async,no_root_squash,no_subtree_check)' >> /etc/exports +fi +exportfs -ra +systemctl restart nfs-kernel-server + +if systemctl is-active --quiet docker 2>/dev/null; then + iptables -P FORWARD ACCEPT + systemctl disable --now docker +fi + +if command -v ufw >/dev/null 2>&1; then + ufw disable +fi + +rm -rf "$repo_dir" +mkdir -p "$repo_dir" +cp "$DEB_DIR"/*.deb "$repo_dir/" +( + cd "$repo_dir" + dpkg-scanpackages . /dev/null > Packages + gzip -9c Packages > Packages.gz +) +chown -R www-data:www-data /var/www/html/repos +chmod -R 755 /var/www/html/repos +systemctl enable --now nginx + +repo_url="http://${management_ip}/repos/cloudstack/${repo_version}" +if ! curl -fsI "${repo_url}/Packages.gz" >/dev/null; then + echo "The local CloudStack apt repository is not reachable at ${repo_url}" >&2 + exit 1 +fi + +printf 'deb [trusted=yes] %s ./\n' "$repo_url" \ + > /etc/apt/sources.list.d/cloudstack.list +apt_get update +apt_get install -y \ + "cloudstack-common=${package_version}" \ + "cloudstack-management=${package_version}" +apt-cache policy cloudstack-management + +# Leftover cloud/cloud_usage from a previous CloudStack on the snapshot has +# vm_template but a version table that does not match the indexes, so 4.23 +# then fails mid-upgrade (Duplicate key name uc_hypervisor) and never binds +# 8096. Always recreate against the packages just installed. Marvin rebuilds +# the zone afterward. +db_trace_state "before setup-databases" +systemctl stop cloudstack-management || true +echo "Creating CloudStack databases (force-recreate if they already exist)" +cloudstack-setup-databases \ + "${cloud_db_user}:${cloud_db_password}@localhost" \ + "--deploy-as=${mysql_root_user}:${mysql_root_password}" \ + --force-recreate +db_trace_state "after setup-databases" +# The 4.0.0 schema has no integration.api.port row, so an UPDATE here matches +# nothing and management boots with the ConfigKey default of 0, which disables +# the listener. The key is not dynamic, so the row has to exist before start. +"${mysql_cloud[@]}" cloud -e \ + "INSERT INTO configuration (category, instance, component, name, value) + VALUES ('Advanced', 'DEFAULT', 'management-server', 'integration.api.port', '8096') + ON DUPLICATE KEY UPDATE value = '8096';" +integration_port="$("${mysql_cloud[@]}" --batch --skip-column-names cloud -e \ + "SELECT value FROM configuration WHERE name = 'integration.api.port';")" +if [[ "$integration_port" != "8096" ]]; then + echo "integration.api.port is '${integration_port}', not 8096; the integration API would never bind" >&2 + exit 1 +fi + +setup_management=(cloudstack-setup-management --no-start) +if cloudstack-setup-management --help 2>&1 | grep -q -- '--systemvm-templates'; then + setup_management+=(--systemvm-templates kvm-x86_64) +else + echo "cloudstack-setup-management has no --systemvm-templates; installing the KVM template before management starts" >&2 + systemctl stop cloudstack-management || true + /usr/share/cloudstack-common/scripts/storage/secondary/cloud-install-sys-tmplt \ + -m /export/secondary \ + -u http://download.cloudstack.org/systemvm/4.22/systemvmtemplate-4.22.0-x86_64-kvm.qcow2.bz2 \ + -h kvm \ + -F +fi +"${setup_management[@]}" +db_trace_state "after cloudstack-setup-management --no-start" + +apt_get install -y "cloudstack-agent=${package_version}" +apt-cache policy cloudstack-agent + +sed -i -E 's|^#?vnc_listen.*|vnc_listen = "0.0.0.0"|' /etc/libvirt/qemu.conf +if ! grep -q '^LIBVIRTD_ARGS=' /etc/default/libvirtd; then + echo 'LIBVIRTD_ARGS="--listen"' >> /etc/default/libvirtd +else + sed -i 's|^LIBVIRTD_ARGS=.*|LIBVIRTD_ARGS="--listen"|' /etc/default/libvirtd +fi +set_libvirt_property() { + local key="$1" + local value="$2" + if grep -Eq "^#?${key}[[:space:]]*=" /etc/libvirt/libvirtd.conf; then + sed -i -E "s|^#?${key}[[:space:]]*=.*|${key} = ${value}|" /etc/libvirt/libvirtd.conf + else + printf '%s = %s\n' "$key" "$value" >> /etc/libvirt/libvirtd.conf + fi +} +set_libvirt_property listen_tls 0 +set_libvirt_property listen_tcp 1 +set_libvirt_property tcp_port '"16509"' +set_libvirt_property mdns_adv 0 +set_libvirt_property auth_tcp '"none"' +systemctl mask \ + libvirtd.socket \ + libvirtd-ro.socket \ + libvirtd-admin.socket \ + libvirtd-tls.socket \ + libvirtd-tcp.socket +systemctl enable --now libvirtd + +agent_properties=/etc/cloudstack/agent/agent.properties +touch "$agent_properties" +set_agent_property() { + local key="$1" + local value="$2" + if grep -q "^${key}=" "$agent_properties"; then + sed -i "s|^${key}=.*|${key}=${value}|" "$agent_properties" + else + printf '%s=%s\n' "$key" "$value" >> "$agent_properties" + fi +} + +if ! grep -q '^guid=.\+' "$agent_properties"; then + set_agent_property guid "$(uuidgen)" +fi +set_agent_property public.network.device "$BRIDGE" +set_agent_property private.network.device "$BRIDGE" + +systemctl daemon-reload +systemctl enable cloudstack-management cloudstack-agent +systemctl restart libvirtd +libvirtd_wait=0 +until systemctl is-active --quiet libvirtd; do + if (( libvirtd_wait >= 30 )); then + echo "libvirtd did not become active after restart" >&2 + systemctl status libvirtd --no-pager || true + exit 1 + fi + sleep 1 + libvirtd_wait=$((libvirtd_wait + 1)) +done +db_trace_state "before first start of cloudstack-management" +systemctl start cloudstack-management +systemctl restart cloudstack-agent +db_trace_state "after first start of cloudstack-management" diff --git a/private-cicd/scripts/generate-presubmit-report.py b/private-cicd/scripts/generate-presubmit-report.py new file mode 100644 index 000000000000..3067cfa26724 --- /dev/null +++ b/private-cicd/scripts/generate-presubmit-report.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate browsable HTML artifacts and the final presubmit mail body.""" + +import argparse +import html +import json +import re +import shutil +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + + +STAGES = [ + "Validate source request", + "Check eligibility", + "Abort superseded run", + "Checkout CI scripts", + "Start PR reporting", + "Validate builder", + "Checkout source", + "Build and unit tests", + "Build Debian packages", + "Deploy and run ONTAP integration", +] + +TERMINAL_STATUSES = {"SUCCESS", "FAILURE", "ABORTED", "SKIPPED"} +TEXT_SUFFIXES = {".log", ".txt", ".tsv", ".properties"} +STATUS_ORDER = {"FAIL": 0, "EXCEPTION": 0, "FAILED": 0, "SKIP": 1, "PASS": 2} +ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--results", required=True, type=Path) + parser.add_argument("--stage-events", required=True, type=Path) + parser.add_argument("--build-url", required=True) + parser.add_argument("--result", required=True) + parser.add_argument("--source", required=True) + parser.add_argument("--source-url", default="") + parser.add_argument("--title", default="") + parser.add_argument("--source-sha", required=True) + parser.add_argument("--diff-number", default="") + parser.add_argument("--vm", default="") + parser.add_argument("--started-at", required=True) + parser.add_argument("--finished-at", required=True) + parser.add_argument("--debian-log", type=Path) + return parser.parse_args() + + +def parse_time(value): + if not value: + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def format_duration(start, finish): + if not start or not finish: + return "—" + seconds = max(0, int((finish - start).total_seconds())) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + if hours: + return f"{hours}h {minutes}m {seconds}s" + if minutes: + return f"{minutes}m {seconds}s" + return f"{seconds}s" + + +def slug(value): + return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "test" + + +def artifact_url(build_url, relative_path, fragment=""): + encoded = "/".join(quote(part) for part in Path(relative_path).parts) + url = f"{build_url.rstrip('/')}/artifact/{encoded}" + return f"{url}#{quote(fragment)}" if fragment else url + + +def status_style(status): + status = status.upper() + if status in {"FAILURE", "FAIL", "FAILED", "EXCEPTION", "ABORTED"}: + return "color:#b42318;font-weight:bold" + if status in {"SUCCESS", "PASS"}: + return "color:#067647;font-weight:bold" + if status in {"SKIP", "SKIPPED", "NOT RUN"}: + return "color:#475467;font-weight:bold" + return "font-weight:bold" + + +def display_test_status(status): + status = str(status or "FAIL").upper() + return "PASS" if status == "SUCCESS" else status + + +def html_document(title, body): + return f"""<!doctype html> +<html> +<head><meta charset="utf-8"><title>{html.escape(title)} + +{body} + + +""" + + +def log_to_html(source, destination): + lines = source.read_text(encoding="utf-8", errors="replace").splitlines() + rendered = [] + test_pattern = re.compile(r"TestName:\s+(\S+)\s+\|\s+Status\s+:\s+(\S+)") + anchor_positions = {} + seen_anchors = set() + for index, original_line in enumerate(lines): + line = ANSI_ESCAPE.sub("", original_line) + match = test_pattern.search(line) + if not match: + continue + anchor_name = f"test-{slug(match.group(1))}" + if anchor_name in seen_anchors: + continue + seen_anchors.add(anchor_name) + anchor_positions.setdefault(max(0, index - 40), []).append(anchor_name) + + for index, original_line in enumerate(lines): + line = ANSI_ESCAPE.sub("", original_line) + anchor = "".join( + f'' + for anchor_name in anchor_positions.get(index, [])) + escaped = html.escape(line) + error_pattern = r"\b(FAIL(?:ED)?|FAILURE|EXCEPTION|ERROR|Traceback)\b" + if re.search(error_pattern, line, re.I): + escaped = ( + f'' + f"{escaped}") + elif re.search(r"\b(SUCCESS|PASS)\b", line): + escaped = f'{escaped}' + rendered.append(f"{anchor}{escaped}") + body = ( + f"

{html.escape(source.name)}

" + f'
'
+        + "\n".join(rendered)
+        + "
" + ) + destination.write_text( + html_document(source.name, body), encoding="utf-8") + + +def generate_log_artifacts(results, debian_log): + if debian_log and debian_log.is_file(): + copied_log = results / "logs" / "deb-build.log" + copied_log.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(debian_log, copied_log) + + candidates = [ + results / "unit-tests" / "maven.log", + results / "logs" / "deb-build.log", + results / "phase2" / "configure-cloudstack.log", + results / "phase2" / "health-check.log", + ] + marvin = results / "phase2" / "marvin" + candidates.extend(marvin.glob("*.log")) + candidates.extend( + path for path in (marvin / "ontap-results").rglob("*") + if path.suffix.lower() in TEXT_SUFFIXES) + for source in sorted(set(candidates)): + if not source.is_file(): + continue + log_to_html(source, Path(f"{source}.html")) + + +def parse_stage_events(stage_events, results): + destination = results / "stage-events.tsv" + if stage_events.is_file(): + shutil.copyfile(stage_events, destination) + events = {stage: [] for stage in STAGES} + if not destination.is_file(): + return events + for line in destination.read_text(encoding="utf-8").splitlines(): + fields = line.split("\t") + if len(fields) != 3 or fields[1] not in events: + continue + try: + timestamp = parse_time(fields[0]) + except ValueError: + continue + events[fields[1]].append((timestamp, fields[2].upper())) + return events + + +def stage_log(stage, results): + mapping = { + "Build and unit tests": results / "unit-tests" / "maven.log.html", + "Build Debian packages": results / "logs" / "deb-build.log.html", + } + if stage == "Deploy and run ONTAP integration": + marvin = results / "phase2" / "marvin" + for summary_path in sorted(marvin.rglob("summary.json")): + try: + payload = json.loads(summary_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if payload.get("totals", {}).get("failed", 0): + protocol = payload.get("run", {}).get( + "protocol", summary_path.parent.name) + failed_log = marvin / f"{protocol}.log.html" + if failed_log.is_file(): + return failed_log + for protocol in ("iscsi", "nfs3", "setup_zone"): + protocol_log = marvin / f"{protocol}.log.html" + if protocol_log.is_file(): + return protocol_log + path = mapping.get(stage) + return path if path and path.is_file() else None + + +def stage_rows(events, results, build_url): + rows = [] + failed_stage = "" + for stage in STAGES: + stage_events = events[stage] + started = next( + (time for time, status in stage_events if status == "STARTED"), + None) + waiting = next( + (time for time, status in stage_events + if status == "WAITING_FOR_VM"), + None) + terminal = next( + ((time, status) for time, status in reversed(stage_events) + if status in TERMINAL_STATUSES), + None) + finished, status = terminal if terminal else (None, "NOT RUN") + if status == "FAILURE" and not failed_stage: + failed_stage = stage + queue_duration = format_duration(waiting, started) if waiting else "—" + relative_log = stage_log(stage, results) + if relative_log: + relative_log = relative_log.relative_to(results.parent) + log_link = ( + f'' + "HTML log") + else: + console_url = build_url.rstrip("/") + "/console" + log_link = ( + f'' + "Console") + started_text = ( + started.strftime("%Y-%m-%d %H:%M:%S UTC") if started else "—") + rows.append( + "" + f"{html.escape(stage)}" + f'{html.escape(status)}' + f"{html.escape(started_text)}" + f"{html.escape(format_duration(started, finished))}" + f"{html.escape(queue_duration)}" + f"{log_link}" + "") + return "\n".join(rows), failed_stage + + +def test_log_for_summary(summary_path, test): + suite_dir = summary_path.parent / "suites" / test.get("tag", "") + for name in ("stdout.log.html", "results.txt.html"): + suite_log = suite_dir / name + if suite_log.is_file(): + return suite_log + protocol = summary_path.parent.name + run = test.get("_run", {}) + protocol = run.get("protocol", protocol) + fallback = summary_path.parents[2] / f"{protocol}.log.html" + return fallback if fallback.is_file() else None + + +def load_tests(results): + tests = [] + summaries = sorted( + (results / "phase2" / "marvin").rglob("summary.json")) + for summary_path in summaries: + try: + payload = json.loads(summary_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(payload.get("tests"), list): + continue + run = payload.get("run", {}) + protocol = run.get("protocol", summary_path.parent.name) + summary_tsv = summary_path.with_name("summary.tsv") + source_rows = [] + if summary_tsv.is_file(): + for line in summary_tsv.read_text(encoding="utf-8").splitlines(): + fields = line.split("\t", 4) + if len(fields) >= 4: + source_rows.append({ + "tag": fields[0], + "label": fields[1], + "name": fields[2], + "status": fields[3], + "detail": fields[4] if len(fields) == 5 else "", + }) + else: + source_rows = payload["tests"] + for test in source_rows: + row = dict(test) + row["_run"] = run + row["_protocol"] = protocol + row["_summary"] = summary_path + tests.append(row) + return sorted( + tests, + key=lambda item: ( + STATUS_ORDER.get(display_test_status(item.get("status")), 0), + item.get("_protocol", ""), + item.get("tag", ""), + item.get("name", ""), + )) + + +def test_rows(tests, results, build_url): + rows = [] + for test in tests: + status = display_test_status(test.get("status")) + log_path = test_log_for_summary(test["_summary"], test) + if log_path: + relative = log_path.relative_to(results.parent) + link = artifact_url( + build_url, relative, f"test-{slug(test.get('name', 'test'))}") + log_cell = f'HTML log' + else: + log_cell = "—" + suite = str(test.get("label") or test.get("tag") or "") + rows.append( + "" + f"{html.escape(str(test.get('_protocol', '')))}" + f"{html.escape(suite)}" + f"{html.escape(str(test.get('name', '')))}" + f'{html.escape(status)}' + f"{log_cell}" + "") + return "\n".join(rows) + + +def table(headers, rows): + cells = "".join( + f'{html.escape(header)}' + for header in headers) + styled_rows = rows.replace( + "", '') + styled_rows = re.sub( + r'', + r'', + styled_rows) + return ( + '' + f"{cells}{styled_rows}
") + + +def main(): + args = parse_args() + args.results.mkdir(parents=True, exist_ok=True) + generate_log_artifacts(args.results, args.debian_log) + events = parse_stage_events(args.stage_events, args.results) + stages_html, failed_stage = stage_rows( + events, args.results, args.build_url) + tests = load_tests(args.results) + tests_html = test_rows(tests, args.results, args.build_url) + + started = parse_time(args.started_at) + finished = parse_time(args.finished_at) + report_url = artifact_url( + args.build_url, + args.results.relative_to(args.results.parent) / "report.html") + result_style = status_style(args.result) + source = html.escape(args.source) + if args.source_url.startswith("https://github.com/"): + source = ( + f'{source}') + title_row = ( + f"Title" + f"{html.escape(args.title)}" + if args.title else "") + diff_row = ( + f"Diff" + f"#{html.escape(args.diff_number)}" + if args.diff_number else "") + header = f""" +

CloudStack ONTAP presubmit

+

{html.escape(args.result)}

+ + + +{title_row} +{diff_row} + + + + + + + + + + +
Source{source}
Commit{html.escape(args.source_sha)}
VM{html.escape(args.vm or 'not assigned')}
Started{html.escape(args.started_at)}
Finished{html.escape(args.finished_at)}
Total duration{html.escape(format_duration(started, finished))}
+

Jenkins build · +Console · +HTML report

+""" + failure = ( + f'

' + f'Failed stage: {html.escape(failed_stage)}

' + if failed_stage else "") + stage_table = table( + ["Stage", "Status", "Started", "Duration", "VM wait", "Details"], + stages_html) + test_section = ( + "

ONTAP tests

" + "

Failures are listed first. Select a log link to jump to the " + "test.

" + + table(["Protocol", "Suite", "Test", "Status", "Details"], tests_html) + if tests else + "

ONTAP tests

No Marvin test summary was available.

") + body = ( + header + failure + "

Pipeline stages

" + + stage_table + test_section) + document = html_document("CloudStack ONTAP presubmit report", body) + (args.results / "report.html").write_text(document, encoding="utf-8") + (args.results / "email-body.html").write_text(body, encoding="utf-8") + (args.results / "failed-stage.txt").write_text( + failed_stage, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/private-cicd/scripts/github-check-run.sh b/private-cicd/scripts/github-check-run.sh new file mode 100755 index 000000000000..e00821cede8c --- /dev/null +++ b/private-cicd/scripts/github-check-run.sh @@ -0,0 +1,377 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# GitHub reporting for one presubmit run: create prints the new +# cloudstack-ontap-presubmit check-run ID for the head SHA, complete PATCHes +# that run with a conclusion, commit-email prints the head commit author +# address used for presubmit mail, and diagnose probes the App's token type, +# repository access and Checks permissions to explain a refused check run. +# Dependencies: +# Commands: curl, python3 (stdlib), mktemp. +# GITHUB_ACCESS_TOKEN, GITHUB_REPOSITORY — required by every subcommand; the +# token is passed to curl through a 0600 config file so it stays out of the +# process list. +# GITHUB_HEAD_SHA — create, commit-email and diagnose. BUILD_URL — create and +# complete. GITHUB_CHECK_RUN_ID, GITHUB_CHECK_CONCLUSION — complete. +# Optional: PR_ID and BUILD_TAG label the check run. +# Called by the Jenkinsfile. Does not call other private-cicd scripts. + +set -euo pipefail + +readonly CHECK_NAME='cloudstack-ontap-presubmit' +readonly API_ROOT='https://api.github.com' + +require_env() { + local name + for name in "$@"; do + if [[ -z "${!name:-}" ]]; then + echo "Required environment variable ${name} is empty." >&2 + exit 2 + fi + done +} + +github_api() { + local method="$1" + local endpoint="$2" + local payload_file="${3:-}" + local auth_config status + auth_config="$(mktemp)" + chmod 600 "$auth_config" + printf 'header = "Authorization: Bearer %s"\n' \ + "$GITHUB_ACCESS_TOKEN" > "$auth_config" + local args=( + --fail-with-body + --silent + --show-error + --request "$method" + --config "$auth_config" + --header 'Accept: application/vnd.github+json' + --header 'X-GitHub-Api-Version: 2022-11-28' + ) + if [[ -n "$payload_file" ]]; then + args+=(--header 'Content-Type: application/json' --data-binary "@${payload_file}") + fi + set +e + curl "${args[@]}" "${API_ROOT}${endpoint}" + status=$? + set -e + rm -f "$auth_config" + return "$status" +} + +# Prints the HTTP status code and writes the response body to $3. Unlike +# github_api this never fails, so a caller can branch on the status code. +github_probe() { + local method="$1" + local endpoint="$2" + local body_file="$3" + local payload_file="${4:-}" + local auth_config + auth_config="$(mktemp)" + chmod 600 "$auth_config" + printf 'header = "Authorization: Bearer %s"\n' \ + "$GITHUB_ACCESS_TOKEN" > "$auth_config" + local args=( + --silent + --show-error + --request "$method" + --config "$auth_config" + --header 'Accept: application/vnd.github+json' + --header 'X-GitHub-Api-Version: 2022-11-28' + --output "$body_file" + --write-out '%{http_code}' + ) + if [[ -n "$payload_file" ]]; then + args+=(--header 'Content-Type: application/json' --data-binary "@${payload_file}") + fi + curl "${args[@]}" "${API_ROOT}${endpoint}" || true + rm -f "$auth_config" +} + +# GitHub explains every refusal in the response body. Printing it is the +# difference between "HTTP 403" and knowing which permission is missing. +print_github_error() { + local body_file="$1" + [[ -s "$body_file" ]] || return 0 + python3 - "$body_file" >&2 <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + try: + payload = json.load(handle) + except json.JSONDecodeError: + handle.seek(0) + print(f" body: {handle.read().strip()}") + else: + print(f" message: {payload.get('message', '(none)')}") + for item in payload.get("errors") or []: + print(f" error: {item}") + if payload.get("documentation_url"): + print(f" docs: {payload['documentation_url']}") +PY +} + +create_check() { + require_env GITHUB_ACCESS_TOKEN GITHUB_REPOSITORY GITHUB_HEAD_SHA BUILD_URL + local payload response + payload="$(mktemp)" + response="$(mktemp)" + trap "rm -f '$payload' '$response'" EXIT + + python3 - "$payload" <<'PY' +import json +import os +import sys + +payload = { + "name": "cloudstack-ontap-presubmit", + "head_sha": os.environ["GITHUB_HEAD_SHA"], + "status": "in_progress", + "details_url": os.environ["BUILD_URL"], + "external_id": os.environ.get("BUILD_TAG", ""), + "output": { + "title": f"CloudStack ONTAP presubmit for PR-{os.environ.get('PR_ID', 'unknown')}", + "summary": ( + f"Jenkins build started for {os.environ['GITHUB_HEAD_SHA'][:12]}. " + f"Details: {os.environ['BUILD_URL']}" + ), + }, +} +with open(sys.argv[1], "w", encoding="utf-8") as output: + json.dump(payload, output) +PY + + github_api POST "/repos/${GITHUB_REPOSITORY}/check-runs" "$payload" > "$response" || { + local status=$? + echo "GitHub refused to create the check run (curl exit ${status}):" >&2 + print_github_error "$response" + echo " Re-run this job with SOURCE_MODE=diagnose to identify the cause." >&2 + return "$status" + } + python3 - "$response" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as response: + print(json.load(response)["id"]) +PY +} + +complete_check() { + require_env GITHUB_ACCESS_TOKEN GITHUB_REPOSITORY GITHUB_CHECK_RUN_ID \ + GITHUB_CHECK_CONCLUSION BUILD_URL + local payload response + payload="$(mktemp)" + response="$(mktemp)" + trap "rm -f '$payload' '$response'" EXIT + + python3 - "$payload" <<'PY' +import json +import os +import sys + +conclusion = os.environ["GITHUB_CHECK_CONCLUSION"] +payload = { + "name": "cloudstack-ontap-presubmit", + "status": "completed", + "conclusion": conclusion, + "details_url": os.environ["BUILD_URL"], + "output": { + "title": f"CloudStack ONTAP presubmit: {conclusion}", + "summary": ( + f"Jenkins build completed with conclusion {conclusion}. " + f"Details and logs: {os.environ['BUILD_URL']}" + ), + }, +} +with open(sys.argv[1], "w", encoding="utf-8") as output: + json.dump(payload, output) +PY + + if ! github_api PATCH \ + "/repos/${GITHUB_REPOSITORY}/check-runs/${GITHUB_CHECK_RUN_ID}" \ + "$payload" > "$response"; then + echo "GitHub refused to complete the check run:" >&2 + print_github_error "$response" + return 1 + fi +} + +# Read-only probes plus one write probe, ordered so that the first failure +# names the cause. Reads succeed for almost any token, so a create-only 403 +# cannot be diagnosed from the worker log alone. +diagnose_app() { + require_env GITHUB_ACCESS_TOKEN GITHUB_REPOSITORY GITHUB_HEAD_SHA + local body payload status verdict archived + body="$(mktemp)" + payload="$(mktemp)" + trap "rm -f '$body' '$payload'" EXIT + verdict='' + + echo "Repository : ${GITHUB_REPOSITORY}" + echo "Head SHA : ${GITHUB_HEAD_SHA}" + if [[ "${GITHUB_APP:-}" =~ ^[0-9]+$ ]]; then + echo 'Credential : username is numeric, consistent with a GitHub App ID' + else + echo 'Credential : username is not numeric; a Jenkins GitHub App' + echo ' credential supplies the numeric App ID as username' + fi + echo + + status="$(github_probe GET '/installation/repositories?per_page=1' "$body")" + echo "[1] GET /installation/repositories -> HTTP ${status}" + case "$status" in + 200) + echo ' Token is a GitHub App installation token.' + ;; + 401) + print_github_error "$body" + verdict='GitHub rejected the token itself. Re-check the App ID and private key in the Jenkins credential; the key must be unencrypted PKCS#8.' + ;; + 403) + print_github_error "$body" + echo ' Only GitHub App installation tokens may create check runs.' + verdict='The credential is not issuing a GitHub App installation token. Set its Kind to GitHub App.' + ;; + *) + print_github_error "$body" + ;; + esac + + status="$(github_probe GET "/repos/${GITHUB_REPOSITORY}" "$body")" + echo "[2] GET /repos/${GITHUB_REPOSITORY} -> HTTP ${status}" + if [[ "$status" == '200' ]]; then + archived="$(python3 -c \ + 'import json,sys; print(json.load(open(sys.argv[1])).get("archived"))' \ + "$body")" + echo " archived=${archived}" + if [[ "$archived" == 'True' ]]; then + verdict="${verdict:-The repository is archived, so GitHub rejects every write including check runs.}" + fi + else + print_github_error "$body" + fi + + status="$(github_probe GET \ + "/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_HEAD_SHA}/check-runs?per_page=1" \ + "$body")" + echo "[3] GET commits//check-runs (needs Checks: read) -> HTTP ${status}" + case "$status" in + 200) + echo ' Checks: read is granted to this installation.' + ;; + 403) + print_github_error "$body" + verdict="${verdict:-The installation holds no Checks permission. Approve the updated permission request on the ${GITHUB_REPOSITORY} installation.}" + ;; + 404) + print_github_error "$body" + verdict="${verdict:-The App is not installed on ${GITHUB_REPOSITORY}, or the repository name is wrong.}" + ;; + *) + print_github_error "$body" + ;; + esac + + python3 - "$payload" <<'PY' +import json +import os +import sys + +payload = { + "name": "cloudstack-ontap-presubmit-diagnostic", + "head_sha": os.environ["GITHUB_HEAD_SHA"], + "status": "completed", + "conclusion": "neutral", + "output": { + "title": "GitHub App Checks write probe", + "summary": ( + "Created by SOURCE_MODE=diagnose to verify Checks write access. " + "This check is informational and never gates a merge." + ), + }, +} +with open(sys.argv[1], "w", encoding="utf-8") as output: + json.dump(payload, output) +PY + + status="$(github_probe POST "/repos/${GITHUB_REPOSITORY}/check-runs" \ + "$body" "$payload")" + echo "[4] POST /repos/${GITHUB_REPOSITORY}/check-runs -> HTTP ${status}" + case "$status" in + 200|201) + echo ' Checks: write works.' + echo " A neutral cloudstack-ontap-presubmit-diagnostic check now" + echo " exists on ${GITHUB_HEAD_SHA:0:12}." + verdict="${verdict:-Checks write succeeded, so the presubmit check will publish on the next worker run.}" + ;; + 403) + print_github_error "$body" + verdict="${verdict:-Checks write is denied while reads succeed. Approve the updated Checks permission on the installation.}" + ;; + 422) + print_github_error "$body" + verdict="${verdict:-GitHub rejected the head SHA because it is absent from ${GITHUB_REPOSITORY}, which happens for pull requests opened from a fork.}" + ;; + *) + print_github_error "$body" + ;; + esac + + echo + echo "Verdict: ${verdict:-No single cause identified; read the probe output above.}" +} + +commit_email() { + require_env GITHUB_ACCESS_TOKEN GITHUB_REPOSITORY GITHUB_HEAD_SHA + local response + response="$(mktemp)" + trap "rm -f '$response'" EXIT + github_api GET \ + "/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_HEAD_SHA}" > "$response" + python3 - "$response" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as response: + payload = json.load(response) +print(payload.get("commit", {}).get("author", {}).get("email") or "") +PY +} + +case "${1:-}" in + create) + create_check + ;; + complete) + complete_check + ;; + commit-email) + commit_email + ;; + diagnose) + diagnose_app + ;; + *) + echo "Usage: $0 {create|complete|commit-email|diagnose}" >&2 + exit 2 + ;; +esac diff --git a/private-cicd/scripts/list-eligible-prs.py b/private-cicd/scripts/list-eligible-prs.py new file mode 100644 index 000000000000..cfdda26680bd --- /dev/null +++ b/private-cicd/scripts/list-eligible-prs.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""List open, non-draft pull requests that the OpenLab poller may queue.""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime + + +API_ROOT = "https://api.github.com" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--repository", required=True) + parser.add_argument("--base-branch", default="main") + parser.add_argument( + "--updated-after", + required=True, + help="Return PRs updated strictly after this ISO-8601 UTC timestamp.", + ) + return parser.parse_args() + + +def parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def github_get(url, token): + request = urllib.request.Request( + url, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "cloudstack-ontap-presubmit-discovery", + }, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read().decode("utf-8") + link = response.headers.get("Link", "") + except urllib.error.HTTPError as error: + detail = error.read().decode("utf-8", errors="replace") + raise SystemExit( + f"GitHub API {error.code} for {url}: {detail}") from error + next_url = "" + for part in link.split(","): + if 'rel="next"' in part: + next_url = part.split(";")[0].strip().strip("<>") + return json.loads(body), next_url + + +def list_pulls(repository, base_branch, token): + query = urllib.parse.urlencode({ + "state": "open", + "base": base_branch, + "sort": "updated", + "direction": "desc", + "per_page": "100", + }) + url = f"{API_ROOT}/repos/{repository}/pulls?{query}" + pulls = [] + while url: + page, url = github_get(url, token) + if not isinstance(page, list): + raise SystemExit(f"Unexpected GitHub pulls payload: {page}") + pulls.extend(page) + return pulls + + +def eligible_pulls(pulls, base_branch, updated_after): + eligible = [] + for pull in pulls: + if pull.get("draft"): + continue + if pull.get("base", {}).get("ref") != base_branch: + continue + updated = parse_time(pull.get("updated_at", "1970-01-01T00:00:00Z")) + if updated <= updated_after: + continue + user = pull.get("user") or {} + eligible.append({ + "pr_id": str(pull.get("number", "")), + "pr_title": pull.get("title") or "", + "pr_url": pull.get("html_url") or "", + "pr_head_branch": (pull.get("head") or {}).get("ref") or "", + "pr_head_sha": (pull.get("head") or {}).get("sha") or "", + "pr_author_login": user.get("login") or "", + "pr_author_email": user.get("email") or "", + "updated_at": pull.get("updated_at") or "", + }) + eligible.sort(key=lambda item: parse_time( + item["updated_at"] or "1970-01-01T00:00:00Z")) + return eligible + + +def main(): + args = parse_args() + token = os.environ.get("GITHUB_ACCESS_TOKEN", "").strip() + if not token: + raise SystemExit("GITHUB_ACCESS_TOKEN is empty.") + try: + updated_after = parse_time(args.updated_after) + except ValueError as error: + raise SystemExit( + f"--updated-after is not a valid ISO-8601 timestamp: {error}" + ) from error + eligible = eligible_pulls( + list_pulls(args.repository, args.base_branch, token), + args.base_branch, + updated_after, + ) + json.dump(eligible, sys.stdout) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/private-cicd/scripts/marvin-run.sh b/private-cicd/scripts/marvin-run.sh new file mode 100755 index 000000000000..9e2996a9c74d --- /dev/null +++ b/private-cicd/scripts/marvin-run.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Run ONTAP Marvin on the VM: install the staged archive with setup_env, then +# run run_tests.sh for setup_zone, iscsi, and nfs3. Copies ONTAP results into +# PHASE2_RESULT_DIR. +# Dependencies: +# CLOUDSTACK_DIR — source tree with test/integration/plugins/ontap (required), +# MARVIN_ARCHIVE — Marvin-*.tar.gz produced by build-debs.sh (required). +# PHASE2_RESULT_DIR — log/result output (required). +# ontap.cfg must already exist under test/integration/plugins/ontap/ +# (copied from render-phase2-config.py). Calls setup_env.sh and run_tests.sh. +# Management API must be healthy (check-phase2-health.sh). Called by run-phase2-vm.sh. + +set -euo pipefail + +CLOUDSTACK_DIR="${CLOUDSTACK_DIR:?Set CLOUDSTACK_DIR}" +PHASE2_RESULT_DIR="${PHASE2_RESULT_DIR:?Set PHASE2_RESULT_DIR}" +MARVIN_ARCHIVE="${MARVIN_ARCHIVE:?Set MARVIN_ARCHIVE}" +if [[ ! -f "$MARVIN_ARCHIVE" ]]; then + echo "Marvin archive does not exist: $MARVIN_ARCHIVE" >&2 + exit 1 +fi + +cd "$CLOUDSTACK_DIR" +mkdir -p "$PHASE2_RESULT_DIR" + +echo "==> Preparing Marvin environment" +MARVIN_ARCHIVE="$MARVIN_ARCHIVE" \ + bash test/integration/plugins/ontap/setup_env.sh \ + 2>&1 | tee "$PHASE2_RESULT_DIR/setup-env.log" + +run_suite() { + local filter="$1" + local log_file="$PHASE2_RESULT_DIR/${filter}.log" + + set +e + bash test/integration/plugins/ontap/run_tests.sh "$filter" \ + 2>&1 | tee "$log_file" + local status="${PIPESTATUS[0]}" + set -e + return "$status" +} + +echo "==> Creating and validating the CloudStack zone" +if ! run_suite setup_zone; then + echo "Zone setup failed; protocol suites cannot run" >&2 + exit 1 +fi + +suite_status=0 +echo "==> Running ONTAP iSCSI suites" +run_suite iscsi || suite_status=1 + +echo "==> Running ONTAP NFS3 suites" +run_suite nfs3 || suite_status=1 + +ontap_results=test/integration/plugins/ontap/results +if [[ -d "$ontap_results" ]]; then + cp -a "$ontap_results" "$PHASE2_RESULT_DIR/ontap-results" +fi + +exit "$suite_status" diff --git a/private-cicd/scripts/mvn-full.sh b/private-cicd/scripts/mvn-full.sh new file mode 100755 index 000000000000..7aa9821a1d24 --- /dev/null +++ b/private-cicd/scripts/mvn-full.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Full CloudStack Maven clean install (developer,systemvm; optional simulator). +# When RESULT_DIR is set, copies Surefire XML out of the source tree, writes +# maven.log and stage1-handoff.properties, and exits with Maven's status. +# Dependencies: +# SOURCE_DIR or CLOUDSTACK_DIR — CloudStack repo root (required). +# RESULT_DIR — if set, SOURCE_ID and SOURCE_SHA are required (Jenkins handoff); +# PR_ID and PR_HEAD_SHA are accepted as fallbacks for pull-request builds. +# Optional: SKIP_TESTS, ENABLE_NOREDIST, ENABLE_SIMULATOR (default true), +# MAVEN_THREADS (Jenkins uses 1C; empty disables -T; unset uses nproc). +# Commands: mvn, java. Does not call other private-cicd scripts. +# When RESULT_DIR is set, apply debian/rules packaging flags so build-debs.sh +# can reuse target/ without a second mvn clean. + +set -euo pipefail + +SOURCE_DIR="${SOURCE_DIR:-${CLOUDSTACK_DIR:-}}" +if [[ -z "$SOURCE_DIR" ]]; then + echo "Set SOURCE_DIR to the CloudStack repo root" >&2 + exit 1 +fi +SOURCE_DIR="$(cd "$SOURCE_DIR" && pwd)" + +SKIP_TESTS="${SKIP_TESTS:-false}" +ENABLE_NOREDIST="${ENABLE_NOREDIST:-false}" +ENABLE_SIMULATOR="${ENABLE_SIMULATOR:-true}" +THREADS="${MAVEN_THREADS-$(nproc)}" +RESULT_DIR="${RESULT_DIR:-}" + +mvn_args=(-B -P developer,systemvm) +if [[ "$ENABLE_SIMULATOR" == "true" ]]; then + mvn_args+=(-Dsimulator) +fi +if [[ "$ENABLE_NOREDIST" == "true" ]]; then + mvn_args+=(-Dnoredist) +fi +mvn_args+=(clean install) +if [[ "$SKIP_TESTS" == "true" ]]; then + mvn_args+=(-DskipTests=true) +fi +if [[ -n "$THREADS" ]]; then + mvn_args+=(-T"$THREADS") +fi + +echo "==> Full build in $SOURCE_DIR" + +if [[ -z "$RESULT_DIR" ]]; then + cd "$SOURCE_DIR" + exec mvn "${mvn_args[@]}" +fi + +SOURCE_ID="${SOURCE_ID:-${PR_ID:-}}" +SOURCE_SHA="${SOURCE_SHA:-${PR_HEAD_SHA:-}}" +: "${SOURCE_ID:?SOURCE_ID (or PR_ID) is required when RESULT_DIR is set}" +: "${SOURCE_SHA:?SOURCE_SHA (or PR_HEAD_SHA) is required when RESULT_DIR is set}" + +mkdir -p "$RESULT_DIR" +RESULT_DIR="$(cd "$RESULT_DIR" && pwd)" +rm -rf "$RESULT_DIR" +mkdir -p "$RESULT_DIR/surefire" + +cd "$SOURCE_DIR" +# server/pom.xml reads ${project.basedir}/../${cs.replace.properties}, so the +# path must stay relative to the source root like it is in debian/rules. +replace_properties="replace.properties.tmp" +cp "$SOURCE_DIR/packaging/debian/replace.properties" "$replace_properties" +# debian/rules takes the second in pom.xml. A grep|head pipeline +# would SIGPIPE under pipefail, so read the file in one pass. +version="$(awk -F'[<>]' '// { matches++; if (matches == 2) { print $3; exit } }' pom.xml)" +if [[ -z "$version" ]]; then + echo "Could not read the project version from pom.xml" >&2 + exit 1 +fi +echo "VERSION=${version}" >> "$replace_properties" +mvn_args+=(-Dsystemvm "-Dcs.replace.properties=${replace_properties}") + +echo "==> Command: mvn ${mvn_args[*]}" + +set +e +mvn "${mvn_args[@]}" 2>&1 | tee "$RESULT_DIR/maven.log" +maven_status="${PIPESTATUS[0]}" +set -e + +report_count="$( + find . -type f -path '*/target/surefire-reports/*.xml' -print | wc -l | tr -d '[:space:]' +)" +if [[ "$report_count" -gt 0 ]]; then + while IFS= read -r -d '' report; do + report_dir="$RESULT_DIR/surefire/$(dirname "$report")" + mkdir -p "$report_dir" + cp "$report" "$report_dir/" + done < <(find . -type f -path '*/target/surefire-reports/*.xml' -print0) + find . -type f -path '*/target/surefire-reports/*.xml' -print0 \ + | tar --null -czf "$RESULT_DIR/surefire-reports.tar.gz" --files-from - +fi + +if [[ "$maven_status" -eq 0 ]]; then + result="SUCCESS" +else + result="FAILURE" +fi + +cat > "$RESULT_DIR/stage1-handoff.properties" <&2 + exit 1 +fi + +ssh-keygen -t ed25519 -N '' -f "$ssh_key" -C "cloudstack-presubmit-$(date -u +%Y%m%dT%H%M%SZ)" >/dev/null +chmod 600 "$ssh_key" + +python3 "$SCRIPT_DIR/render-phase2-config.py" \ + --inventory "$INVENTORY_FILE" \ + --vm-id "$VM_ID" \ + --ontap-config "$ontap_config" \ + --secrets "$secrets_file" + +test -d "$SOURCE_DIR/test/integration/plugins/ontap" +test -f "$DEB_DIR/package-manifest.tsv" +test -f "$DEB_DIR/marvin/SHA256SUMS" + +shopt -s nullglob +marvin_archives=("$DEB_DIR"/marvin/Marvin-*.tar.gz) +shopt -u nullglob +if [[ "${#marvin_archives[@]}" -ne 1 ]]; then + echo "Expected one staged Marvin archive under $DEB_DIR/marvin; found ${#marvin_archives[@]}" >&2 + exit 1 +fi +(cd "$DEB_DIR/marvin" && sha256sum --check SHA256SUMS) + +( + cd "$DEB_DIR" + tar -czf "$debs_archive" ./*.deb package-manifest.tsv +) + +mkdir -p "$assets_root/test/integration/plugins" "$assets_root/marvin" +cp -a "$SOURCE_DIR/test/integration/plugins/ontap" "$assets_root/test/integration/plugins/" +cp "${marvin_archives[0]}" "$assets_root/marvin/" +tar -czf "$assets_archive" -C "$assets_root" . + +remote="${VM_SSH_USER}@${VM_HOST}" +remote_root="/tmp/cloudstack-phase2-${SOURCE_ID}-${SOURCE_SHA:0:12}" +remote_scripts="$remote_root/scripts" +remote_source="$remote_root/source" +remote_debs="$remote_root/debs" +remote_results="$remote_root/results" + +ssh_options=( + -i "$ssh_key" + -o BatchMode=yes + -o ConnectTimeout=15 + -o IdentitiesOnly=yes + -o ServerAliveInterval=60 + -o ServerAliveCountMax=120 + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=$known_hosts" +) +password_ssh_options=( + -o PreferredAuthentications=password + -o PubkeyAuthentication=no + -o NumberOfPasswordPrompts=1 + -o ConnectTimeout=15 + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=$known_hosts" +) + +echo "Waiting up to ${SSH_WAIT_SECONDS}s for password SSH to $remote" +export SSHPASS="$VM_SSH_PASSWORD" +deadline=$((SECONDS + SSH_WAIT_SECONDS)) +while true; do + if ssh-keyscan -T 10 "$VM_HOST" > "$known_hosts.tmp" 2>/dev/null; then + mv "$known_hosts.tmp" "$known_hosts" + if sshpass -e ssh "${password_ssh_options[@]}" "$remote" true >/dev/null 2>&1; then + break + fi + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for password SSH to $remote" >&2 + exit 1 + fi + sleep 10 +done + +echo "Installing a one-run SSH public key on $remote" +pubkey="$(cat "$ssh_key.pub")" +sshpass -e ssh "${password_ssh_options[@]}" "$remote" \ + "umask 077; mkdir -p .ssh; touch .ssh/authorized_keys; chmod 700 .ssh; chmod 600 .ssh/authorized_keys" +printf '%s\n' "$pubkey" | sshpass -e ssh "${password_ssh_options[@]}" "$remote" \ + 'cat >> .ssh/authorized_keys' +unset SSHPASS +unset VM_SSH_PASSWORD + +if ! ssh "${ssh_options[@]}" "$remote" true >/dev/null 2>&1; then + echo "Password bootstrap succeeded but key SSH to $remote failed" >&2 + exit 1 +fi + +ssh "${ssh_options[@]}" "$remote" \ + "rm -rf '$remote_root' && mkdir -p '$remote_scripts' '$remote_source' '$remote_debs' '$remote_results'" + +scp "${ssh_options[@]}" \ + "$SCRIPT_DIR/configure-cloudstack-vm.sh" \ + "$SCRIPT_DIR/check-phase2-health.sh" \ + "$SCRIPT_DIR/collect-phase2-logs.sh" \ + "$SCRIPT_DIR/marvin-run.sh" \ + "$SCRIPT_DIR/redact-phase2-results.py" \ + "$SCRIPT_DIR/run-phase2-vm.sh" \ + "$trusted_prereqs" \ + "$remote:$remote_scripts/" +scp "${ssh_options[@]}" "$debs_archive" "$remote:$remote_root/" +scp "${ssh_options[@]}" "$assets_archive" "$remote:$remote_root/" +scp "${ssh_options[@]}" "$ontap_config" "$remote:$remote_root/ontap.cfg" +scp "${ssh_options[@]}" "$secrets_file" "$remote:$remote_root/secrets.json" + +ssh "${ssh_options[@]}" "$remote" \ + "tar -xzf '$remote_root/phase2-debs.tar.gz' -C '$remote_debs' && \ + tar -xzf '$remote_root/phase2-assets.tar.gz' -C '$remote_source' && \ + cp '$remote_root/ontap.cfg' '$remote_source/test/integration/plugins/ontap/ontap.cfg' && \ + chmod 600 '$remote_root/secrets.json' '$remote_source/test/integration/plugins/ontap/ontap.cfg' && \ + chmod +x '$remote_scripts/'*.sh" + +printf -v remote_command \ + 'PHASE2_ROOT=%q BRIDGE=%q %q/run-phase2-vm.sh' \ + "$remote_root" "$BRIDGE" "$remote_scripts" + +set +e +ssh "${ssh_options[@]}" "$remote" "$remote_command" +phase2_status=$? +set -e + +# Transferred as a stream because scp rejects the '.' entry name that +# "$remote:dir/." produces. pipefail is still in effect under set +e, so $? +# reports a failure on either side of the pipe. +set +e +ssh "${ssh_options[@]}" "$remote" "tar -czf - -C '$remote_results' ." \ + | tar -xzf - -C "$RESULT_DIR" +result_copy_status=$? +set -e +ssh "${ssh_options[@]}" "$remote" "rm -rf '$remote_root'" || true + +if [[ "$phase2_status" -eq 0 && "$result_copy_status" -ne 0 ]]; then + echo "Phase 2 passed but result retrieval failed" >&2 + exit "$result_copy_status" +fi +exit "$phase2_status" diff --git a/private-cicd/scripts/run-phase2-vm.sh b/private-cicd/scripts/run-phase2-vm.sh new file mode 100755 index 000000000000..e24624b59e67 --- /dev/null +++ b/private-cicd/scripts/run-phase2-vm.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# On-VM Phase 2 driver: configure CloudStack, health-check, run Marvin, then on +# EXIT collect logs, redact secrets, write phase2.properties, and delete +# secrets.json / ontap.cfg. +# Dependencies: +# PHASE2_ROOT — scripts/, source/, debs/, results/, secrets.json (required). +# BRIDGE (default cloudbr0). Must run as root on the target VM. +# Calls: configure-cloudstack-vm.sh, check-phase2-health.sh, marvin-run.sh, +# collect-phase2-logs.sh, redact-phase2-results.py. +# debs/ must contain packages and package-manifest.tsv from build-debs.sh. +# source/ must include ONTAP tests, marvin/Marvin-*.tar.gz, and ontap.cfg. +# Invoked by run-phase2-remote.sh over SSH (or by hand on a throwaway VM). + +set -euo pipefail + +PHASE2_ROOT="${PHASE2_ROOT:?PHASE2_ROOT is required}" +BRIDGE="${BRIDGE:-cloudbr0}" + +scripts_dir="$PHASE2_ROOT/scripts" +source_dir="$PHASE2_ROOT/source" +deb_dir="$PHASE2_ROOT/debs" +result_dir="$PHASE2_ROOT/results" +secrets_file="$PHASE2_ROOT/secrets.json" +ontap_config="$source_dir/test/integration/plugins/ontap/ontap.cfg" +manifest_file="$deb_dir/package-manifest.tsv" + +mkdir -p "$result_dir" + +shopt -s nullglob +marvin_archives=("$source_dir"/marvin/Marvin-*.tar.gz) +shopt -u nullglob +if [[ "${#marvin_archives[@]}" -ne 1 ]]; then + echo "Expected one Marvin archive under $source_dir/marvin; found ${#marvin_archives[@]}" >&2 + exit 1 +fi +marvin_archive="${marvin_archives[0]}" + +on_exit() { + status=$? + trap - EXIT + OUTPUT_DIR="$result_dir" "$scripts_dir/collect-phase2-logs.sh" || true + python3 "$scripts_dir/redact-phase2-results.py" \ + --results "$result_dir" \ + --secrets "$secrets_file" \ + --ontap-config "$ontap_config" || true + printf 'PHASE2_RESULT=%s\n' "$([[ "$status" -eq 0 ]] && echo SUCCESS || echo FAILURE)" \ + > "$result_dir/phase2.properties" + rm -f "$secrets_file" "$ontap_config" + exit "$status" +} +trap on_exit EXIT + +DEB_DIR="$deb_dir" \ +SECRETS_FILE="$secrets_file" \ +ONTAP_CONFIG="$ontap_config" \ +BRIDGE="$BRIDGE" \ + "$scripts_dir/configure-cloudstack-vm.sh" \ + 2>&1 | tee "$result_dir/configure-cloudstack.log" + +MANIFEST_FILE="$manifest_file" \ +ONTAP_CONFIG="$ontap_config" \ +BRIDGE="$BRIDGE" \ + "$scripts_dir/check-phase2-health.sh" \ + 2>&1 | tee "$result_dir/health-check.log" + +CLOUDSTACK_DIR="$source_dir" \ +PHASE2_RESULT_DIR="$result_dir/marvin" \ +MARVIN_ARCHIVE="$marvin_archive" \ + "$scripts_dir/marvin-run.sh" diff --git a/private-cicd/scripts/test_list_eligible_prs.py b/private-cicd/scripts/test_list_eligible_prs.py new file mode 100644 index 000000000000..288d967fe80a --- /dev/null +++ b/private-cicd/scripts/test_list_eligible_prs.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Unit tests for list-eligible-prs.py.""" + +import importlib.util +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT = Path(__file__).with_name("list-eligible-prs.py") +SPEC = importlib.util.spec_from_file_location("list_eligible_prs", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def pull(number, updated_at, base="main", draft=False): + return { + "number": number, + "title": f"PR {number}", + "html_url": f"https://github.com/NetApp/cloudstack/pull/{number}", + "updated_at": updated_at, + "draft": draft, + "base": {"ref": base}, + "head": {"ref": f"feature/{number}", "sha": f"{number:040x}"}, + "user": {"login": f"user-{number}", "email": None}, + } + + +class EligiblePullsTest(unittest.TestCase): + def test_filters_draft_base_and_watermark(self): + watermark = MODULE.parse_time("2026-09-02T08:00:00Z") + pulls = [ + pull(1, "2026-09-02T08:00:01Z"), + pull(2, "2026-09-02T08:00:02Z", draft=True), + pull(3, "2026-09-02T08:00:03Z", base="release"), + pull(4, "2026-09-02T08:00:00Z"), + pull(5, "2026-09-02T07:59:59Z"), + ] + + result = MODULE.eligible_pulls(pulls, "main", watermark) + + self.assertEqual(["1"], [item["pr_id"] for item in result]) + self.assertEqual(f"{1:040x}", result[0]["pr_head_sha"]) + + def test_returns_oldest_updated_first(self): + watermark = MODULE.parse_time("2026-09-02T08:00:00Z") + pulls = [ + pull(3, "2026-09-02T08:00:03Z"), + pull(1, "2026-09-02T08:00:01Z"), + pull(2, "2026-09-02T08:00:02Z"), + ] + + result = MODULE.eligible_pulls(pulls, "main", watermark) + + self.assertEqual(["1", "2", "3"], [item["pr_id"] for item in result]) + + def test_list_pulls_follows_pagination(self): + with mock.patch.object( + MODULE, + "github_get", + side_effect=[([pull(1, "2026-09-02T08:00:01Z")], + "https://api.github.com/page-2"), + ([pull(2, "2026-09-02T08:00:02Z")], "")]) as get: + result = MODULE.list_pulls("NetApp/cloudstack", "main", "token") + + self.assertEqual([1, 2], [item["number"] for item in result]) + self.assertEqual( + "https://api.github.com/page-2", + get.call_args_list[1].args[0], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/private-cicd/scripts/validate-local.sh b/private-cicd/scripts/validate-local.sh new file mode 100755 index 000000000000..8464f51ceb91 --- /dev/null +++ b/private-cicd/scripts/validate-local.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Purpose: +# Syntax-check private-cicd shell/Python scripts and parse YAML under config/. +# Optional --with-docker builds docker/Dockerfile.driver. No CloudStack source required. +# Dependencies: +# bash; python3 (optional, for .py compile and YAML if PyYAML is present). +# YAML: ruby+psych, python3+yaml, or yq. docker only with --with-docker. +# CICD_ROOT — scripts/, config/, docker/ (default: parent of this file). +# Does not call Maven, packaging, or Phase 2 scripts. + +set -euo pipefail + +usage() { + cat << 'EOF' +Usage: validate-local.sh [--with-docker] + + Validates shell scripts (bash -n) and YAML under this repo's private-cicd + (or standalone CI repo) root — independent of CloudStack merge scope. + + --with-docker Build the CloudStack package-builder image and run smoke checks. + +Environment: + CICD_ROOT Root containing scripts/, config/, and docker/ (default: inferred). + +Exit status: 0 if all checks pass, non-zero otherwise. +EOF +} + +WITH_DOCKER=false +for arg in "$@"; do + case "$arg" in + -h|--help) usage; exit 0 ;; + --with-docker) WITH_DOCKER=true ;; + *) echo "Unknown option: $arg" >&2; usage >&2; exit 2 ;; + esac +done + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -n "${CICD_ROOT:-}" ]]; then + CICD_ROOT="$(cd "$CICD_ROOT" && pwd)" +else + CICD_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +fi + +echo "==> CICD_ROOT=$CICD_ROOT" + +failures=0 + +run_check() { + local name="$1" + shift + echo "==> $name" + if "$@"; then + echo " OK" + else + echo " FAILED" >&2 + failures=$((failures + 1)) + fi +} + +# --- Shell: bash -n on all scripts/*.sh (no CloudStack tree required) +while IFS= read -r -d '' f; do + run_check "bash -n: ${f#$CICD_ROOT/}" bash -n "$f" +done < <(find "$CICD_ROOT/scripts" -maxdepth 1 -type f -name '*.sh' -print0 2>/dev/null || true) + +if [[ ! -d "$CICD_ROOT/scripts" ]]; then + echo "No scripts directory at $CICD_ROOT/scripts" >&2 + failures=$((failures + 1)) +fi + +# --- Python: parse without importing optional runtime dependencies +if command -v python3 >/dev/null 2>&1; then + while IFS= read -r -d '' f; do + run_check "Python syntax: ${f#$CICD_ROOT/}" \ + python3 -c 'import pathlib,sys; compile(pathlib.Path(sys.argv[1]).read_text(), sys.argv[1], "exec")' "$f" + done < <(find "$CICD_ROOT/scripts" -maxdepth 1 -type f -name '*.py' -print0 2>/dev/null || true) +else + echo "==> Python syntax: skipped (python3 not installed)" >&2 +fi + +# --- YAML under config/ +yaml_ok=false +if command -v ruby >/dev/null 2>&1 && ruby -ryaml -e 'true' >/dev/null 2>&1; then + yaml_check() { ruby -ryaml -e "YAML.load_file(ARGV[0])" "$1"; } + yaml_ok=true +elif command -v python3 >/dev/null 2>&1; then + if python3 -c 'import yaml' >/dev/null 2>&1; then + yaml_check() { python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$1"; } + yaml_ok=true + fi +elif command -v yq >/dev/null 2>&1; then + yaml_check() { yq e '.' "$1" >/dev/null; } + yaml_ok=true +fi + +if [[ -d "$CICD_ROOT/config" ]]; then + shopt -s nullglob + yfiles=( + "$CICD_ROOT"/config/*.yaml + "$CICD_ROOT"/config/*.yml + "$CICD_ROOT"/config/*.yaml.example + "$CICD_ROOT"/config/*.yml.example + ) + shopt -u nullglob + if [[ ${#yfiles[@]} -eq 0 ]]; then + echo "==> YAML: no *.yaml in $CICD_ROOT/config (skipped)" + elif [[ "$yaml_ok" == true ]]; then + for f in "${yfiles[@]}"; do + run_check "YAML: ${f#$CICD_ROOT/}" yaml_check "$f" + done + else + echo "==> YAML: skipped (install Ruby+psych, PyYAML, or yq to validate)" >&2 + fi +else + echo "==> YAML: no config directory (skipped)" +fi + +# --- CloudStack Jenkins builder image (optional) +if [[ "$WITH_DOCKER" == true ]]; then + builder_dockerfile="$CICD_ROOT/docker/Dockerfile.driver" + if ! command -v docker >/dev/null 2>&1; then + echo "==> Docker builder image: docker is not installed" >&2 + failures=$((failures + 1)) + elif [[ ! -f "$builder_dockerfile" ]]; then + echo "==> Docker builder image: missing $builder_dockerfile" >&2 + failures=$((failures + 1)) + else + run_check "Docker builder image" \ + docker build \ + -f "$builder_dockerfile" \ + -t cloudstack-presubmit-driver:validate \ + "$CICD_ROOT" + fi +fi + +if [[ "$failures" -ne 0 ]]; then + echo "==> Done with $failures failure(s)." >&2 + exit 1 +fi + +echo "==> All checks passed." diff --git a/private-cicd/scripts/vcenter-revert.py b/private-cicd/scripts/vcenter-revert.py new file mode 100755 index 000000000000..ca63f3219ef7 --- /dev/null +++ b/private-cicd/scripts/vcenter-revert.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Revert one vCenter VM to an exact, named snapshot. + +Purpose: + Find the VM by name, revert to a uniquely named snapshot, optionally power on. + +Dependencies: + Python packages: pyVim / pyVmomi (pyvmomi). + Environment: VCENTER_USER, VCENTER_PASSWORD. + CLI: --host, --vm, --snapshot; optional --port, --timeout, --power-on, --insecure. + Called by Jenkins before run-phase2-remote.sh. Does not SSH or install packages. +""" + +import argparse +import atexit +import os +import ssl +import sys +import time + +from pyVim.connect import Disconnect, SmartConnect +from pyVmomi import vim + + +def wait_for_task(task, description, timeout): + deadline = time.monotonic() + timeout + while task.info.state in (vim.TaskInfo.State.queued, vim.TaskInfo.State.running): + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out while waiting to {description}") + time.sleep(2) + + if task.info.state != vim.TaskInfo.State.success: + raise RuntimeError(f"Failed to {description}: {task.info.error}") + + +def find_vm(content, vm_name): + view = content.viewManager.CreateContainerView( + content.rootFolder, [vim.VirtualMachine], True + ) + try: + matches = [vm for vm in view.view if vm.name == vm_name] + finally: + view.Destroy() + + if not matches: + raise ValueError(f"vCenter VM not found: {vm_name}") + if len(matches) > 1: + raise ValueError(f"More than one vCenter VM has the name: {vm_name}") + return matches[0] + + +def find_snapshot(snapshot_roots, snapshot_name): + matches = [] + + def visit(nodes): + for node in nodes: + if node.name == snapshot_name: + matches.append(node.snapshot) + visit(node.childSnapshotList) + + visit(snapshot_roots) + if not matches: + raise ValueError(f"Snapshot not found: {snapshot_name}") + if len(matches) > 1: + raise ValueError(f"More than one snapshot has the name: {snapshot_name}") + return matches[0] + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--host", required=True) + parser.add_argument("--vm", required=True) + parser.add_argument("--snapshot", required=True) + parser.add_argument("--port", type=int, default=443) + parser.add_argument("--timeout", type=int, default=1800) + parser.add_argument("--power-on", action="store_true") + parser.add_argument( + "--insecure", + action="store_true", + help="Disable TLS certificate verification for an internal lab vCenter.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + username = os.environ.get("VCENTER_USER") + password = os.environ.get("VCENTER_PASSWORD") + if not username or not password: + raise ValueError("VCENTER_USER and VCENTER_PASSWORD must be set") + + context = None + if args.insecure: + context = ssl._create_unverified_context() # pylint: disable=protected-access + + service_instance = SmartConnect( + host=args.host, + user=username, + pwd=password, + port=args.port, + sslContext=context, + ) + atexit.register(Disconnect, service_instance) + + vm = find_vm(service_instance.RetrieveContent(), args.vm) + if vm.snapshot is None: + raise ValueError(f"VM has no snapshots: {args.vm}") + + snapshot = find_snapshot(vm.snapshot.rootSnapshotList, args.snapshot) + print(f"Reverting VM '{args.vm}' to snapshot '{args.snapshot}'") + wait_for_task( + snapshot.RevertToSnapshot_Task(), + f"revert '{args.vm}' to '{args.snapshot}'", + args.timeout, + ) + + if args.power_on and vm.runtime.powerState != vim.VirtualMachinePowerState.poweredOn: + print(f"Powering on VM '{args.vm}'") + wait_for_task(vm.PowerOnVM_Task(), f"power on '{args.vm}'", args.timeout) + + print("Snapshot revert completed") + + +if __name__ == "__main__": + try: + main() + except Exception as error: # pylint: disable=broad-except + print(f"ERROR: {error}", file=sys.stderr) + sys.exit(1) diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 934600eb2b61..00ff20b85258 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1338,7 +1338,7 @@ public static HypervisorType getHypervisorTypeFromFormat(long dcId, ImageFormat type = HypervisorType.Hyperv; } } if (format == ImageFormat.RAW) { - // Currently, KVM only supports RBD, PowerFlex, and FiberChannel images of type RAW. + // Currently, KVM only supports RBD, PowerFlex, FiberChannel and OntapiSCSI images of type RAW. // This results in a weird collision with OVM volumes which // can only be raw, thus making KVM RBD volumes show up as OVM // rather than RBD. This block of code can (hopefully) by checking to @@ -1355,7 +1355,8 @@ public static HypervisorType getHypervisorTypeFromFormat(long dcId, ImageFormat StoragePoolType.PowerFlex, StoragePoolType.CLVM, StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(pool.getPoolType())) { + StoragePoolType.FiberChannel, + StoragePoolType.OntapiSCSI).contains(pool.getPoolType())) { // This case will note the presence of non-qcow2 primary stores, suggesting KVM without NFS. Otherwse, // If this check is not passed, the hypervisor type will remain OVM. type = HypervisorType.KVM; diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index e82ec525f01b..42b12475bd9d 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -220,6 +220,12 @@ public VolumeResponse newVolumeResponse(ResponseView view, VolumeJoinVO volume) volResponse.setDiskOfferingName(volume.getDiskOfferingName()); volResponse.setDiskOfferingDisplayText(volume.getDiskOfferingDisplayText()); } + DiskOffering diskOffering = computeOnlyDiskOffering != null + ? computeOnlyDiskOffering + : ApiDBUtils.findDiskOfferingById(volume.getDiskOfferingId()); + if (diskOffering != null) { + volResponse.setCustomizedIops(diskOffering.isCustomizedIops()); + } if (view == ResponseView.Full) { volResponse.setStorageType(volume.isUseLocalStorage() ? ServiceOffering.StorageType.local.toString() : ServiceOffering.StorageType.shared.toString()); diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index 3be7384ffb71..8ed70f32f9af 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -573,6 +573,8 @@ public long getUsedIops(StoragePoolVO pool) { throw new CloudRuntimeException("Storage driver in CapacityManagerImpl.getUsedIops(StoragePoolVO) is not a PrimaryDataStoreDriver."); } + //Managed pool, return value contains used size of the pool. + //Unmanaged pool, return value contains used pool size and template size. @Override public long getAllocatedPoolCapacity(StoragePoolVO pool, VMTemplateVO templateForVmCreation) { long totalAllocatedSize; @@ -580,18 +582,7 @@ public long getAllocatedPoolCapacity(StoragePoolVO pool, VMTemplateVO templateFo // if the storage pool is managed, the used bytes can be larger than the sum of the sizes of all of the non-destroyed volumes // in this case, call getUsedBytes(StoragePoolVO) if (pool.isManaged()) { - totalAllocatedSize = getUsedBytes(pool); - - if (templateForVmCreation != null) { - VMTemplateStoragePoolVO templatePoolVO = _templatePoolDao.findByPoolTemplate(pool.getId(), templateForVmCreation.getId(), null); - if (templatePoolVO == null) { - // template is not installed in the pool, consider the template size for allocation - long templateForVmCreationSize = templateForVmCreation.getSize() != null ? templateForVmCreation.getSize() : 0; - totalAllocatedSize += templateForVmCreationSize; - } - } - - return totalAllocatedSize; + return getUsedBytes(pool); } else { // Get size for all the non-destroyed volumes. Pair sizes = _volumeDao.getNonDestroyedCountAndTotalByPool(pool.getId()); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 69f04393174c..73085980a410 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1265,10 +1265,18 @@ public PrimaryDataStoreInfo updateStoragePool(UpdateStoragePoolCmd cmd) throws I boolean changes = false; Long updatedCapacityBytes = null; Long capacityBytes = cmd.getCapacityBytes(); + // retrieve current details and merge/overlay input to capture changes + Map details = null; + details = _storagePoolDetailsDao.listDetailsKeyPairs(id); + if (inputDetails != null) { + details.putAll(inputDetails); + changes = true; + } if (capacityBytes != null) { if (capacityBytes != pool.getCapacityBytes()) { updatedCapacityBytes = capacityBytes; + details.put(PrimaryDataStoreLifeCycle.CAPACITY_BYTES, String.valueOf(updatedCapacityBytes)); changes = true; } } @@ -1278,40 +1286,29 @@ public PrimaryDataStoreInfo updateStoragePool(UpdateStoragePoolCmd cmd) throws I if (capacityIops != null) { if (!capacityIops.equals(pool.getCapacityIops())) { updatedCapacityIops = capacityIops; + details.put(PrimaryDataStoreLifeCycle.CAPACITY_IOPS, String.valueOf(updatedCapacityIops)); changes = true; } } - // retrieve current details and merge/overlay input to capture changes - Map details = null; - details = _storagePoolDetailsDao.listDetailsKeyPairs(id); - if (inputDetails != null) { - details.putAll(inputDetails); - changes = true; - } - if (changes) { - StoragePoolVO storagePool = _storagePoolDao.findById(id); - DataStoreProvider dataStoreProvider = _dataStoreProviderMgr.getDataStoreProvider(storagePool.getStorageProviderName()); + DataStoreProvider dataStoreProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName()); DataStoreLifeCycle dataStoreLifeCycle = dataStoreProvider.getDataStoreLifeCycle(); - if (dataStoreLifeCycle instanceof PrimaryDataStoreLifeCycle) { + if (cmd.getUrl() != null) { + details.put("url", cmd.getUrl()); + } + ((PrimaryDataStoreLifeCycle)dataStoreLifeCycle).updateStoragePool(pool, details); if (updatedCapacityBytes != null) { - details.put(PrimaryDataStoreLifeCycle.CAPACITY_BYTES, updatedCapacityBytes != null ? String.valueOf(updatedCapacityBytes) : null); - _storagePoolDao.updateCapacityBytes(id, updatedCapacityBytes); + pool.setCapacityBytes(updatedCapacityBytes); } if (updatedCapacityIops != null) { - details.put(PrimaryDataStoreLifeCycle.CAPACITY_IOPS, updatedCapacityIops != null ? String.valueOf(updatedCapacityIops) : null); - _storagePoolDao.updateCapacityIops(id, updatedCapacityIops); + pool.setCapacityIops(updatedCapacityIops); } - if (cmd.getUrl() != null) { - details.put("url", cmd.getUrl()); - } - _storagePoolDao.update(id, storagePool); + _storagePoolDao.update(id, pool); _storagePoolDao.updateDetails(id, details); } } - return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 8674014addd7..1aaeb0550933 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -884,7 +884,7 @@ public VolumeVO allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long minIops = 0L; maxIops = 0L; } else { - if (minIops == null || minIops <= 0) { + if (minIops == null || minIops < 0) { throw new InvalidParameterValueException("The min IOPS must be greater than 0."); } @@ -1127,6 +1127,13 @@ private VolumeVO createVolumeOnStoragePool(Long volumeId, Long storageId) throws throw new InvalidParameterValueException(String.format("Disk offering: %s is not compatible with the storage pool", diskOffering.getUuid())); } + HypervisorType hypervisorType = _volsDao.getHypervisorType(volume.getId()); + DiskProfile diskProfile = new DiskProfile(volume, diskOffering, hypervisorType); + if (!storageMgr.storagePoolHasEnoughSpace(Collections.singletonList(new Pair<>(volume, diskProfile)), storagePool, storagePool.getClusterId())) { + throw new InvalidParameterValueException(String.format("Storage pool %s does not have enough space to create volume %s.", + storagePool.getName(), volume.getUuid())); + } + DataStore dataStore = dataStoreMgr.getDataStore(storageId, DataStoreRole.Primary); VolumeInfo volumeInfo = volFactory.getVolume(volumeId, dataStore); AsyncCallFuture createVolumeFuture = volService.createVolumeAsync(volumeInfo, dataStore); @@ -1184,6 +1191,9 @@ public VolumeVO createVolume(long volumeId, Long vmId, Long snapshotId, Long sto created = false; VolumeInfo vol = volFactory.getVolume(volumeId); vol.stateTransit(Volume.Event.DestroyRequested); + if (e instanceof InvalidParameterValueException) { + throw (InvalidParameterValueException) e; + } throw new CloudRuntimeException(String.format("Failed to create volume: %s", volume), e); } finally { if (!created) { @@ -1446,11 +1456,13 @@ public VolumeVO resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationExcep if (currentSize != newSize || !compareEqualsIncludingNullOrZero(newMaxIops, volume.getMaxIops()) || !compareEqualsIncludingNullOrZero(newMinIops, volume.getMinIops())) { volumeResizeRequired = true; } - if (!volumeMigrateRequired && !volumeResizeRequired && newDiskOffering != null) { - _volsDao.updateDiskOffering(volume.getId(), newDiskOffering.getId()); - volume = _volsDao.findById(volume.getId()); - updateStorageWithTheNewDiskOffering(volume, newDiskOffering); + if (!volumeMigrateRequired && !volumeResizeRequired) { + if (newDiskOffering != null && newDiskOffering.getId() != diskOffering.getId()) { + _volsDao.updateDiskOffering(volume.getId(), newDiskOffering.getId()); + volume = _volsDao.findById(volume.getId()); + updateStorageWithTheNewDiskOffering(volume, newDiskOffering); + } return volume; } @@ -2321,7 +2333,7 @@ public Volume changeDiskOfferingForVolumeInternal(Long volumeId, Long newDiskOff if (volumeMigrateRequired) { logger.warn(String.format("Volume change offering operation succeeded for volume ID: %s but volume resize operation failed, so please try resize volume operation separately", volume.getUuid())); } else { - throw new CloudRuntimeException(String.format("Volume disk offering change operation failed for volume ID [%s] because the volume resize operation failed.", volume.getUuid())); + throw new CloudRuntimeException(String.format("Volume disk offering change operation failed for volume ID [%s] because the volume resize operation failed: %s", volume.getUuid(), e.getMessage())); } } } @@ -2365,7 +2377,7 @@ public Volume changeDiskOfferingForVolumeInternal(Long volumeId, Long newDiskOff if (volumeMigrateRequired) { logger.warn(String.format("Volume change offering operation succeeded for volume ID: %s but volume resize operation failed, so please try resize volume operation separately", volume.getUuid())); } else { - throw new CloudRuntimeException(String.format("Volume change offering operation failed for volume ID: %s due to resize volume operation failed", volume.getUuid())); + throw new CloudRuntimeException(String.format("Volume change offering operation failed for volume ID: %s due to resize volume operation failed: %s", volume.getUuid(), e.getMessage()), e); } } } diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 2d23e22a0152..8bbd84e90717 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -51,6 +51,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -1639,7 +1640,13 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc StoragePoolType poolType = volume.getStoragePoolType(); - if (isKvmAndFileBasedStorage && backupSnapToSecondary) { + // NetApp ONTAP managed PRIMARY snapshots remain on primary/array storage (FlexVol). + // They must not use secondary archive bookkeeping (postSnapshotDirectlyToSecondary) or a physical + // secondary copy — delete is handled via StorageSystemSnapshotStrategy → driver deleteAsync. + boolean archiveSnapshotToSecondary = backupSnapToSecondary + && !isManagedPrimaryLocationSnapshot(storagePool, payload); + + if (isKvmAndFileBasedStorage && archiveSnapshotToSecondary) { DataStore imageStore = snapshotSrv.findSnapshotImageStore(snapshot); if (imageStore == null) { throw new CloudRuntimeException(String.format("Could not find any secondary storage to allocate snapshot [%s].", snapshot)); @@ -1662,7 +1669,7 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc SnapshotInfo snapshotOnPrimary = snapshotStrategy.takeSnapshot(snapshot); - if (backupSnapToSecondary) { + if (archiveSnapshotToSecondary) { if (!isKvmAndFileBasedStorage) { backupSnapshotToSecondary(payload.getAsyncBackup(), snapshotStrategy, snapshotOnPrimary, payload.getZoneIds(), payload.getStoragePoolIds()); if (!payload.getAsyncBackup() && ClvmPoolManager.isClvmPoolType(storagePool.getPoolType())) { @@ -1672,7 +1679,15 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc postSnapshotDirectlyToSecondary(snapshot, snapshotOnPrimary, snapshotId); } } else { - logger.debug("Skipping backup of snapshot [{}] to secondary due to configuration [{}].", snapshotOnPrimary.getUuid(), SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key()); + if (backupSnapToSecondary && isManagedPrimaryLocationSnapshot(storagePool, payload)) { + logger.info("takeSnapshot: snapshot [{}] on NetApp ONTAP managed primary pool [{}] with locationType=PRIMARY — " + + "keeping snapshot on primary/array storage only; not archiving to secondary " + + "(backup.snapshot.after.take is ignored for this snapshot class)", + snapshotId, storagePool.getId()); + } else { + logger.debug("Skipping backup of snapshot [{}] to secondary due to configuration [{}].", + snapshotOnPrimary.getUuid(), SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key()); + } if (CollectionUtils.isNotEmpty(payload.getStoragePoolIds()) && payload.getAsyncBackup()) { snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.COPY); @@ -1688,10 +1703,10 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc snapshotZoneDao.addSnapshotToZone(snapshotId, snapshot.getDataCenterId()); DataStoreRole dataStoreRole; - if (payload.getAsyncBackup() && backupSnapToSecondary && !isKvmAndFileBasedStorage) { + if (payload.getAsyncBackup() && archiveSnapshotToSecondary && !isKvmAndFileBasedStorage) { dataStoreRole = DataStoreRole.Primary; } else { - dataStoreRole = backupSnapToSecondary ? snapshotHelper.getDataStoreRole(snapshot) : DataStoreRole.Primary; + dataStoreRole = archiveSnapshotToSecondary ? snapshotHelper.getDataStoreRole(snapshot) : DataStoreRole.Primary; } List snapshotStoreRefs = _snapshotStoreDao.listReadyBySnapshot(snapshotId, dataStoreRole); @@ -1707,7 +1722,7 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, volume.getSize() - snapshotStoreRef.getPhysicalSize()); if (!payload.getAsyncBackup()) { - if (backupSnapToSecondary) { + if (archiveSnapshotToSecondary) { copyNewSnapshotToZones(snapshotId, snapshot.getDataCenterId(), payload.getZoneIds()); } if (CollectionUtils.isNotEmpty(payload.getStoragePoolIds())) { @@ -1737,6 +1752,12 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc return snapshot; } + /** + * KVM file-based fast-path: records snapshot on image store without copying bytes, then drops the + * primary {@code snapshot_data_store} row. Used only for non-managed primary storage when + * {@code backup.snapshot.after.take} is enabled. NetApp ONTAP managed PRIMARY snapshots never + * call this method — see {@link #isManagedPrimaryLocationSnapshot}. + */ private void postSnapshotDirectlyToSecondary(SnapshotInfo snapshot, SnapshotInfo snapshotOnPrimary, Long snapshotId) { logger.debug("{} was directly copied to secondary storage because the hypervisor is KVM, the primary storage is file-based and the [{}] configuration" + " is set to true.", snapshot.getSnapshotVO().toString(), SnapshotInfo.BackupSnapshotAfterTakingSnapshot); @@ -1753,6 +1774,21 @@ private void postSnapshotDirectlyToSecondary(SnapshotInfo snapshot, SnapshotInfo snapshotDetailsDao.removeDetail(snapshotOnPrimary.getId(), AsyncJob.Constants.MS_ID); } + /** + * Returns true for volume snapshots on NetApp ONTAP managed primary/array storage. + * + *

ONTAP volume snapshots stay on the FlexVol on primary — they are not moved or + * mirrored to secondary storage. The primary {@code snapshot_data_store} row must + * remain so volume-snapshot DELETE uses {@code StorageSystemSnapshotStrategy} and + * the primary datastore driver.

+ * + *

Other managed storage providers are not affected by this check.

+ */ + private boolean isManagedPrimaryLocationSnapshot(StoragePool storagePool, CreateSnapshotPayload payload) { + return storagePool != null && storagePool.isManaged() + && DataStoreProvider.ONTAP_PLUGIN_NAME.equals(storagePool.getStorageProviderName()); + } + @Override public boolean isHypervisorKvmAndFileBasedStorage(VolumeInfo volumeInfo, StoragePool storagePool) { Set fileBasedStores = Set.of(Storage.StoragePoolType.SharedMountPoint, Storage.StoragePoolType.NetworkFilesystem, Storage.StoragePoolType.Filesystem); diff --git a/server/src/test/java/com/cloud/api/query/dao/VolumeJoinDaoImplTest.java b/server/src/test/java/com/cloud/api/query/dao/VolumeJoinDaoImplTest.java index 8ac60f4a64c8..19d7b277c4e8 100755 --- a/server/src/test/java/com/cloud/api/query/dao/VolumeJoinDaoImplTest.java +++ b/server/src/test/java/com/cloud/api/query/dao/VolumeJoinDaoImplTest.java @@ -16,31 +16,200 @@ // under the License. package com.cloud.api.query.dao; -import com.cloud.api.query.vo.VolumeJoinVO; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.util.function.Consumer; + +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.context.CallContext; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.api.ApiDBUtils; +import com.cloud.api.ApiResponseHelper; +import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.Volume; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; + @RunWith(MockitoJUnitRunner.class) public class VolumeJoinDaoImplTest extends GenericDaoBaseWithTagInformationBaseTest { + private static final long DISK_OFFERING_ID = 10L; + @InjectMocks private VolumeJoinDaoImpl _volumeJoinDaoImpl; + @Mock + private AnnotationDao annotationDao; + + @Mock + private AccountManager _accountMgr; + private VolumeJoinVO volume = new VolumeJoinVO(); private VolumeResponse volumeResponse = new VolumeResponse(); @Before public void setup() { - prepareSetup(); + lenient().when(annotationDao.hasAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(false); + lenient().when(_accountMgr.isRootAdmin(eq(1L))).thenReturn(true); } @Test public void testUpdateVolumeTagInfo(){ + prepareSetup(); testUpdateTagInformation(_volumeJoinDaoImpl, volume, volumeResponse); } + @Test + public void testNewVolumeResponseSetsCustomizedIopsForComputeOfferingRoot() { + VolumeJoinVO volumeJoin = mockVolumeWithDiskOffering(); + DiskOfferingVO computeOnlyOffering = mock(DiskOfferingVO.class); + when(computeOnlyOffering.isCustomizedIops()).thenReturn(true); + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(serviceOffering.getId()).thenReturn(5L); + when(serviceOffering.getName()).thenReturn("FullCustom_Compute"); + when(serviceOffering.getDisplayText()).thenReturn("FullCustom_Compute"); + + VolumeResponse response = newVolumeResponse(volumeJoin, apiDBUtils -> { + apiDBUtils.when(() -> ApiDBUtils.findComputeOnlyDiskOfferingById(DISK_OFFERING_ID)) + .thenReturn(computeOnlyOffering); + apiDBUtils.when(() -> ApiDBUtils.findServiceOfferingByComputeOnlyDiskOffering(DISK_OFFERING_ID, false)) + .thenReturn(serviceOffering); + }); + + assertTrue(response.isCustomizedIops()); + assertEquals("5", response.getServiceOfferingId()); + assertEquals("FullCustom_Compute", response.getServiceOfferingName()); + assertNull(response.getDiskOfferingId()); + } + + @Test + public void testNewVolumeResponseSetsCustomizedIopsForDiskOffering() { + VolumeJoinVO volumeJoin = mockVolumeWithDiskOffering(); + when(volumeJoin.getDiskOfferingUuid()).thenReturn("disk-offering-uuid"); + when(volumeJoin.getDiskOfferingName()).thenReturn("NFS_Disk_Custom75"); + when(volumeJoin.getDiskOfferingDisplayText()).thenReturn("NFS_Disk_Custom75"); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.isCustomizedIops()).thenReturn(true); + + VolumeResponse response = newVolumeResponse(volumeJoin, apiDBUtils -> { + apiDBUtils.when(() -> ApiDBUtils.findComputeOnlyDiskOfferingById(DISK_OFFERING_ID)).thenReturn(null); + apiDBUtils.when(() -> ApiDBUtils.findDiskOfferingById(DISK_OFFERING_ID)).thenReturn(diskOffering); + }); + + assertTrue(response.isCustomizedIops()); + assertEquals("disk-offering-uuid", response.getDiskOfferingId()); + assertNull(response.getServiceOfferingId()); + } + + @Test + public void testNewVolumeResponseSetsCustomizedIopsFalseWhenOfferingIsFixed() { + VolumeJoinVO volumeJoin = mockVolumeWithDiskOffering(); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.isCustomizedIops()).thenReturn(false); + + VolumeResponse response = newVolumeResponse(volumeJoin, apiDBUtils -> { + apiDBUtils.when(() -> ApiDBUtils.findComputeOnlyDiskOfferingById(DISK_OFFERING_ID)).thenReturn(null); + apiDBUtils.when(() -> ApiDBUtils.findDiskOfferingById(DISK_OFFERING_ID)).thenReturn(diskOffering); + }); + + assertFalse(response.isCustomizedIops()); + } + + @Test + public void testNewVolumeResponseSetsCustomizedIopsFalseForFixedComputeOffering() { + VolumeJoinVO volumeJoin = mockVolumeWithDiskOffering(); + DiskOfferingVO computeOnlyOffering = mock(DiskOfferingVO.class); + when(computeOnlyOffering.isCustomizedIops()).thenReturn(false); + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(serviceOffering.getId()).thenReturn(5L); + when(serviceOffering.getName()).thenReturn("Fixed_Compute"); + when(serviceOffering.getDisplayText()).thenReturn("Fixed_Compute"); + + VolumeResponse response = newVolumeResponse(volumeJoin, apiDBUtils -> { + apiDBUtils.when(() -> ApiDBUtils.findComputeOnlyDiskOfferingById(DISK_OFFERING_ID)) + .thenReturn(computeOnlyOffering); + apiDBUtils.when(() -> ApiDBUtils.findServiceOfferingByComputeOnlyDiskOffering(DISK_OFFERING_ID, false)) + .thenReturn(serviceOffering); + }); + + assertFalse(response.isCustomizedIops()); + assertEquals("5", response.getServiceOfferingId()); + assertEquals("Fixed_Compute", response.getServiceOfferingName()); + assertNull(response.getDiskOfferingId()); + } + + @Test + public void testNewVolumeResponseLeavesCustomizedIopsNullWhenOfferingValueIsNull() { + VolumeJoinVO volumeJoin = mockVolumeWithDiskOffering(); + when(volumeJoin.getDiskOfferingUuid()).thenReturn("disk-offering-uuid"); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.isCustomizedIops()).thenReturn(null); + + VolumeResponse response = newVolumeResponse(volumeJoin, apiDBUtils -> { + apiDBUtils.when(() -> ApiDBUtils.findComputeOnlyDiskOfferingById(DISK_OFFERING_ID)).thenReturn(null); + apiDBUtils.when(() -> ApiDBUtils.findDiskOfferingById(DISK_OFFERING_ID)).thenReturn(diskOffering); + }); + + assertNull(response.isCustomizedIops()); + assertNull(response.getCustomizedIops()); + } + + private VolumeJoinVO mockVolumeWithDiskOffering() { + VolumeJoinVO volumeJoin = mock(VolumeJoinVO.class); + when(volumeJoin.getUuid()).thenReturn("volume-uuid"); + when(volumeJoin.getName()).thenReturn("ROOT-67"); + when(volumeJoin.getDiskOfferingId()).thenReturn(DISK_OFFERING_ID); + when(volumeJoin.getVmId()).thenReturn(0L); + when(volumeJoin.getState()).thenReturn(Volume.State.Ready); + when(volumeJoin.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(volumeJoin.getTemplateId()).thenReturn(0L); + when(volumeJoin.getTagId()).thenReturn(0L); + when(volumeJoin.getMinIops()).thenReturn(0L); + when(volumeJoin.getMaxIops()).thenReturn(1000L); + when(volumeJoin.getDeleteProtection()).thenReturn(false); + when(volumeJoin.isDisplayVolume()).thenReturn(true); + when(volumeJoin.getKmsWrappedKeyId()).thenReturn(null); + when(volumeJoin.getJobId()).thenReturn(null); + return volumeJoin; + } + + private VolumeResponse newVolumeResponse(VolumeJoinVO volumeJoin, Consumer> apiStubs) { + Account callingAccount = mock(Account.class); + when(callingAccount.getId()).thenReturn(1L); + CallContext callContext = mock(CallContext.class); + when(callContext.getCallingAccount()).thenReturn(callingAccount); + + try (MockedStatic apiDBUtils = mockStatic(ApiDBUtils.class); + MockedStatic callContextMocked = mockStatic(CallContext.class); + MockedStatic apiResponseHelperMocked = mockStatic(ApiResponseHelper.class)) { + apiStubs.accept(apiDBUtils); + callContextMocked.when(CallContext::current).thenReturn(callContext); + apiResponseHelperMocked.when(() -> ApiResponseHelper.populateOwner(any(), any())) + .thenAnswer(invocation -> null); + return _volumeJoinDaoImpl.newVolumeResponse(ResponseView.Restricted, volumeJoin); + } + } + } diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 12258e2cc160..459f32286e02 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -54,6 +54,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -2054,6 +2055,75 @@ public void testResizeVolumeInAllocateState() throws ResourceAllocationException verify(volumeServiceMock, times(0)).resize(any(VolumeInfo.class)); } } + + @Test + public void testResizeVolumeNoChangeSkipsDriver() throws ResourceAllocationException, ExecutionException, InterruptedException { + ResizeVolumeCmd cmd = new ResizeVolumeCmd(); + ReflectionTestUtils.setField(cmd, "id", volumeMockId); + + testResizeVolumeSetup(); + when(storageMgr.storagePoolHasEnoughSpaceForResize(any(), nullable(Long.class), nullable(Long.class))).thenReturn(true); + + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + Volume result = volumeApiServiceImpl.resizeVolume(cmd); + + Assert.assertSame(volumeVoMock, result); + verify(volumeServiceMock, times(0)).resize(any(VolumeInfo.class)); + verify(volumeDaoMock, times(0)).updateDiskOffering(anyLong(), anyLong()); + } + } + + @Test + public void testResizeVolumeSameDiskOfferingSkipsDriver() throws ResourceAllocationException, ExecutionException, InterruptedException { + ResizeVolumeCmd cmd = new ResizeVolumeCmd(); + ReflectionTestUtils.setField(cmd, "id", volumeMockId); + ReflectionTestUtils.setField(cmd, "newDiskOfferingId", newDiskOfferingMockId); + + testResizeVolumeSetup(); + when(newDiskOfferingMock.getDiskSize()).thenReturn(volumeSizeMock); + when(diskOfferingMock.getId()).thenReturn(diskOfferingMockId); + when(newDiskOfferingMock.getId()).thenReturn(diskOfferingMockId); + when(storageMgr.storagePoolHasEnoughSpaceForResize(any(), nullable(Long.class), nullable(Long.class))).thenReturn(true); + + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + Volume result = volumeApiServiceImpl.resizeVolume(cmd); + + Assert.assertSame(volumeVoMock, result); + verify(volumeServiceMock, times(0)).resize(any(VolumeInfo.class)); + verify(volumeDaoMock, times(0)).updateDiskOffering(anyLong(), anyLong()); + } + } + + @Test + public void testResizeVolumeNewDiskOfferingSameSizeUpdatesOfferingWithoutResize() throws ResourceAllocationException, ExecutionException, InterruptedException { + ResizeVolumeCmd cmd = new ResizeVolumeCmd(); + ReflectionTestUtils.setField(cmd, "id", volumeMockId); + ReflectionTestUtils.setField(cmd, "newDiskOfferingId", newDiskOfferingMockId); + + testResizeVolumeSetup(); + when(newDiskOfferingMock.getDiskSize()).thenReturn(volumeSizeMock); + when(diskOfferingMock.getId()).thenReturn(diskOfferingMockId); + when(newDiskOfferingMock.getId()).thenReturn(newDiskOfferingMockId); + when(storageMgr.storagePoolHasEnoughSpaceForResize(any(), nullable(Long.class), nullable(Long.class))).thenReturn(true); + + DataStore offeringStore = Mockito.mock(DataStore.class); + PrimaryDataStoreDriver driver = Mockito.mock(PrimaryDataStoreDriver.class); + when(dataStoreMgr.getDataStore(anyLong(), eq(DataStoreRole.Primary))).thenReturn(offeringStore); + when(offeringStore.getDriver()).thenReturn(driver); + when(driver.informStorageForDiskOfferingChange()).thenReturn(true); + + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + Mockito.clearInvocations(volumeDaoMock); + Volume result = volumeApiServiceImpl.resizeVolume(cmd); + + Assert.assertSame(volumeVoMock, result); + verify(volumeDaoMock).updateDiskOffering(volumeMockId, newDiskOfferingMockId); + verify(volumeDaoMock, times(2)).findById(volumeMockId); + verify(driver).updateStorageWithTheNewDiskOffering(volumeVoMock, newDiskOfferingMock); + verify(volumeServiceMock, times(0)).resize(any(VolumeInfo.class)); + } + } + private UserVmVO getMockedVm() { UserVmVO vm = Mockito.mock(UserVmVO.class); Mockito.when(vm.getId()).thenReturn(1L); diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md new file mode 100644 index 000000000000..6a40a1402c2e --- /dev/null +++ b/test/integration/plugins/ontap/README.md @@ -0,0 +1,426 @@ + +# NetApp ONTAP Integration Tests — README + +This folder contains end-to-end integration tests for the NetApp ONTAP primary storage plugin in Apache CloudStack. The tests use the **Marvin** framework to drive real CloudStack API calls against a live management server and verify outcomes on a real ONTAP storage system. + +CI wiring: +- Pipeline: `private-cicd/Jenkinsfile` (downstream only) +- Inventory schema: `private-cicd/config/vm-inventory.yaml.example` +- Runtime config: generated from a populated Jenkins Secret file and separate + Jenkins credentials; populated inventory and real credentials are never + committed +- Test order: `setup_zone`, `iscsi`, then `nfs3` +- Archived results: `presubmit-results/phase2/marvin/` + +The pipeline installs the version-matched Marvin archive produced by the +CloudStack package build, stages this directory on a reverted lab VM, and runs +the protocol batches sequentially. See +[`private-cicd/docs/PRIVATE-CICD-GUIDE.md`](../../../../private-cicd/docs/PRIVATE-CICD-GUIDE.md) +for the deployment, health-gate, and artifact contracts. + +--- + +## Directory layout + +``` +test/integration/plugins/ontap/ +├── ontap.cfg # Environment config (IPs, credentials, zone info) +├── ontap_test_base.py # Shared base class and ONTAP REST client +├── guest_ssh.py # Guest SSH (paramiko) for the VM workflow suites +├── setup_env.sh # venv: Marvin plus requirements.txt +├── requirements.txt # setuptools, pyvmomi, paramiko pins +├── TEST_CASES.md # Full test case reference table (151 tests) +├── README.md # This file +│ +├── nfs3/ +│ ├── pool/ +│ │ ├── test_pool_lifecycle.py # Pool create/disable/enable/maintenance/delete +│ │ ├── test_pool_with_volumes.py # Same lifecycle with a CS volume present +│ │ └── test_zone_scoped_pool.py # Zone-scoped pool (attachZone) +│ ├── volume/ +│ │ └── test_volume_lifecycle.py # Volume create/delete/negative-delete +│ ├── instance/ +│ │ └── test_vm_volume_attach.py # Pool + volume + VM + resize, snapshots, attach/detach + template cache +│ └── template/ +│ └── test_template_cache_negative.py # Template-cache boundary cases +│ +└── iscsi/ + ├── pool/ + │ ├── test_pool_lifecycle.py # iSCSI pool lifecycle + igroup assertions + │ ├── test_pool_with_volumes.py # Same lifecycle with a LUN-backed volume + │ └── test_zone_scoped_pool.py # Zone-scoped iSCSI pool + ├── volume/ + │ └── test_volume_lifecycle.py # LUN create/delete/negative-delete + ├── instance/ + │ └── test_vm_volume_attach.py # Pool + LUN + VM + resize, snapshots, LUN-map lifecycle + template cache + └── template/ + └── test_template_cache_negative.py # Template-cache boundary cases +``` + +--- + +## What is being tested + +The ONTAP plugin (`plugins/storage/volume/ontap/`) integrates CloudStack's primary storage API with the NetApp ONTAP REST API. Every test suite verifies **both sides** of an operation: + +1. **CloudStack side** — the expected `listStoragePools` / `listVolumes` / `listVirtualMachines` state after each API call. +2. **ONTAP side** — the actual ONTAP object state (FlexVol, LUN, igroup, export policy, LUN-map) via direct REST API queries. + +### NFS3 vs iSCSI — key differences + +| Aspect | NFS3 | iSCSI | +| -------- | ------ | ------- | +| ONTAP object per pool | FlexVol + export policy | FlexVol; igroups are shared per KVM host and SVM | +| ONTAP object per CS volume | None (FlexVol is shared) | One LUN inside the FlexVol | +| Host connectivity | NFS mount | iSCSI login (IQN-based) | +| Volume detach from running VM | Works via virtio hot-unplug | Requires KVM guest to support SCSI hot-unplug | + +--- + +## Prerequisites + +Before running any test: + +1. **CloudStack management server** running with the ONTAP plugin deployed (jar in `/usr/share/cloudstack-management/lib/`). +2. **Integration API port 8096 enabled** — run on the management server: + ```sql + UPDATE configuration SET value='8096' WHERE name='integration.api.port'; + ``` + Then restart: `systemctl restart cloudstack-management` +3. **MySQL accessible remotely** from your laptop (port 3306). If not: + ```bash + sudo sed -i 's/^bind-address.*/bind-address = 0.0.0.0/' /etc/mysql/mysql.conf.d/mysqld.cnf + sudo iptables -I INPUT -p tcp --dport 3306 -j ACCEPT + sudo systemctl restart mysql + ``` +4. **ONTAP SVM** with NFS3 service and/or iSCSI service enabled, and at least one data LIF per protocol. +5. **KVM cluster** registered in CloudStack. For iSCSI tests, every KVM host must have iSCSI configured (its `storageUrl` starts with `iqn.`). +6. **`ontap.cfg` populated** — see the [Configuration](#configuration--ontapcfg) section. + +### Python / Marvin setup + +```bash +# Install Marvin from the repo's bundled tarball +python3 -m pip install --user \ + "$(ls tools/marvin/dist/Marvin-*.tar.gz | tail -1)" + +# Verify +python3 -c "import marvin; print('Marvin OK')" +``` + +The VM workflow suites also need a venv with pinned dependencies. From the repo root: + +```bash +bash test/integration/plugins/ontap/setup_env.sh +``` + +The script is safe to re-run. It creates `test/integration/plugins/ontap/.venv` when that directory is missing, then installs into it: + +- Marvin (from `MARVIN_ARCHIVE` when set, otherwise from `tools/marvin` after generating `commands.xml`) +- the pins in `requirements.txt`: `setuptools>=40.3.0,<81`, `pyvmomi==8.0.2.0.1`, and `paramiko>=3.4,<4` + +`run_tests.sh` uses that venv automatically. + +#### Paramiko (guest SSH) + +`nfs3/instance/test_vm_volume_attach.py` and `iscsi/instance/test_vm_volume_attach.py` log into the guest through `guest_ssh.py`. The lab template is CentOS 5.5, and its sshd only offers SHA-1 key exchange (`diffie-hellman-group14-sha1`) and `ssh-rsa` host keys. Paramiko 4.x removed SHA-1, so those logins fail unless the venv has Paramiko 3.x. + +`setup_env.sh` installs the pin from `requirements.txt`: + +```bash +test/integration/plugins/ontap/.venv/bin/pip install -r test/integration/plugins/ontap/requirements.txt +``` + +`guest_ssh.check_paramiko()` checks for `diffie-hellman-group14-sha1` before the first guest connection and tells you to re-run `setup_env.sh` when the active interpreter has Paramiko 4. Guest login itself comes from `ontap.cfg` (`cloudstack.templateUsername` / `cloudstack.templatePassword`). + +Confirm the venv before a VM workflow run: + +```bash +test/integration/plugins/ontap/.venv/bin/python -c \ + "import paramiko; print(paramiko.__version__); assert 'diffie-hellman-group14-sha1' in paramiko.Transport._kex_info" +``` + +--- + +## Configuration — `ontap.cfg` + +`ontap.cfg` is a JSON file that tells Marvin where CloudStack and ONTAP are. **Never commit real credentials.** + +Key sections: + +```json +{ + "mgtSvr": [{ "mgtSvrIp": "", "port": 8096, "user": "admin", "passwd": "password" }], + "dbSvr": { "dbSvr": "", "port": 3306, "user": "cloud", "passwd": "cloud" }, + "ontap": { "storageIP": "", "svmName": "", "username": "admin", "password": "" } +} +``` + +The test classes read `storageIP`, `svmName`, `username`, and `password` from the `ontap` section at runtime. **No credentials appear in test code.** + +--- + +## Running the tests + +**Always run from the repo root.** The recommended entry point is [`run_tests.sh`](run_tests.sh), which runs suites sequentially (required for shared test state) and writes unified reports under `results/`. + +### Protocol batch commands (recommended) + +Run all suites for one protocol in a single batch, then inspect consolidated results: + +```bash +# iSCSI only — 6 suites, ~40–60 min +bash test/integration/plugins/ontap/run_tests.sh iscsi + +# NFS3 only — 6 suites, ~40–60 min +bash test/integration/plugins/ontap/run_tests.sh nfs3 + +# Full plugin validation: iSCSI batch, then NFS3 batch +bash test/integration/plugins/ontap/run_tests.sh both + +# Default (setup_zone + iscsi + nfs3; excludes cleanup_zone) +bash test/integration/plugins/ontap/run_tests.sh +bash test/integration/plugins/ontap/run_tests.sh all + +# Template-cache negative / boundary suites only +# (the template-cache happy path runs inside the VM attach suites) +bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache_negative +bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache_negative +``` + +Each protocol batch runs suites in this order: pool lifecycle → pool with volumes → volume lifecycle → zone-scoped pool → VM attach (includes template cache seed / reuse / survive) → template cache negative (last). + +| Command | What it runs | +| --------- | -------------- | +| `run_tests.sh iscsi` | All 6 iSCSI suites + unified iSCSI report | +| `run_tests.sh nfs3` | All 6 NFS3 suites + unified NFS3 report | +| `run_tests.sh both` | iSCSI batch, then NFS3 batch + combined report | +| `run_tests.sh all` | `setup_zone`, then `both` (iSCSI before NFS3) | +| `run_tests.sh nfs3_workflow` | Single suite by tag (unchanged) | +| `run_tests.sh nfs3_template_cache_negative` | NFS3 template-cache boundary/negative suite | +| `run_tests.sh iscsi_template_cache_negative` | iSCSI template-cache boundary/negative suite | +| `run_tests.sh setup_zone` | Zone setup only | +| `run_tests.sh cleanup_zone` | Zone teardown (manual; destructive) | + +### Results layout + +After a protocol batch, artifacts are under `test/integration/plugins/ontap/results/`: + +``` +results/-iscsi/ + run.meta.json # protocol, timestamps, per-suite exit codes + summary.tsv # all tests (tab-separated) + summary.json # machine-readable aggregate (CI-friendly) + summary.txt # human-readable TEST SUMMARY + suites/ + iscsi_workflow/ + stdout.log + results.txt # copy of Marvin results + runinfo.txt + ... +``` + +Symlinks: `results/latest-iscsi`, `results/latest-nfs3`, `results/latest-both`. + +For `both` / `all`, the parent folder `results/-both/` contains `iscsi/` and `nfs3/` sub-batches plus a combined `summary.txt` at the top level. + +Marvin also writes raw logs to `/tmp/MarvinLogs//` during execution. + +### Manual nose commands + +```bash +# Single suite (e.g. NFS3 pool lifecycle) +PYTHONPATH=test/integration/plugins/ontap \ +test/integration/plugins/ontap/.venv/bin/python \ + test/integration/plugins/ontap/nose_compat.py --with-marvin \ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \ + test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v + +# By tag +PYTHONPATH=test/integration/plugins/ontap \ +test/integration/plugins/ontap/.venv/bin/python \ + test/integration/plugins/ontap/nose_compat.py --with-marvin \ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \ + -a tags=iscsi_workflow \ + test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py -v +``` + +> **Important:** `PYTHONPATH=test/integration/plugins/ontap` is always required. Test files import `ontap_test_base` from the parent directory. + +> **Python 3.10+:** Always invoke legacy nose through `nose_compat.py`. The wrapper restores the `collections.Callable` alias that nose test discovery requires. + +> **Single-host lab:** Suites within a batch run **sequentially** (not in parallel). iSCSI completes before NFS3 starts in `both`/`all` so the one KVM host is not shared across protocol operations simultaneously. + +--- + +## Code structure — how a test file is organised + +Every test file follows the same layout: + +``` +1. Apache 2.0 license header +2. Module docstring ← workflow summary, prerequisites, run command +3. Imports +4. TestData class ← holds all config values read from ontap.cfg; builds the + createStoragePool command parameters +5. Test class (extends OntapTestBase) + ├── Class-level state attributes (pool, volume, vm, etc.) initialised to None + ├── setUpClass() ← connects to CloudStack; creates a test account and + │ disk offering; resolves zone/cluster/hosts + ├── tearDownClass() ← best-effort cleanup: deletes pool (forced=True), + │ volume, account, disk offering + ├── Helper methods ← _create_pool(), _create_volume(), _poll_pool_state(), + │ _lun_maps() (iSCSI only), etc. + └── test_01 … test_N ← sequential, numbered test methods +``` + +### Key patterns to know + +**Sequential state sharing — always use `self.__class__.`** + +Tests share state via class attributes, never instance attributes: +```python +# Correct +self.__class__.pool = pool +pool = self.__class__.pool + +# Wrong — state is lost between test method invocations +self.pool = pool +``` + +**Guard assertion at the start of every test that depends on a previous one** + +Every test in a sequential workflow starts with an assertion that the previous step's resource exists. This produces a clear, readable failure message instead of a confusing `AttributeError`: +```python +def test_05_enable_storage_pool(self): + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") +``` + +Isolated negative tests own everything they create, so they carry no such guard. In the pool lifecycle suites the two create-rejection negatives are numbered `test_01` (no aggregate space) and `test_02` (invalid capacitybytes) so that a bad create fails within a minute rather than after the full workflow. + +**Creating a storage pool — always use indexed `details[N].key` syntax** + +The CloudStack API for `createStoragePool` requires plugin details to be passed as indexed parameters. **Never call `StoragePool.create()` directly** — it does not support this syntax: +```python +count = 1 +for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 +``` + +**Polling for async state changes** + +CloudStack operations are asynchronous. Use `_poll_pool_state()` rather than reading state immediately after an API call: +```python +result = self._poll_pool_state(pool.id, "Maintenance", timeout=120) +self.assertEqual(result.state, "Maintenance") +``` + +**Shared iSCSI igroups** + +iSCSI igroups are named from the host UUID and SVM, not from the storage pool. +Each iSCSI suite snapshots existing host igroups during `setUpClass()` and +checks that pool-only operations preserve that baseline. Tests may therefore +run while another ONTAP iSCSI pool uses the same SVM. The negative tests that +deliberately delete igroups still require exclusive SVM use and skip when +another ONTAP pool is present. + +--- + +## Shared base — `ontap_test_base.py` + +`OntapTestBase` provides everything individual test classes inherit: + +| What | Purpose | +| ------ | --------- | +| `_setup_cloudstack_resources()` | Creates a test account, domain, disk offering; resolves zone/cluster/hosts | +| `tearDownClass()` | Best-effort cleanup: deletes pool (forced=True), volume, disk offering, account | +| `_poll_pool_state(pool_id, state, timeout)` | Polls `listStoragePools` until pool reaches the target state | +| `_create_volume(pool_id)` | Creates a CloudStack data volume on the given pool | +| `_delete_pool(pool_id, forced)` | Enters Maintenance then calls `deleteStoragePool` | +| `_parse_pool_details(pool)` | Extracts key→value pairs from the pool's `details` list | +| `OntapRestClient` | Thin HTTPS client for ONTAP REST API calls | + +### `OntapRestClient` methods at a glance + +| Method | What it checks | Used in | +| -------- | --------------- | --------- | +| `get_volume(name)` | FlexVol existence and state | All suites | +| `get_export_policy(name)` | NFS export policy existence | NFS3 suites | +| `get_data_lifs(svm_name)` | NFS data LIF count | NFS3 pool lifecycle | +| `get_igroup(svm_name, name)` | iSCSI igroup existence and initiator list | iSCSI suites | +| `list_luns_in_volume(svm_name, vol_name)` | LUNs present in a FlexVol | iSCSI volume/instance suites | +| `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI pool-with-volumes/instance suites | +| `list_files_in_volume(vol_name)` | Files inside a FlexVol | NFS3 instance and pool lifecycle suites | +| `write_file_in_volume(vol_name, filename, size_bytes)` | Write incompressible data into a FlexVol file (no VM) | NFS3/iSCSI pool lifecycle suites | +| `delete_file_in_volume(vol_name, filename)` | Remove a FlexVol file | NFS3/iSCSI pool lifecycle suites | + +--- + +## Test suite quick reference + +| Suite | File | Tests | What it covers | +| ------- | ------ | ------- | --------------- | +| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 21 | Create rejects for aggregate space, invalid capacitybytes, and a duplicate FlexVol name; grow to 300 TiB, reject grow past 300 TiB, safe shrink, disable, resize while disabled, enable, maintenance, grow and shrink in maintenance, delete, then re-create with a volume, reject shrink below used capacity, deploy a VM and attach the volume, cleanup, plus empty-pool deletion with a pre-deleted FlexVol or export policy | +| NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 12 | Lifecycle with a live volume, deletion with a pre-deleted FlexVol or export policy, and host cases for an inactive libvirt pool, a read-only NFS mount, and a deleted mount point | +| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 5 | Zone-scoped pool create, grow and shrink with CloudStack/ONTAP/KVM verification, disable, enable, and delete | +| NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | +| NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 29 | Full VM lifecycle with stopped-VM qcow2 grow, guest SSH, volume and VM snapshots; ROOT on tagged pool seeds/reuses template cache, which survives VM delete | +| NFS3 Template Cache Negative | `nfs3/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | +| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 21 | Create rejects for aggregate space, invalid capacitybytes, and a duplicate FlexVol name; grow to 300 TiB, reject grow past 300 TiB, safe shrink, disable, resize while disabled, enable, maintenance, grow and shrink in maintenance, delete + igroups, then re-create with a volume (LUN), reject shrink below used capacity, deploy a VM and attach the volume, cleanup, plus empty-pool deletion with a pre-deleted FlexVol or igroups | +| iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 13 | Lifecycle with a live LUN, deletion with a pre-deleted FlexVol or igroups, maintenance with pre-deleted LUN maps, and host cases for a logged-out iSCSI session, an existing session during volume delete, and a replaced by-path symlink | +| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 5 | Zone-scoped pool create, grow and shrink with CloudStack/ONTAP verification, disable, enable, and delete | +| iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | +| iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 29 | Full VM lifecycle; stopped-VM LUN grow, LUN-map checks, guest SSH, volume and VM snapshots; ROOT on tagged pool seeds/reuses `cs_tmpl_*` LUN cache | +| iSCSI Template Cache Negative | `iscsi/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | + +For the goal, dependencies, and exact success criteria of every individual test, see [TEST_CASES.md](TEST_CASES.md). + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +| --------- | ------------- | ----- | +| `paramiko … has no SHA-1 key exchange` / guest SSH fails on CentOS 5.5 | Paramiko 4.x is installed; the guest sshd only offers SHA-1 | Re-run `bash test/integration/plugins/ontap/setup_env.sh` and run tests with `test/integration/plugins/ontap/.venv` (`paramiko>=3.4,<4`) | +| `ModuleNotFoundError: No module named 'ontap_test_base'` | Missing `PYTHONPATH` prefix | Prefix every run with `PYTHONPATH=test/integration/plugins/ontap` | +| `Marvin Init Failed` | CloudStack API unreachable | Check `mgtSvrIp:8096` is reachable; restart `cloudstack-management` | +| `Lost connection to MySQL` | MySQL not accepting remote connections | Enable remote MySQL access (see Prerequisites §3) | +| `sh: python: command not found` (repeated) | Marvin internal call — harmless on macOS | Ignore; Marvin Init still succeeds | +| Pool state never reaches `Maintenance` | KVM agent not responding | Check `cloudstack-agent` on KVM host; verify host is connected in CloudStack UI | +| iSCSI `test_24` error 530 | KVM guest does not ACK SCSI hot-unplug | Detach of the data LUN from a running VM; see TEST_CASES.md Suite 10 `test_24_detach_volume_from_vm` | +| ONTAP REST `401 Unauthorized` | Wrong credentials in `ontap.cfg` | Verify `username`/`password` under `ontap` section | +| `ISCSI protocol is not enabled on SVM` / many pool tests EXCEPTION then FAIL | SVM iSCSI service off or no `data_iscsi` LIF | Enable iSCSI on the SVM and assign `default-data-iscsi` (or `default-data-blocks`) to at least one data LIF; `run_tests.sh` now fails this check before Marvin starts | +| `No ready KVM user template available` | Template still downloading | Re-run `setup_zone` (step 12 waits for template readiness); or wait in CloudStack UI | +| `setup_zone` steps 11–12 slow on first run | System VMs and template download after zone enable | Normal — first run may take up to ~60 min; re-runs pass quickly when already ready | +| `cleanup_zone` pool delete fails | Pool stuck in Maintenance or KVM NFS mount stale | Re-run cleanup; check host connectivity; manually `umount /mnt/` on KVM if needed | +| `deleteZone failed` after cleanup | VMs, pools, or hosts still present in zone | Re-run `cleanup_zone`; check CloudStack UI for remaining resources | + +--- + +## Adding new test cases + +1. Pick the existing file closest to what you need and copy its structure. +2. Read `ontap_test_base.py` for the exact method signatures you can reuse. +3. Copy the `_create_pool()` helper from an existing file that matches your protocol — **never** call `StoragePool.create()`. +4. Number your methods `test_01`, `test_02`, … and add `@attr(tags=[""], required_hardware=True)` to each. +5. Use `self.__class__.` for all state shared between test methods. +6. Syntax-check before the first full run: `python3 -m py_compile .py` +7. Add your test cases to [TEST_CASES.md](TEST_CASES.md). diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md new file mode 100644 index 000000000000..4145e3c5eb79 --- /dev/null +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -0,0 +1,353 @@ + + +# ONTAP Integration Test Cases + +Complete reference for all 151 test cases across 12 test suites. +Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. + +--- + +## How to read the tables + +| Column | Meaning | +| -------- | --------- | +| **Test method** | Exact Python method name | +| **Goal** | What CloudStack workflow step is being exercised | +| **Depends on** | Which earlier tests must have passed (class state they consume) | +| **CloudStack success criteria** | What the CS API must return for the test to pass | +| **ONTAP success criteria** | What the ONTAP REST API must show for the test to pass | +| **Type** | `positive` = happy path, `negative` = tests a rejection/error condition, `cleanup` = teardown step | + +--- + +## Suite 1 — NFS3 Pool Lifecycle + +**File:** `nfs3/pool/test_pool_lifecycle.py` +**Class:** `TestOntapNFS3PrimaryStorageWorkflow` +**Tag:** `nfs3_workflow` +**Total:** 21 tests | **Scope:** cluster-scoped NFS3 pool. tests 01, 02, 20, and 21 are isolated; test 04 reuses the FlexVol from test 03 + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_reject_create_when_no_aggregate_space` | Reject pool creation when requested capacity exceeds every assigned online aggregate's available space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no CS pool created | No FlexVol created | negative | +| 02 | `test_02_reject_create_with_invalid_capacitybytes_values` | Reject pool creation when `capacitybytes` has invalid values | isolated | `CloudstackAPIException` for each value containing `invalid value` for `capacitybytes`, or `must be at least` when the plugin validates the size; no CS pool created | No FlexVol created | negative | +| 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | +| 04 | `test_04_reject_create_when_flexvol_name_exists` | Reject a second pool create when the FlexVol from test 03 already exists on ONTAP | test_03 | `CloudstackAPIException`; only the test_03 pool remains, still `Up` | Existing FlexVol stays `online`; export policy still present | negative | +| 05 | `test_05_grow_storage_pool` | Grow the original empty pool to 300 TiB | test_03 | `capacitybytes` reaches 300 TiB; pool stays `Up` | FlexVol `space.size` reaches 300 TiB; remains `online`; export policy unchanged | positive | +| 06 | `test_06_reject_grow_above_max_size` | Reject a grow past the 300 TiB ONTAP FlexVol maximum (overshoots by 1 GiB; asserts ONTAP's autosize-maximum error, not a generic failure such as aggregate space) | test_05 | `CloudstackAPIException`; `capacitybytes` stays at 300 TiB; pool stays `Up` | FlexVol `space.size` unchanged; export policy unchanged | negative | +| 07 | `test_07_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_05 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol `space.size` reaches the safe target; remains `online`; export policy unchanged | positive | +| 08 | `test_08_disable_storage_pool` | Disable the pool (admin operation) | test_07 | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | +| 09 | `test_09_grow_and_shrink_storage_pool_while_disabled` | Grow then shrink the pool back while it is Disabled | test_08 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; KVM hosts report each resized NFS capacity; `pool.state` stays `"Disabled"` throughout | FlexVol `space.size` reaches each target and stays `online`; export policy still present and still covers every host IP | positive | +| 10 | `test_10_enable_storage_pool` | Re-enable the pool | test_09 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 11 | `test_11_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_10 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | +| 12 | `test_12_grow_and_shrink_storage_pool_in_maintenance` | Grow then shrink the pool back while it is in Maintenance | test_11 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; `pool.state` stays `"Maintenance"` throughout | FlexVol `space.size` reaches each target and stays `online`; export policy still present | positive | +| 13 | `test_13_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_12 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 14 | `test_14_delete_pool_from_maintenance` | Enter maintenance then permanently delete the original pool | test_13 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol and export policy deleted | positive | +| 15 | `test_15_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack data volume | test_14 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; export policy present | positive | +| 16 | `test_16_create_vm_and_attach_volume` | Deploy a VM and attach the ONTAP data volume to it | test_15 | VM reaches `Running`; volume reports `virtualmachineid` equal to the VM id; VM still `Running` after attach; pool capacity unchanged | FlexVol still `online`; volume data file materialised in the FlexVol (NFS3 creates it lazily at attach) | positive | +| 17 | `test_17_reject_shrink_below_used_capacity` | Reject shrink below the used space of the VM-attached volume, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API when the VM-attached thin volume leaves used space under the minimum, and removes it before returning | test_16 (`pool`, `volume`, `vm`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and export policy unchanged; used space stays above the FlexVol minimum | negative | +| 18 | `test_18_grow_and_shrink_pool_with_vm_attached` | Grow then shrink the pool back while the volume is attached to the running VM | test_16 (`vm`, `volume`) | Both resizes reach the requested capacity; pool stays `Up`; VM stays `Running` and keeps `virtualmachineid` on the volume | FlexVol reaches each size and stays `online`; export policy still covers every host IP | positive | +| 19 | `test_19_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_18 | VM destroyed; volume and pool no longer listed | FlexVol and export policy deleted | cleanup | +| 20 | `test_20_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; export policy cleaned up | negative | +| 21 | `test_21_delete_pool_with_export_policy_predeleted` | Delete an empty pool after its export policy was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; export policy remains absent | negative | + +--- + +## Suite 2 — NFS3 Pool with Volumes + +**File:** `nfs3/pool/test_pool_with_volumes.py` +**Class:** `TestOntapNFS3PoolWithVolumes` +**Tag:** `nfs3_with_volumes` +**Total:** 12 tests | **Scope:** cluster-scoped NFS3 pool with a live CloudStack volume, plus isolated negative workflows + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_pool_and_volume` | Create NFS3 pool and immediately allocate a data volume | setUpClass | `pool.state == "Up"`, volume object non-None | FlexVol `online`; export policy present | positive | +| 02 | `test_02_disable_pool_volume_survives` | Disable pool while a volume exists — volume must survive | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed in `listVolumes` | FlexVol still `online` | positive | +| 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume present | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online` | positive | +| 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance while volume present | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online` | positive | +| 05 | `test_05_cancel_maintenance_with_volume` | Cancel maintenance with volume — verifies the NFS3 cancel-maintenance fix | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online` | positive | +| 06 | `test_06_forced_false_delete_rejected` | Attempt to delete pool (forced=False) with volume present — must be rejected | test_05 | `deleteStoragePool(forced=False)` raises `CloudstackAPIException`; pool still listed in `Maintenance` state | FlexVol still `online`; no ONTAP objects removed | negative | +| 07 | `test_07_force_delete_pool_and_cleanup` | Cancel maintenance, delete volume, then force-delete pool | test_06 | Pool no longer listed; volume no longer listed | FlexVol deleted; export policy deleted | cleanup | +| 08 | `test_08_delete_pool_with_volume_flexvol_missing` | Force-delete a pool with a CS volume after its FlexVol was removed directly from ONTAP | isolated | Pool removed; leftover volume record cleaned | FlexVol remains absent | negative | +| 09 | `test_09_delete_pool_with_volume_export_policy_missing` | Force-delete a pool with a CS volume after its export policy was removed directly from ONTAP | isolated | Pool removed; leftover volume record cleaned | FlexVol deleted; export policy remains absent | negative | +| 10 | `test_10_libvirt_pool_inactive` | Libvirt pool for this storage pool is inactive on the KVM host | isolated | Pool reaches Maintenance; volume remains | FlexVol stays online; export policy remains | negative | +| 11 | `test_11_nfs_mount_read_only` | NFS mount for this pool is read-only on the KVM host | isolated | Pool reaches Maintenance; writes to the mount fail | FlexVol stays online; export policy remains | negative | +| 12 | `test_12_nfs_mount_point_deleted` | NFS mount point for this pool is missing on the KVM host | isolated | Pool reaches Maintenance with its mount point absent | FlexVol stays online; export policy remains | negative | + +--- + +## Suite 3 — NFS3 Zone-Scoped Pool + +**File:** `nfs3/pool/test_zone_scoped_pool.py` +**Class:** `TestOntapZoneScopedPool` +**Tag:** `zone_pool` +**Total:** 5 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped NFS3 pool; CloudStack calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; export policy exists and contains **every** cluster host IP; at least one NFS data LIF present | positive | +| 02 | `test_02_grow_and_shrink_zone_scoped_pool` | Grow the zone-scoped pool, then shrink it to its original size | test_01 | Both capacity updates reach the requested size; pool remains `Up`; KVM hosts report each resized NFS capacity | FlexVol reaches each requested size and remains `online`; export policy still covers every host IP | positive | +| 03 | `test_03_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_02 | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | +| 04 | `test_04_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_03 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | +| 05 | `test_05_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_04 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | + +--- + +## Suite 4 — NFS3 Volume Lifecycle + +**File:** `nfs3/volume/test_volume_lifecycle.py` +**Class:** `TestOntapNFS3VolumeLifecycle` +**Tag:** `nfs3_volume` +**Total:** 5 tests | **Scope:** NFS3 CloudStack volume create/delete semantics (NFS3 volumes are metadata-only in CS) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_pool_and_volume` | Create NFS3 pool and allocate a CloudStack data volume | setUpClass | `pool.state == "Up"`; volume object non-None | FlexVol `online` after volume allocation; export policy present — **no new ONTAP object per volume** (FlexVol is shared) | positive | +| 02 | `test_02_delete_volume` | Delete the CS data volume — for NFS3 only the CS record is removed | test_01 (`pool`, `volume`) | Volume no longer listed in `listVolumes` | FlexVol still `online` and **unaffected**; export policy still present | positive | +| 03 | `test_03_recreate_volume_for_delete_tests` | Re-create a volume on the pool (setup for negative tests 04–05) | test_02 | New volume object non-None | FlexVol still `online` | positive | +| 04 | `test_04_forced_false_delete_with_volume_fails` | Enter maintenance then attempt `deleteStoragePool(forced=False)` while volume exists — must be rejected | test_03 (`pool`, `volume`) | `deleteStoragePool(forced=False)` raises `CloudstackAPIException`; pool still in `Maintenance` state | No ONTAP objects removed | negative | +| 05 | `test_05_delete_volume_and_force_delete_pool` | Delete volume from Maintenance, then force-delete pool | test_04 | Volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | positive | + +--- + +## Suite 5 — NFS3 VM + Volume Attach + +**File:** `nfs3/instance/test_vm_volume_attach.py` +**Class:** `TestOntapVMVolumeAttach` +**Tag:** `vm_volume_workflow` +**Total:** 29 tests | **Scope:** end-to-end — NFS3 pool, data volume resize, running VM (ROOT on the ONTAP pool via a tagged compute offering), primary template cache seed / reuse / survive VM delete, attach/detach, guest SSH, volume snapshots, and VM snapshots. Guest login uses `guest_ssh.py` (paramiko) against the CentOS 5.5 template. + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_nfs3_pool` | Create NFS3 ONTAP primary storage pool tagged `-tmpl-cache` | setUpClass (zone, cluster, template, tagged SO) | `pool.state == "Up"` | FlexVol `online`; export policy present | positive | +| 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume on the ONTAP pool | test_01 (`pool`) | Volume non-None and listed in `listVolumes` | FlexVol still `online` | positive | +| 03 | `test_03_deploy_vm` | Deploy a VM with the tagged SO — ROOT on ONTAP; seeds template cache | test_02 (`pool`, `volume`) | `vm.state == "Running"`; ROOT `storageid` = pool; `template_spool_ref` Ready/DOWNLOADED | Cache file present at spool `install_path` | positive | +| 03a | `test_03a_deploy_second_vm_reuses_template_cache` | Deploy VM-2 — reuses cache | test_03 | VM-2 Running; ROOT on pool; still exactly one `template_spool_ref` | Same cache file (no second cache) | positive | +| 03b | `test_03b_expunge_second_vm_template_cache_survives` | Expunge VM-2 — cache must remain (lazy GC) | test_03a | spool_ref still Ready | Cache file still present | positive | +| 04 | `test_04_attach_volume_to_vm` | Attach the ONTAP data volume to the running VM (hot-plug) | test_03b (`vm`, `volume`) | `volume.virtualmachineid == vm.id`; `attachVolume` job succeeds | FlexVol `online`; after attach, a data file matching volume UUID present in FlexVol (`list_files_in_volume`) | positive | +| 05 | `test_05_stop_vm_export_retained` | Stop the running VM with volume attached | test_04 | `vm.state == "Stopped"` | FlexVol still `online`; NFS export policy still present | positive | +| 06 | `test_06_grow_stopped_vm_volume` | Grow the attached volume while the VM is stopped | test_05 | Volume is `Ready` at the requested byte size | `qemu-img info` reports the requested virtual size; ONTAP file remains present | positive | +| 07 | `test_07_shrink_rejected` | Attempt to shrink the grown qcow2 volume with `shrinkok=true` | test_06 | Resize is rejected; grown size remains unchanged | qcow2 virtual size remains unchanged | negative | +| 08 | `test_08_start_vm_volume_accessible` | Start the stopped VM | test_07 | `vm.state == "Running"`; KVM data-disk capacity equals target | FlexVol still `online` | positive | +| 09 | `test_09_open_ssh_and_write_root_data_files` | Port-forward SSH to the guest and write ROOT and DATA marker files | test_08 (`vm`, `volume`, guest network) | ROOT and DATA both on the ONTAP pool; DATA still attached; SSH login succeeds; both markers read back | FlexVol still `online` | positive | +| 10 | `test_10_root_volume_snapshot_create` | VS-01: snapshot the ROOT volume of the running VM | test_09 | Snapshot `BackedUp`; VM stays `Running` | One new FlexVol snapshot | positive | +| 11 | `test_11_data_volume_snapshot_create` | VS-03: snapshot the attached DATA volume (kept for test_14) | test_10 | Snapshot `BackedUp`; VM stays `Running` | One new FlexVol snapshot, kept until test_14 | positive | +| 12 | `test_12_volume_revert_running_vm_rejected` | VS-19: revert a volume snapshot while the VM is Running | test_11 | Revert is rejected | FlexVol unchanged | negative | +| 13 | `test_13_root_volume_revert` | VS-11: write A, snapshot ROOT, write B, stop, revert, start | test_12 | After revert and start, ROOT marker is A | New FlexVol snapshot for the snapshot; revert snapshot removed in tearDown | positive | +| 14 | `test_14_data_volume_revert` | Snapshot DATA again, revert to the test_11 snapshot then to this one, then delete both while the VM is running | test_11, test_13 | Older revert restores the test_11 DATA marker and the later snapshot stays listed; later revert restores its marker; both snapshots are then gone and the VM stays `Running` | Both FlexVol snapshots remain through the older revert, then both are removed | positive | +| 15 | `test_15_vm_snapshot_create_running` | VMS-01: VM snapshot of the running VM (kept for test_17) | test_14 | VM snapshot `Ready`; VM stays `Running` | One new FlexVol snapshot | positive | +| 16 | `test_16_vm_snapshot_create_stopped` | VMS-03: stop, VM snapshot, start (kept for test_17) | test_15 | VM snapshot `Ready` | One new FlexVol snapshot | positive | +| 17 | `test_17_vm_snapshot_s1_running_s2_stopped_revert` | VMS-09: revert to the test_15 snapshot, then to the test_16 snapshot, then delete both while the VM is running | test_15, test_16 | Each revert finishes; guest file matches that snapshot; both VM snapshots are then gone and the VM stays `Running` | FlexVol snapshots survive the reverts, then both are removed | positive | +| 18 | `test_18_vm_snapshot_duplicate_requests` | VMS-53: two VM snapshot requests for the same VM at once | test_17 | Both snapshots `Ready` (a rejected request is retried) | Two new FlexVol snapshots | positive | +| 19 | `test_19_deploy_second_vm` | Deploy VM-B whose ROOT disk is on the same ONTAP pool | test_18 | VM-B `Running`; ROOT `storageid` = pool | ROOT file present on the shared FlexVol | positive | +| 20 | `test_20_volume_revert_leaves_other_vm_untouched` | VS-28: revert VM-A's ROOT | test_19 | VM-A's ROOT marker is restored; VM-B's file is unchanged | FlexVol stays `online` | positive | +| 21 | `test_21_concurrent_vm_snapshots_two_vms` | VMS-54: VM snapshots of both VMs at the same time | test_20 | Both VM snapshots `Ready` | Two new FlexVol snapshots | positive | +| 22 | `test_22_concurrent_volume_snapshot_and_revert` | Volume snapshot of VM-A's ROOT while VM-B's ROOT reverts | test_21 | Snapshot `BackedUp` and revert both finish (retried on error 530) | New FlexVol snapshot for the volume snapshot | positive | +| 23 | `test_23_concurrent_vm_snapshot_and_revert` | VM snapshot of VM-A while VM-B reverts to its VM snapshot | test_22 | Both operations finish; VM-A snapshot `Ready` | New FlexVol snapshot for VM-A | positive | +| 24 | `test_24_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM (TDS Detach NFS3) | test_23 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | +| 25 | `test_25_grow_detached_volume` | Grow the detached, materialized data volume | test_24 | API and DB size equal target | qcow2 virtual size equals target | positive | +| 26 | `test_26_destroy_vm_with_vm_snapshot` | VMS-38: expunge VM-B while it still has a VM snapshot | test_19 (VM-B) | VM-B no longer listed | VM-B's FlexVol snapshot removed | positive | +| 27 | `test_27_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, force-delete pool | test_25, test_26 | VM no longer listed; spool_ref still Ready after VM expunge; volume no longer listed; pool no longer listed | Cache file present after VM expunge; FlexVol deleted; export policy deleted | cleanup | + +--- + +## Suite 6 — iSCSI Pool Lifecycle + +**File:** `iscsi/pool/test_pool_lifecycle.py` +**Class:** `TestOntapISCSIPoolLifecycle` +**Tag:** `iscsi_workflow` +**Total:** 21 tests | **Scope:** cluster-scoped iSCSI pool. tests 01, 02, 20, and 21 are isolated; test 04 reuses the FlexVol from test 03 + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_reject_create_when_no_aggregate_space` | Reject pool creation when requested capacity exceeds every assigned online aggregate's available space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no CS pool created | No FlexVol created | negative | +| 02 | `test_02_reject_create_with_invalid_capacitybytes_values` | Reject pool creation when `capacitybytes` has invalid values | isolated | `CloudstackAPIException` for each value containing `invalid value` for `capacitybytes`, or `must be at least` when the plugin validates the size; no CS pool created | No FlexVol created | negative | +| 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | +| 04 | `test_04_reject_create_when_flexvol_name_exists` | Reject a second pool create when the FlexVol from test 03 already exists on ONTAP | test_03 | `CloudstackAPIException`; only the test_03 pool remains, still `Up` | Existing FlexVol stays `online` | negative | +| 05 | `test_05_grow_storage_pool` | Grow the original empty pool to 300 TiB | test_03 | `capacitybytes` reaches 300 TiB; pool stays `Up` | FlexVol reaches 300 TiB and remains `online`; igroups remain present | positive | +| 06 | `test_06_reject_grow_above_max_size` | Reject a grow past the 300 TiB ONTAP FlexVol maximum (overshoots by 1 GiB; asserts ONTAP's autosize-maximum error, not a generic failure such as aggregate space) | test_05 | `CloudstackAPIException`; `capacitybytes` stays at 300 TiB; pool stays `Up` | FlexVol `space.size` unchanged; igroups unchanged | negative | +| 07 | `test_07_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_05 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol reaches the safe target and remains `online`; igroups remain present | positive | +| 08 | `test_08_disable_storage_pool` | Disable the pool | test_07 | `pool.state == "Disabled"` | FlexVol still `online` | positive | +| 09 | `test_09_grow_and_shrink_storage_pool_while_disabled` | Grow then shrink the pool back while it is Disabled | test_08 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; `pool.state` stays `"Disabled"` throughout | FlexVol `space.size` reaches each target and stays `online`; per-host igroups still present | positive | +| 10 | `test_10_enable_storage_pool` | Re-enable the pool | test_09 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 11 | `test_11_enter_maintenance_mode` | Put pool into maintenance | test_10 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | +| 12 | `test_12_grow_and_shrink_storage_pool_in_maintenance` | Grow then shrink the pool back while it is in Maintenance | test_11 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; `pool.state` stays `"Maintenance"` throughout | FlexVol `space.size` reaches each target and stays `online`; per-host igroups still present | positive | +| 13 | `test_13_cancel_maintenance_mode` | Cancel maintenance | test_12 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 14 | `test_14_enter_maintenance_and_delete_pool` | Enter maintenance then delete the original pool | test_13 | Pool no longer listed | FlexVol and all host igroups deleted | positive | +| 15 | `test_15_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack volume | test_14 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; at least one LUN present | positive | +| 16 | `test_16_create_vm_and_attach_volume` | Deploy a VM and attach the ONTAP data volume (LUN) to it | test_15 | VM reaches `Running`; volume reports `virtualmachineid` equal to the VM id; VM still `Running` after attach | FlexVol still `online`; LUN still present; per-host igroups unchanged | positive | +| 17 | `test_17_reject_shrink_below_used_capacity` | Reject shrink below the used space of the VM-attached LUN, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API when the VM-attached thin volume leaves used space under the minimum, and removes it before returning | test_16 (`pool`, `volume`, `vm`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and LUN uuid/name list unchanged; used space stays above the FlexVol minimum | negative | +| 18 | `test_18_grow_and_shrink_pool_with_vm_attached` | Grow then shrink the pool back while the volume (LUN) is attached to the running VM | test_16 (`vm`, `volume`) | Both resizes reach the requested capacity; pool stays `Up`; VM stays `Running` and keeps `virtualmachineid` on the volume | FlexVol reaches each size and stays `online`; LUN still present | positive | +| 19 | `test_19_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_18 | VM destroyed; volume and pool no longer listed | LUN, FlexVol, and igroups deleted | cleanup | +| 20 | `test_20_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; igroups cleaned up | negative | +| 21 | `test_21_delete_pool_with_igroups_predeleted` | Delete an empty pool after host igroups were removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; igroups remain absent | negative | + +--- + +## Suite 7 — iSCSI Pool with Volumes + +**File:** `iscsi/pool/test_pool_with_volumes.py` +**Class:** `TestOntapISCSIPoolWithVolumes` +**Tag:** `iscsi_workflow` +**Total:** 13 tests | **Scope:** cluster-scoped iSCSI pool with a live CloudStack volume (LUN), plus isolated negative workflows + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_pool_and_volume` | Create iSCSI pool and allocate a data volume (creates LUN) | setUpClass | `pool.state == "Up"`; volume non-None | FlexVol `online`; ≥1 LUN in FlexVol | positive | +| 02 | `test_02_disable_pool_volume_survives` | Disable pool with volume present | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance with volume | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 06 | `test_06_forced_false_delete_rejected` | Attempt `deleteStoragePool(forced=False)` with LUN-backed volume present — must be rejected | test_05 | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | +| 07 | `test_07_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_06 (`pool`, `volume`) | Volume gone; pool gone | LUN and FlexVol deleted; shared igroup baseline restored | cleanup | +| 08 | `test_08_delete_pool_with_volume_flexvol_missing` | Force-delete a pool with a CS volume after its FlexVol and LUN were removed directly | isolated | Pool removed; leftover volume record cleaned | FlexVol and LUN remain absent | negative | +| 09 | `test_09_delete_pool_with_volume_igroups_missing` | Force-delete a pool with a CS volume after host igroups were removed directly | isolated | Pool removed; leftover volume record cleaned | FlexVol deleted; igroups remain absent | negative | +| 10 | `test_10_enter_maintenance_lun_maps_predeleted` | Enter maintenance after LUN maps were removed directly on ONTAP | isolated | Pool reaches Maintenance | LUN maps remain absent | negative | +| 11 | `test_11_iscsi_session_logged_out` | Enter maintenance after only the test iSCSI session is logged out | isolated | Pool reaches Maintenance; volume remains | Test LUN remains | negative | +| 12 | `test_12_delete_volume_with_existing_iscsi_session` | Delete the test volume while its iSCSI session is already logged in | isolated | Volume is removed; no extra session is created | Test LUN is removed | negative | +| 13 | `test_13_corrupt_iscsi_by_path` | Delete the test volume after its by-path symlink is replaced with a regular file | isolated | Volume is removed; the planted file remains | Test LUN is removed | negative | + +--- + +## Suite 8 — iSCSI Zone-Scoped Pool + +**File:** `iscsi/pool/test_zone_scoped_pool.py` +**Class:** `TestOntapISCSIZoneScopedPool` +**Tag:** `iscsi_zone_pool` +**Total:** 5 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped iSCSI pool | setUpClass | `pool.state == "Up"` | FlexVol `online`; shared host igroups unchanged from suite-start baseline | positive | +| 02 | `test_02_grow_and_shrink_zone_scoped_pool` | Grow the zone-scoped pool, then shrink it to its original size | test_01 | Both capacity updates reach the requested size; pool remains `Up` | FlexVol reaches each requested size and remains `online`; shared igroups remain unchanged | positive | +| 03 | `test_03_disable_zone_scoped_pool` | Disable pool | test_02 | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | +| 04 | `test_04_enable_zone_scoped_pool` | Re-enable pool | test_03 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | +| 05 | `test_05_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_04 | Pool no longer listed | FlexVol and test-pool maps deleted; shared igroup baseline restored | positive | + +--- + +## Suite 9 — iSCSI Volume Lifecycle + +**File:** `iscsi/volume/test_volume_lifecycle.py` +**Class:** `TestOntapISCSIVolumeLifecycle` +**Tag:** `iscsi_volume` +**Total:** 5 tests | **Scope:** iSCSI CloudStack volume create/delete semantics (each CS volume maps to an ONTAP LUN) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_pool_and_volume` | Create iSCSI pool and allocate a data volume — a LUN is created inside the pool's FlexVol | setUpClass | `pool.state == "Up"`; volume non-None | FlexVol `online`; ≥1 LUN in FlexVol (`list_luns_in_volume`) | positive | +| 02 | `test_02_delete_volume` | Delete the volume — the LUN is removed from the FlexVol | test_01 (`pool`, `volume`) | Volume no longer listed | LUN no longer in FlexVol; FlexVol itself still `online` | positive | +| 03 | `test_03_recreate_volume_for_delete_tests` | Re-create a volume (LUN re-created) — setup for negative tests | test_02 | New volume non-None | LUN present in FlexVol again | positive | +| 04 | `test_04_forced_false_delete_with_volume_fails` | Enter maintenance then attempt `deleteStoragePool(forced=False)` with LUN present — must be rejected | test_03 (`pool`, `volume`) | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | +| 05 | `test_05_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_04 | Volume gone; pool gone | LUN and FlexVol deleted; shared igroup baseline restored | positive | + +--- + +## Suite 10 — iSCSI VM + Volume Attach + +**File:** `iscsi/instance/test_vm_volume_attach.py` +**Class:** `TestOntapVMVolumeAttachISCSI` +**Tag:** `iscsi_vm_workflow` +**Total:** 29 tests | **Scope:** end-to-end — iSCSI pool, data volume resize (LUN), running VM (ROOT on the ONTAP pool via a tagged compute offering), `cs_tmpl_` LUN cache seed / reuse / survive VM delete, attach/stop/start/detach, guest SSH, volume snapshots, and VM snapshots. LUN-map checks are limited to the data volume's LUN. Guest login uses `guest_ssh.py` (paramiko) against the CentOS 5.5 template. + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool tagged `-tmpl-cache` | setUpClass (tagged SO) | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; igroup per cluster host with host IQN | positive | +| 02 | `test_02_create_ontap_data_volume` | Allocate and grow a detached data volume | test_01 (`pool`) | API and DB size equal target | LUN exists and `space.size` equals target | positive | +| 03 | `test_03_deploy_vm` | Deploy VM with the tagged SO — ROOT on ONTAP; seeds `cs_tmpl_*`; verify 0 data-volume LUN-maps before attach | test_02 (`volume`) | `vm.state == "Running"`; ROOT on pool; spool_ref Ready (`local_path` = LUN uuid) | Exactly one `/vol//cs_tmpl_` LUN; 0 data-volume LUN-maps | positive | +| 03a | `test_03a_deploy_second_vm_reuses_template_cache` | Deploy VM-2 — reuse cache | test_03 | VM-2 Running; ROOT on pool; still one spool_ref | Still one `cs_tmpl_*`; non-cache LUN count +1 | positive | +| 03b | `test_03b_expunge_second_vm_template_cache_survives` | Expunge VM-2 — cache LUN remains | test_03a | spool_ref still Ready | VM-2 ROOT LUN gone (non-cache count back to baseline); `cs_tmpl_*` still present | positive | +| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created (TDS SN 27) | test_03b (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | +| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed (TDS VM Stop iSCSI) | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | +| 06 | `test_06_grow_stopped_vm_volume` | Grow the attached volume while the VM is stopped | test_05 | Volume is `Ready` at the requested byte size | LUN `space.size` equals the requested size; LUN remains unmapped | positive | +| 07 | `test_07_shrink_rejected` | Attempt to shrink the grown LUN with `shrinkok=true` | test_06 | Resize is rejected; grown size remains unchanged | LUN `space.size` remains unchanged | negative | +| 08 | `test_08_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created (TDS VM Start iSCSI) | test_07 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | +| 09 | `test_09_open_ssh_and_write_root_data_files` | Port-forward SSH to the guest and write ROOT and DATA marker files | test_08 (`vm`, `volume`, guest network) | ROOT and DATA both on the ONTAP pool; DATA still attached; SSH login succeeds; both markers read back | Data LUN still mapped | positive | +| 10 | `test_10_root_volume_snapshot_create` | VS-01: snapshot the ROOT volume of the running VM | test_09 | Snapshot `BackedUp`; VM stays `Running` | One new FlexVol snapshot | positive | +| 11 | `test_11_data_volume_snapshot_create` | VS-03: snapshot the attached DATA volume (kept for test_14) | test_10 | Snapshot `BackedUp`; VM stays `Running` | One new FlexVol snapshot, kept until test_14 | positive | +| 12 | `test_12_volume_revert_running_vm_rejected` | VS-20: revert a volume snapshot while the VM is Running | test_11 | Revert is rejected | FlexVol unchanged | negative | +| 13 | `test_13_root_volume_revert` | VS-12: write A, snapshot ROOT, write B, stop, revert, start | test_12 | After revert and start, ROOT marker is A | New FlexVol snapshot for the snapshot; revert snapshot removed in tearDown | positive | +| 14 | `test_14_data_volume_revert` | Snapshot DATA again, revert to the test_11 snapshot then to this one, then delete both while the VM is running | test_11, test_13 | Older revert restores the test_11 DATA marker and the later snapshot stays listed; later revert restores its marker; both snapshots are then gone and the VM stays `Running` | Both FlexVol snapshots remain through the older revert, then both are removed | positive | +| 15 | `test_15_vm_snapshot_create_running` | VM snapshot of the running VM, ROOT + DATA LUNs (kept for test_17) | test_14 | VM snapshot `Ready`; VM stays `Running` | One new FlexVol snapshot | positive | +| 16 | `test_16_vm_snapshot_create_stopped` | Stop, VM snapshot, start (kept for test_17) | test_15 | VM snapshot `Ready` | One new FlexVol snapshot | positive | +| 17 | `test_17_vm_snapshot_s1_running_s2_stopped_revert` | VMS-09: revert to the test_15 snapshot, then to the test_16 snapshot, then delete both while the VM is running | test_15, test_16 | Each revert finishes; guest file matches that snapshot; both VM snapshots are then gone and the VM stays `Running` | FlexVol snapshots survive the reverts, then both are removed | positive | +| 18 | `test_18_vm_snapshot_duplicate_requests` | VMS-53: two VM snapshot requests for the same VM at once | test_17 | Both snapshots `Ready` (a rejected request is retried) | Two new FlexVol snapshots | positive | +| 19 | `test_19_deploy_second_vm` | Deploy VM-B whose ROOT LUN is on the same ONTAP pool | test_18 | VM-B `Running`; ROOT on pool | One more non-cache LUN; `cs_tmpl_*` still one | positive | +| 20 | `test_20_volume_revert_leaves_other_vm_untouched` | VS-28: revert VM-A's ROOT | test_19 | VM-A's ROOT marker is restored; VM-B's file is unchanged | FlexVol stays `online`; both ROOT LUNs remain | positive | +| 21 | `test_21_concurrent_vm_snapshots_two_vms` | VMS-55: VM snapshots of both VMs at the same time | test_20 | Both VM snapshots `Ready` | Two new FlexVol snapshots | positive | +| 22 | `test_22_concurrent_volume_snapshot_and_revert` | Volume snapshot of VM-A's ROOT while VM-B's ROOT reverts | test_21 | Snapshot `BackedUp` and revert both finish (retried on error 530) | New FlexVol snapshot for the volume snapshot | positive | +| 23 | `test_23_concurrent_vm_snapshot_and_revert` | VM snapshot of VM-A while VM-B reverts to its VM snapshot | test_22 | Both operations finish; VM-A snapshot `Ready` | New FlexVol snapshot for VM-A | positive | +| 24 | `test_24_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM (TDS Detach iSCSI) | test_23 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps for the data LUN; LUN still in FlexVol | positive ⚠️ | +| 25 | `test_25_grow_detached_volume` | Grow the detached, materialized data volume | test_24 | API and DB size equal target | LUN `space.size` equals target | positive | +| 26 | `test_26_destroy_vm_with_vm_snapshot` | Expunge VM-B while it still has a VM snapshot | test_19 (VM-B) | VM-B no longer listed | VM-B ROOT LUN gone; `cs_tmpl_*` still present | positive | +| 27 | `test_27_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_25, test_26 | VM gone; spool_ref still Ready after VM expunge; volume gone; pool gone | `cs_tmpl_*` present after VM expunge; FlexVol deleted; all LUNs and igroups deleted | cleanup | + +> ⚠️ **test_24 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. + +--- + +## Suite 11 — NFS3 Template Cache Negative / Boundary + +**File:** `nfs3/template/test_template_cache_negative.py` +**Class:** `TestOntapNfs3TemplateCacheNegative` +**Tag:** `nfs3_template_cache_negative` +**Total:** 3 tests | **Scope:** Boundary conditions for NFS3 primary template cache (isolated from happy path) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_tag_mismatch_does_not_seed_cache` | SO tags ≠ pool tags | setUpClass | Deploy may succeed elsewhere; ROOT not on ONTAP pool; no `template_spool_ref` for pool | No cache file for template on FlexVol | negative | +| 02 | `test_02_undersized_pool_deploy_fails` | Matching tags but `capacitybytes` ≪ template size | setUpClass | Deploy fails / never Running; spool_ref not Ready/DOWNLOADED | No cache file | negative | +| 03 | `test_03_deleted_cache_blocks_reuse` | Seed cache, delete file out-of-band, redeploy | setUpClass | spool_ref still Ready after ONTAP delete; second deploy fails | Cache file absent after delete | negative | + +--- + +## Suite 12 — iSCSI Template Cache Negative / Boundary + +**File:** `iscsi/template/test_template_cache_negative.py` +**Class:** `TestOntapIscsiTemplateCacheNegative` +**Tag:** `iscsi_template_cache_negative` +**Total:** 3 tests | **Scope:** Boundary conditions for iSCSI primary template cache (isolated from happy path) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +| --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | +| 01 | `test_01_tag_mismatch_does_not_seed_cache` | SO tags ≠ pool tags | setUpClass | ROOT not on ONTAP pool; no `template_spool_ref` | No `cs_tmpl_*` LUN | negative | +| 02 | `test_02_undersized_pool_deploy_fails` | Matching tags but undersized capacity | setUpClass | Deploy fails; spool_ref not Ready/DOWNLOADED | No `cs_tmpl_*` LUN | negative | +| 03 | `test_03_deleted_cache_blocks_reuse` | Seed cache, delete LUN out-of-band, redeploy | setUpClass | spool_ref still Ready; second deploy fails | `cs_tmpl_*` absent after delete | negative | + +--- + +## Cross-suite summary + +| Suite | Protocol | Scope | Tests | Status | +| ------- | --------- | ------- | ------- | -------- | +| NFS3 Pool Lifecycle | NFS3 | Cluster | 21 | ✅ | +| NFS3 Pool with Volumes | NFS3 | Cluster | 12 | ✅ | +| NFS3 Zone-Scoped Pool | NFS3 | Zone | 5 | ✅ | +| NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | +| NFS3 VM + Volume Attach | NFS3 | Cluster | 29 | ⏳ resize and snapshot cases pending lab validation | +| NFS3 Template Cache Negative | NFS3 | Cluster | 3 | 🆕 | +| iSCSI Pool Lifecycle | iSCSI | Cluster | 21 | ✅ | +| iSCSI Pool with Volumes | iSCSI | Cluster | 13 | ✅ | +| iSCSI Zone-Scoped Pool | iSCSI | Zone | 5 | ✅ | +| iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | +| iSCSI VM + Volume Attach | iSCSI | Cluster | 29 | ⏳ resize and snapshot cases pending lab validation; detach limitation | +| iSCSI Template Cache Negative | iSCSI | Cluster | 3 | 🆕 | +| **Total** | | | **151** | **Resize and snapshot cases pending lab validation** | diff --git a/test/integration/plugins/ontap/__init__.py b/test/integration/plugins/ontap/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/aggregate_results.py b/test/integration/plugins/ontap/aggregate_results.py new file mode 100644 index 000000000000..b88c36075f27 --- /dev/null +++ b/test/integration/plugins/ontap/aggregate_results.py @@ -0,0 +1,251 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Merge Marvin suite results into summary.tsv, summary.json, and summary.txt.""" + +from __future__ import print_function + +import argparse +import json +import re +import sys +from datetime import datetime, timezone + + +def parse_results_txt(path): + """Return list of (tag, label, test_id, status, detail) from Marvin results.txt.""" + rows = [] + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + m = re.search(r"TestName: (\S+) \| Status : (\S+)", line) + if m: + rows.append(("", "", m.group(1), m.group(2), "")) + continue + m = re.match(r"(.+?) \.\.\. SKIP: (.+)$", line) + if m: + name = m.group(1).strip() + detail = m.group(2).strip() + rows.append(("", "", name, "SKIP", detail)) + return rows + + +def parse_tsv_line(line): + parts = line.rstrip("\n").split("\t", 4) + while len(parts) < 5: + parts.append("") + return tuple(parts) + + +def normalize_status(status): + st = (status or "").upper() + if st == "SUCCESS": + return "PASS", "pass" + if st == "SKIP": + return "SKIP", "skip" + return "FAIL", "fail" + + +def format_summary_text(rows): + lines = [ + "================================================================", + " TEST SUMMARY", + "================================================================", + ] + pass_n = fail_n = skip_n = 0 + current_label = None + for tag, label, test_id, status, detail in rows: + group = "[%s] %s" % (tag, label) if tag else label + if group != current_label: + if current_label is not None: + lines.append("") + lines.append(" %s" % group) + current_label = group + mark, bucket = normalize_status(status) + if bucket == "pass": + pass_n += 1 + elif bucket == "skip": + skip_n += 1 + else: + fail_n += 1 + suffix = "" + st = (status or "").upper() + if st == "SKIP" and detail: + suffix = " — %s" % detail + elif st not in ("SUCCESS", "SKIP"): + suffix = " — %s" % status + lines.append(" %-4s %s%s" % (mark, test_id, suffix)) + + total = pass_n + fail_n + skip_n + lines.extend([ + "", + "================================================================", + " TOTAL: %d passed, %d failed, %d skipped (%d tests)" % ( + pass_n, fail_n, skip_n, total), + "================================================================", + ]) + return "\n".join(lines) + "\n", pass_n, fail_n, skip_n + + +def rows_to_json(rows, meta=None): + suites = {} + tests = [] + pass_n = fail_n = skip_n = 0 + for tag, label, test_id, status, detail in rows: + mark, bucket = normalize_status(status) + if bucket == "pass": + pass_n += 1 + elif bucket == "skip": + skip_n += 1 + else: + fail_n += 1 + suite_key = tag or label + if suite_key not in suites: + suites[suite_key] = { + "tag": tag, + "label": label, + "passed": 0, + "failed": 0, + "skipped": 0, + "tests": [], + } + suites[suite_key]["tests"].append({ + "name": test_id, + "status": mark, + "detail": detail or None, + }) + if bucket == "pass": + suites[suite_key]["passed"] += 1 + elif bucket == "skip": + suites[suite_key]["skipped"] += 1 + else: + suites[suite_key]["failed"] += 1 + tests.append({ + "tag": tag, + "label": label, + "name": test_id, + "status": mark, + "detail": detail or None, + }) + + payload = { + "generatedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "totals": { + "passed": pass_n, + "failed": fail_n, + "skipped": skip_n, + "total": pass_n + fail_n + skip_n, + }, + "suites": list(suites.values()), + "tests": tests, + } + if meta: + payload["run"] = meta + return payload + + +def load_rows_from_tsv(path): + rows = [] + with open(path, encoding="utf-8") as fh: + for line in fh: + if line.strip(): + rows.append(parse_tsv_line(line)) + return rows + + +def load_rows_from_suite_specs(specs): + rows = [] + for spec in specs: + parts = spec.split(":", 2) + if len(parts) != 3: + print("Invalid --suite spec (want tag:label:path): %s" % spec, + file=sys.stderr) + sys.exit(1) + tag, label, path = parts + for _tag, _label, test_id, status, detail in parse_results_txt(path): + rows.append((tag, label, test_id, status, detail)) + return rows + + +def write_outputs(out_dir, rows, meta=None): + tsv_path = out_dir + "/summary.tsv" + json_path = out_dir + "/summary.json" + txt_path = out_dir + "/summary.txt" + + with open(tsv_path, "w", encoding="utf-8") as fh: + for row in rows: + fh.write("\t".join(row) + "\n") + + summary_text, pass_n, fail_n, skip_n = format_summary_text(rows) + with open(txt_path, "w", encoding="utf-8") as fh: + fh.write(summary_text) + + payload = rows_to_json(rows, meta=meta) + with open(json_path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + fh.write("\n") + + return pass_n, fail_n, skip_n, summary_text + + +def main(): + parser = argparse.ArgumentParser( + description="Aggregate Marvin test results into summary files.") + parser.add_argument( + "--out-dir", required=True, + help="Directory for summary.tsv, summary.json, summary.txt") + parser.add_argument( + "--summary-tsv", + help="Read existing tab-separated summary (from run_tests.sh)") + parser.add_argument( + "--suite", action="append", default=[], + help="Suite spec tag:label:path/to/results.txt (repeatable)") + parser.add_argument( + "--meta-json", + help="JSON string or path to run metadata merged into summary.json") + parser.add_argument( + "--print", dest="print_summary", action="store_true", + help="Print human-readable summary to stdout") + args = parser.parse_args() + + if args.summary_tsv: + rows = load_rows_from_tsv(args.summary_tsv) + elif args.suite: + rows = load_rows_from_suite_specs(args.suite) + else: + print("Provide --summary-tsv or at least one --suite", file=sys.stderr) + sys.exit(1) + + meta = None + if args.meta_json: + if args.meta_json.startswith("{"): + meta = json.loads(args.meta_json) + else: + with open(args.meta_json, encoding="utf-8") as fh: + meta = json.load(fh) + + pass_n, fail_n, skip_n, summary_text = write_outputs( + args.out_dir.rstrip("/"), rows, meta=meta) + + if args.print_summary: + print(summary_text, end="") + + return 0 if fail_n == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/integration/plugins/ontap/check_ontap_prereqs.py b/test/integration/plugins/ontap/check_ontap_prereqs.py new file mode 100644 index 000000000000..0c34101bc495 --- /dev/null +++ b/test/integration/plugins/ontap/check_ontap_prereqs.py @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fail fast when ontap.cfg asks for a protocol the SVM cannot serve. + +Usage: + python3 check_ontap_prereqs.py [iscsi|nfs3|both] +""" + +from __future__ import print_function + +import json +import ssl +import sys +import urllib.error +import urllib.parse +import urllib.request +from base64 import b64encode + + +def _get(base, auth, path, params, ctx): + url = base + path + "?" + urllib.parse.urlencode(params) + request = urllib.request.Request( + url, + headers={"Authorization": "Basic " + auth, "Accept": "application/json"}, + ) + with urllib.request.urlopen(request, context=ctx, timeout=15) as response: + return json.load(response) + + +def _svm_protocol_enabled(svm, protocol): + block = svm.get(protocol) or {} + return bool(block.get("enabled")) + + +def _check_protocol(base, auth, ctx, svm_name, svm, protocol): + rest_name = "iscsi" if protocol == "iscsi" else "nfs" + service = "data-iscsi" if protocol == "iscsi" else "data-nfs" + label = "iSCSI" if protocol == "iscsi" else "NFS" + + if not _svm_protocol_enabled(svm, rest_name): + raise RuntimeError( + "%s protocol is not enabled on SVM %s. " + "Enable the %s service and assign at least one %s data LIF " + "before running these tests." + % (label, svm_name, label, service) + ) + + lifs = _get( + base, auth, "/api/network/ip/interfaces", + {"svm.name": svm_name, "services": service, "fields": "name,ip,state,enabled"}, + ctx, + ) + records = lifs.get("records") or [] + up_lifs = [ + r for r in records + if r.get("enabled") and (r.get("state") or "").lower() == "up" + and (r.get("ip") or {}).get("address") + ] + if not up_lifs: + raise RuntimeError( + "SVM %s has no UP %s data LIF. Assign service %s to at least one " + "data LIF before running %s tests." + % (svm_name, label, service, label) + ) + + if protocol == "iscsi": + services = _get( + base, auth, "/api/protocols/san/iscsi/services", + {"svm.name": svm_name, "fields": "enabled,target"}, + ctx, + ) + svc_records = services.get("records") or [] + if not svc_records: + raise RuntimeError("No iSCSI service found for SVM %s." % svm_name) + target = (svc_records[0].get("target") or {}).get("name") + if not target: + raise RuntimeError("iSCSI target IQN not found for SVM %s." % svm_name) + + print("ONTAP %s prerequisite OK on SVM %s (%d data LIF(s))" + % (label, svm_name, len(up_lifs))) + + +def main(argv): + if len(argv) < 2 or len(argv) > 3: + print("Usage: %s [iscsi|nfs3|both]" % argv[0], file=sys.stderr) + return 2 + + cfg_path = argv[1] + wanted = argv[2] if len(argv) == 3 else "both" + if wanted not in ("iscsi", "nfs3", "both"): + print("Unknown protocol filter: %s" % wanted, file=sys.stderr) + return 2 + + with open(cfg_path, encoding="utf-8") as stream: + config = json.load(stream) + + ontap = config.get("ontap") or {} + pool = config.get("storagePool") or {} + protocols = pool.get("protocols") or {} + storage_ip = ontap.get("storageIP") + svm_name = ontap.get("svmName") + username = ontap.get("username") + password = ontap.get("password") + if not all([storage_ip, svm_name, username, password is not None]): + print("ontap.cfg is missing storageIP, svmName, username, or password", file=sys.stderr) + return 1 + + checks = [] + if wanted in ("iscsi", "both") and protocols.get("iscsi", {}).get("enabled", True): + checks.append("iscsi") + if wanted in ("nfs3", "both") and protocols.get("nfs3", {}).get("enabled", True): + checks.append("nfs3") + if not checks: + print("No enabled protocols to check for filter '%s'" % wanted) + return 0 + + ctx = ssl._create_unverified_context() + auth = b64encode(("%s:%s" % (username, password)).encode()).decode() + base = "https://%s" % storage_ip + try: + svms = _get( + base, auth, "/api/svm/svms", + {"name": svm_name, "fields": "name,uuid,iscsi,nfs"}, + ctx, + ) + except urllib.error.HTTPError as exc: + print("ONTAP REST query failed: HTTP %s" % exc.code, file=sys.stderr) + return 1 + except Exception as exc: + print("ONTAP REST query failed: %s" % exc, file=sys.stderr) + return 1 + + records = svms.get("records") or [] + if not records: + print("SVM '%s' was not found on %s" % (svm_name, storage_ip), file=sys.stderr) + return 1 + svm = records[0] + + try: + for protocol in checks: + _check_protocol(base, auth, ctx, svm_name, svm, protocol) + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/test/integration/plugins/ontap/guest_ssh.py b/test/integration/plugins/ontap/guest_ssh.py new file mode 100644 index 000000000000..749e918214e8 --- /dev/null +++ b/test/integration/plugins/ontap/guest_ssh.py @@ -0,0 +1,216 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +SSH access to the guest VMs deployed by the ONTAP VM workflow tests +(``*/instance/test_vm_volume_attach.py``). + +The template is CentOS 5.5; its login comes from ontap.cfg +(``cloudstack.templateUsername`` / ``cloudstack.templatePassword``, default +root/password). Guests sit on an isolated network, so each VM gets a +port-forwarding rule (public port -> 22) on the network's source-NAT IP. +Markers are plain files written and read with paramiko. +""" + +import logging +import random +import time +import uuid + +import paramiko + +from marvin.cloudstackAPI import ( + createPortForwardingRule as createPortForwardingRuleAPI, + deletePortForwardingRule as deletePortForwardingRuleAPI, + listPublicIpAddresses as listPublicIpAddressesAPI, +) + +logger = logging.getLogger("OntapGuestSsh") + +ROOT_MARKER = "/root/ontap-marker" +DATA_MOUNT = "/mnt/data" +DATA_MARKER = DATA_MOUNT + "/ontap-marker" + +_DATA_MOUNT_CMD = ( + "mkdir -p {m}; " + "mountpoint -q {m} 2>/dev/null || grep -q ' {m} ' /proc/mounts || " + "mount /dev/vdb {m} 2>/dev/null || mount /dev/sdb {m} 2>/dev/null" +).format(m=DATA_MOUNT) + +_DATA_FORMAT_CMD = ( + "mkdir -p {m}; " + "grep -q ' {m} ' /proc/mounts || " + "mount /dev/vdb {m} 2>/dev/null || mount /dev/sdb {m} 2>/dev/null || " + "{{ if [ -b /dev/vdb ]; then d=/dev/vdb; else d=/dev/sdb; fi; " + "mkfs.ext3 -F -q $d && mount $d {m}; }}" +).format(m=DATA_MOUNT) + + +def check_paramiko(): + """Fail fast when paramiko cannot talk to the CentOS 5.5 guest sshd.""" + if "diffie-hellman-group14-sha1" not in paramiko.Transport._kex_info: + raise GuestSshError( + "paramiko %s has no SHA-1 key exchange, which the CentOS 5.5 guest " + "requires. Run test/integration/plugins/ontap/setup_env.sh and use " + "its .venv (it pins paramiko<4)." % paramiko.__version__ + ) + + +def guest_credentials(config): + """Return (username, password) for the template from ontap.cfg.""" + cs_cfg = config.get("cloudstack", {}) + return (cs_cfg.get("templateUsername", "root"), + cs_cfg.get("templatePassword", "password")) + + +def new_token(label="snapshot-test"): + """Return readable, unique text to write into a snapshot test file.""" + readable_label = "".join( + char if char.isalnum() or char in ("-", "_") else "-" + for char in label + ).strip("-") + return "%s-%s" % (readable_label or "snapshot-test", + uuid.uuid4().hex[:12]) + + +class GuestSshError(Exception): + """SSH to the guest or a guest command failed.""" + + +class GuestSsh(object): + """Run shell commands on a guest as root over SSH.""" + + def __init__(self, host, port=22, username="root", password="password", + connect_timeout=600, retry_interval=10): + self.host = host + self.port = int(port) + self.username = username + self.password = password + self.connect_timeout = connect_timeout + self.retry_interval = retry_interval + self._ssh = None + + def __enter__(self): + self.connect() + return self + + def __exit__(self, exc_type, exc, tb): + self.close() + return False + + def connect(self): + """Connect, retrying until the guest's sshd is up after boot.""" + check_paramiko() + deadline = time.time() + self.connect_timeout + last_error = None + while time.time() < deadline: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + ssh.connect(self.host, port=self.port, username=self.username, + password=self.password, timeout=15, + allow_agent=False, look_for_keys=False) + self._ssh = ssh + logger.info("SSH connected to %s:%d", self.host, self.port) + return + except Exception as exc: + last_error = exc + ssh.close() + time.sleep(self.retry_interval) + raise GuestSshError("SSH to %s:%d failed within %ds: %s" + % (self.host, self.port, self.connect_timeout, last_error)) + + def close(self): + if self._ssh is not None: + self._ssh.close() + self._ssh = None + + def run(self, command, check=True, timeout=300): + """Run a command; return stdout. Raise on non-zero exit if check.""" + if self._ssh is None: + self.connect() + _, stdout, stderr = self._ssh.exec_command(command, timeout=timeout) + status = stdout.channel.recv_exit_status() + out = stdout.read().decode("utf-8", "replace") + err = stderr.read().decode("utf-8", "replace") + if check and status != 0: + raise GuestSshError("'%s' on %s exited %d: %s" + % (command, self.host, status, err.strip())) + return out + + def write_root_marker(self, token): + self.run("echo '%s' > %s && sync" % (token, ROOT_MARKER)) + + def read_root_marker(self): + return self.run("cat %s 2>/dev/null; true" % ROOT_MARKER).strip() + + def write_data_marker(self, token): + self.run(_DATA_FORMAT_CMD) + self.run("echo '%s' > %s && sync" % (token, DATA_MARKER)) + + def read_data_marker(self): + self.run(_DATA_MOUNT_CMD, check=False) + return self.run("cat %s 2>/dev/null; true" % DATA_MARKER).strip() + + +# --------------------------------------------------------------------------- +# Port forwarding +# --------------------------------------------------------------------------- + +def source_nat_ip(apiclient, network_id): + """Return the source-NAT public IP record for an isolated network.""" + cmd = listPublicIpAddressesAPI.listPublicIpAddressesCmd() + cmd.associatednetworkid = network_id + cmd.issourcenat = True + cmd.listall = True + ips = apiclient.listPublicIpAddresses(cmd) or [] + if not ips: + raise GuestSshError("No source-NAT IP found for network %s" % network_id) + return ips[0] + + +def open_ssh_port(apiclient, vm_id, network_id, public_port=None): + """Forward a public port on the source-NAT IP to the VM's port 22. + + Returns (public_ip, public_port, rule_id). + """ + ip = source_nat_ip(apiclient, network_id) + last_error = None + for _ in range(10): + port = public_port or random.randint(20000, 29999) + cmd = createPortForwardingRuleAPI.createPortForwardingRuleCmd() + cmd.ipaddressid = ip.id + cmd.privateport = 22 + cmd.publicport = port + cmd.protocol = "TCP" + cmd.virtualmachineid = vm_id + cmd.openfirewall = True + try: + rule = apiclient.createPortForwardingRule(cmd) + return ip.ipaddress, port, rule.id + except Exception as exc: + last_error = exc + if public_port: + break + raise GuestSshError("Could not create SSH port forward for VM %s: %s" + % (vm_id, last_error)) + + +def close_ssh_port(apiclient, rule_id): + cmd = deletePortForwardingRuleAPI.deletePortForwardingRuleCmd() + cmd.id = rule_id + apiclient.deletePortForwardingRule(cmd) diff --git a/test/integration/plugins/ontap/helpers/__init__.py b/test/integration/plugins/ontap/helpers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/helpers/kvm_host.py b/test/integration/plugins/ontap/helpers/kvm_host.py new file mode 100644 index 000000000000..de3b90dcc11e --- /dev/null +++ b/test/integration/plugins/ontap/helpers/kvm_host.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""KVM host SSH helpers for ONTAP Marvin tests. + +Host-side checks (libvirt, qemu-img, NFS mount cleanup) live here so +``ontap_test_base`` stays focused on CloudStack and ONTAP REST helpers. +""" + +import json +import logging +import shlex + +from marvin.sshClient import SshClient + +logger = logging.getLogger("kvm_host") + + +def cleanup_kvm_storage_pool_mounts(kvm_hosts_ssh_creds, pool_uuid): + """Unmount and undefine the libvirt pool for ``pool_uuid`` on each host. + + Call this before the ONTAP FlexVol is deleted so the unmount finishes + while the NFS export is still reachable. Stale NFS mounts can make + KVMHAMonitor reboot the host. + """ + for creds in kvm_hosts_ssh_creds: + host_ip = creds["host"] + try: + ssh = SshClient( + host_ip, 22, + creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + for cmd in [ + "umount -f -l /mnt/{u} 2>/dev/null; true".format(u=pool_uuid), + "virsh pool-destroy {u} 2>/dev/null; true".format(u=pool_uuid), + "virsh pool-undefine {u} 2>/dev/null; true".format(u=pool_uuid), + ]: + try: + ssh.execute(cmd) + except Exception as cmd_ex: + logger.warning( + "cleanup_kvm_storage_pool_mounts: cmd '%s' " + "failed on %s: %s" % (cmd, host_ip, cmd_ex) + ) + except Exception as ex: + logger.warning( + "cleanup_kvm_storage_pool_mounts: SSH to %s failed: %s" + % (host_ip, ex) + ) + + +def get_qcow2_virtual_size(kvm_hosts_ssh_creds, pool_uuid, volume_path): + """Read a qcow2 virtual size from a KVM host mounting the NFS pool.""" + pool_arg = shlex.quote(str(pool_uuid)) + volume_arg = shlex.quote(str(volume_path)) + command = ( + "path=$(virsh vol-path --pool {pool} {volume} 2>/dev/null || true); " + "if [ -z \"$path\" ]; then path=/mnt/{pool}/{volume}; fi; " + "qemu_img=$(command -v cloud-qemu-img || command -v qemu-img); " + "test -n \"$qemu_img\" || " + "{{ echo 'qemu-img executable not found' >&2; exit 1; }}; " + "\"$qemu_img\" info -U --output=json \"$path\"" + ).format(pool=pool_arg, volume=volume_arg) + + errors = [] + for creds in kvm_hosts_ssh_creds: + try: + ssh = SshClient( + creds["host"], 22, creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + output = "\n".join(ssh.execute(command)) + json_start = output.find("{") + if json_start < 0: + raise ValueError(output) + info, _ = json.JSONDecoder().raw_decode(output[json_start:]) + return int(info["virtual-size"]) + except Exception as exc: + errors.append("%s: %s" % (creds.get("host", "?"), exc)) + + raise RuntimeError( + "Could not read qcow2 virtual size from any KVM host: %s" + % "; ".join(errors) + ) + + +def get_vm_data_disk_capacity(kvm_hosts_ssh_creds, instance_name): + """Return the libvirt capacity of the VM's first data disk in bytes. + + The ONTAP VM workflow attaches exactly one data disk. The first libvirt + disk is the root disk and the second is the resized data disk, whether + its source is NFS or iSCSI multipath. + """ + instance_arg = shlex.quote(str(instance_name)) + command = ( + "target=$(virsh domblklist {vm} --details | " + "awk '$2 == \"disk\" {{ count++; if (count == 2) " + "{{ print $3; exit }} }}'); " + "test -n \"$target\" || " + "{{ echo 'data disk target not found' >&2; exit 1; }}; " + "virsh domblkinfo {vm} \"$target\" | " + "awk '$1 == \"Capacity:\" {{ print $2; exit }}'" + ).format(vm=instance_arg) + + errors = [] + for creds in kvm_hosts_ssh_creds: + try: + ssh = SshClient( + creds["host"], 22, creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + output = "\n".join(ssh.execute(command)).strip() + if not output: + raise ValueError("virsh returned no data-disk capacity") + return int(output.splitlines()[-1].strip()) + except Exception as exc: + errors.append("%s: %s" % (creds.get("host", "?"), exc)) + + raise RuntimeError( + "Could not read VM data-disk capacity from any KVM host: %s" + % "; ".join(errors) + ) diff --git a/test/integration/plugins/ontap/helpers/template_cache_base.py b/test/integration/plugins/ontap/helpers/template_cache_base.py new file mode 100644 index 000000000000..93a8d61ea1b0 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_base.py @@ -0,0 +1,303 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared building blocks for ONTAP primary template-cache Marvin checks. + +The template-cache assertions themselves live on OntapTestBase, since every +ONTAP ROOT volume on KVM is cloned from the primary template cache. + + OntapTemplateCacheBase + setUpClass for the standalone negative suite (zone, template, guest + network, ONTAP client) without the sequential pool / VM state. +""" + +from __future__ import print_function + +import base64 +import logging +import random +import time +import unittest + +from marvin.cloudstackAPI import ( + createNetwork as createNetworkAPI, + deleteNetwork as deleteNetworkAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listTemplates as listTemplatesAPI, + stopVirtualMachine as stopVirtualMachineAPI, +) +from ontap_test_base import ( + OntapRestClient, + OntapTestBase, + _list_vms_cmd, + get_datacenter_config, +) +from helpers import template_cache_util as tcu + +logger = logging.getLogger("TemplateCacheBase") + +# ~50 GiB — room for template cache + ROOT clones (+ a data volume) +DEFAULT_TEMPLATE_CACHE_CAPACITY_BYTES = 50 * 1024 * 1024 * 1024 + + +def template_cache_tags(proto_cfg, protocol_cfg_key): + """ + Dedicated storage tag (``-tmpl-cache``) so ROOT is forced + onto the template-cache suite's own pool, not pools of other suites. + """ + base = proto_cfg.get("storagePoolTags") or "ontap-%s" % protocol_cfg_key + return base + "-tmpl-cache" + + +def template_cache_capacity_bytes(pool_cfg): + """Pool size large enough to hold the template cache plus ROOT clones.""" + return (pool_cfg.get("templateCacheCapacitybytes") + or pool_cfg.get("capacitybytes") + or DEFAULT_TEMPLATE_CACHE_CAPACITY_BYTES) + + +def tagged_compute_offering_data(tags, name_prefix="OntapTmplCacheSO"): + """Compute offering whose storage tags match the ONTAP pool.""" + name = "%s_%d" % (name_prefix, random.randint(0, 99999)) + return { + "name": name, + "displaytext": "ONTAP template-cache SO (tagged)", + "cpunumber": 1, + "cpuspeed": 500, + "memory": 512, + "storagetype": "shared", + "tags": tags, + } + + +def find_ready_kvm_template(api_client, zone_id): + """Return the first ready, non-SYSTEM KVM template in the zone, or None.""" + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = zone_id + templates = api_client.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + return kvm_ready[0] if kvm_ready else None + + +class TemplateCacheTestData(object): + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + computeOffering = "computeOffering" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-tmpl-cache", capacitybytes=None, + pool_name_prefix="OntapTmplCache"): + if capacitybytes is None: + capacitybytes = DEFAULT_TEMPLATE_CACHE_CAPACITY_BYTES + encoded_password = base64.b64encode(password.encode()).decode() + suffix = random.randint(0, 99999) + self.testdata = { + self.ontap: { + self.DETAIL_STORAGE_IP: storage_ip, + self.DETAIL_SVM_NAME: svm_name, + self.DETAIL_USERNAME: username, + self.DETAIL_PASSWORD: password, + }, + self.account: { + "email": "ontap-tmpl-cache@test.com", + "firstname": "ONTAP", + "lastname": "TmplCache", + "username": "ontap_tmpl_%d" % suffix, + "password": "password", + }, + self.primaryStorage: { + "name": "%s_%d" % (pool_name_prefix, suffix), + self.scope: scope, + self.provider: provider, + self.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + self.DETAIL_USERNAME: username, + self.DETAIL_PASSWORD: encoded_password, + self.DETAIL_SVM_NAME: svm_name, + self.DETAIL_PROTOCOL: protocol, + self.DETAIL_STORAGE_IP: storage_ip, + }, + }, + self.computeOffering: tagged_compute_offering_data(tags), + } + + +class OntapTemplateCacheBase(OntapTestBase): + """ + Class-level setup for standalone template-cache suites. + + Subclasses must set: + PROTOCOL - "NFS3" or "ISCSI" + NOSE_TAG - nose attr tag string + PROTOCOL_CFG_KEY - "nfs3" or "iscsi" under storagePool.protocols + POOL_URL_SCHEME - used in createStoragePool url (e.g. nfs / iscsi) + POOL_NAME_PREFIX - prefix for pools created by the suite + """ + + PROTOCOL = "NFS3" + NOSE_TAG = "template_cache" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapTmplCache" + + template_id = None # API UUID + template_db_id = None # numeric DB id + network_id = None + _created_network_id = None + + @classmethod + def setUpClass(cls): + super(OntapTemplateCacheBase, cls).setUpClass() + testclient = super(OntapTemplateCacheBase, cls).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + + proto_cfg = pool_cfg.get("protocols", {}).get(cls.PROTOCOL_CFG_KEY, {}) + if not proto_cfg.get("enabled", True): + raise unittest.SkipTest( + "%s tests disabled in ontap.cfg " + "(set protocols.%s.enabled=true to enable)" + % (cls.PROTOCOL, cls.PROTOCOL_CFG_KEY) + ) + + tags = template_cache_tags(proto_cfg, cls.PROTOCOL_CFG_KEY) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + + cls.testdata = TemplateCacheTestData( + storage_ip, svm_name, username, password, + protocol=cls.PROTOCOL, scope=scope, provider=provider, + tags=tags, capacitybytes=template_cache_capacity_bytes(pool_cfg), + pool_name_prefix=cls.POOL_NAME_PREFIX, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + cls.storage_tags = tags + + cls._setup_cloudstack_resources( + config, cls.testdata[TemplateCacheTestData.account] + ) + + template = find_ready_kvm_template(cls.apiClient, cls.zone.id) + if template is None: + raise unittest.SkipTest( + "No ready user KVM template in zone '%s'" % cls.zone.name + ) + cls.template_id = template.id + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) + + cls.network_id = None + cls._created_network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.supportedservices = "SourceNat" + offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + if not offerings: + raise unittest.SkipTest( + "No Isolated network offering for Advanced zone" + ) + cn = createNetworkAPI.createNetworkCmd() + cn.name = "ontap-tmpl-cache-net-%d" % random.randint(0, 9999) + cn.displaytext = cn.name + cn.networkofferingid = offerings[0].id + cn.zoneid = cls.zone.id + cn.account = cls.account.name + cn.domainid = cls.domain.id + created = cls.apiClient.createNetwork(cn) + cls.network_id = created.id + cls._created_network_id = created.id + + @classmethod + def tearDownClass(cls): + if cls._created_network_id is not None: + try: + dn = deleteNetworkAPI.deleteNetworkCmd() + dn.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn) + except Exception as e: + logger.warning( + "tearDownClass: delete network failed: %s" % e + ) + cls._created_network_id = None + + super(OntapTemplateCacheBase, cls).tearDownClass() + + @classmethod + def _destroy_vm_static(cls, vm_id): + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if not vms: + return + state = (vms[0].state or "").lower() + if state not in ("stopped", "destroyed", "expunging", "error"): + stop = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop.id = vm_id + cls.apiClient.stopVirtualMachine(stop) + deadline = time.time() + 300 + while time.time() < deadline: + cur = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if cur and cur[0].state.lower() == "stopped": + break + time.sleep(10) + dest = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest.id = vm_id + dest.expunge = True + cls.apiClient.destroyVirtualMachine(dest) diff --git a/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py b/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py new file mode 100644 index 000000000000..e54c32541d99 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py @@ -0,0 +1,643 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Negative / boundary workflow for ONTAP primary template-cache Marvin suites. + +The happy path (seed / reuse / cache survives VM delete) runs inside the VM +instance suites (``*/instance/test_vm_volume_attach.py``). These cases need +their own pool and / or offering (mismatched tags, undersized capacity, a +deliberately broken cache), so they stay isolated and cannot cascade into the +instance workflow. + +Cases: + 01 Tag mismatch — SO tags do not match ONTAP pool; no spool_ref on pool + 02 Undersized pool — capacity << template size; deploy must fail + 03 Cache deleted out-of-band — Ready spool_ref but missing ONTAP object; + subsequent deploy must fail +""" + +from __future__ import print_function + +import copy +import logging +import random +import time + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteServiceOffering as deleteServiceOfferingAPI, + deployVirtualMachine as deployVirtualMachineAPI, + enableStorageMaintenance, +) +from marvin.lib.base import ServiceOffering, StoragePool + +from helpers import template_cache_util as tcu +from helpers.template_cache_base import ( + OntapTemplateCacheBase, + TemplateCacheTestData, +) + +logger = logging.getLogger("TemplateCacheNegativeWorkflow") + + +class OntapTemplateCacheNegativeWorkflow(OntapTemplateCacheBase): + """ + Independent negative boundary tests. + + Reuses setUpClass (zone / template / ONTAP client) from the shared base + but creates per-test pools and offerings so state stays isolated. + """ + + NOSE_TAG = "template_cache_negative" + POOL_NAME_PREFIX = "OntapTmplCacheNeg" + + @classmethod + def setUpClass(cls): + super(OntapTemplateCacheNegativeWorkflow, cls).setUpClass() + # Lab public IP pool is tiny (often only 3 addresses: SSVM + CPVM + + # one SourceNat). Starting leftover VRs from prior runs consumes the + # last IP and then this suite's Allocated guest network cannot be + # implemented — seed deploy fails with "Unable to create a deployment". + cls._reclaim_stale_template_cache_networks() + cls._cleanup_stale_negative_pools() + cls._ensure_guest_network_router_running() + + @classmethod + def _cleanup_stale_negative_pools(cls): + """ + Force-delete leftover OntapTmplCacheNeg* pools from prior runs. + + Shared storage_tags on a leftover Up pool will steal ROOT placement + from the pool created by step_02/step_03 and make assertions fail. + """ + from marvin.cloudstackAPI import listStoragePools as listStoragePoolsAPI + + cmd = listStoragePoolsAPI.listStoragePoolsCmd() + cmd.zoneid = cls.zone.id + pools = cls.apiClient.listStoragePools(cmd) or [] + prefixes = ( + cls.POOL_NAME_PREFIX, + "OntapIscsiTmplNeg", + "OntapNfs3TmplNeg", + "OntapTmplCacheNeg", + ) + for pool in pools: + name = getattr(pool, "name", "") or "" + if not any(name.startswith(prefix) for prefix in prefixes): + continue + logger.info("Cleaning leftover negative-suite pool %s" % name) + try: + # Instance method helpers need a throwaway binder. + binder = cls.__new__(cls) + binder.apiClient = cls.apiClient + binder.ontap = cls.ontap + binder._force_delete_pool(pool) + except Exception as ex: + logger.warning("Could not delete leftover pool %s: %s" % (name, ex)) + + @classmethod + def _reclaim_stale_template_cache_networks(cls): + """ + Destroy leftover ontap-tmpl-cache-net-* networks (and their VRs/VMs) + from prior suite accounts so SourceNat public IPs are freed. + Keeps the network created for this class (cls.network_id). + """ + from marvin.cloudstackAPI import ( + deleteNetwork as deleteNetworkAPI, + destroyRouter as destroyRouterAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + listNetworks as listNetworksAPI, + listRouters as listRoutersAPI, + listVirtualMachines as listVirtualMachinesAPI, + stopVirtualMachine as stopVirtualMachineAPI, + ) + + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.listall = True + networks = cls.apiClient.listNetworks(net_cmd) or [] + for net in networks: + name = getattr(net, "name", "") or "" + # Match names from positive/negative suite network create + # (ontap-tmpl-cache-net-*). + if "tmpl-cache-net" not in name: + continue + if cls.network_id and str(net.id) == str(cls.network_id): + continue + + logger.info( + "Reclaiming stale template-cache network %s (%s)" + % (name, net.id) + ) + + vm_cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + vm_cmd.listall = True + vm_cmd.networkid = net.id + for vm in (cls.apiClient.listVirtualMachines(vm_cmd) or []): + try: + state = (vm.state or "").lower() + if state not in ( + "destroyed", "expunging", "error", "stopped" + ): + stop = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop.id = vm.id + cls.apiClient.stopVirtualMachine(stop) + dest = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest.id = vm.id + dest.expunge = True + cls.apiClient.destroyVirtualMachine(dest) + except Exception as ex: + logger.warning( + "Reclaim: destroy VM %s on %s: %s" % (vm.id, name, ex) + ) + + rtr_cmd = listRoutersAPI.listRoutersCmd() + rtr_cmd.zoneid = cls.zone.id + rtr_cmd.networkid = net.id + rtr_cmd.listall = True + for router in (cls.apiClient.listRouters(rtr_cmd) or []): + try: + dr = destroyRouterAPI.destroyRouterCmd() + dr.id = router.id + cls.apiClient.destroyRouter(dr) + logger.info( + "Reclaim: destroyed router %s for %s" + % (router.name, name) + ) + except Exception as ex: + logger.warning( + "Reclaim: destroy router %s: %s" % (router.id, ex) + ) + + try: + dn = deleteNetworkAPI.deleteNetworkCmd() + dn.id = net.id + cls.apiClient.deleteNetwork(dn) + logger.info("Reclaim: deleted network %s" % name) + except Exception as ex: + logger.warning( + "Reclaim: delete network %s: %s" % (net.id, ex) + ) + + @classmethod + def _ensure_guest_network_router_running(cls): + """ + Ensure the VR for *this suite's* guest network is Running. + + Do not start unrelated zone routers — that can exhaust the lab's + public IP range and block implementing cls.network_id. + """ + from marvin.cloudstackAPI import ( + listRouters as listRoutersAPI, + startRouter as startRouterAPI, + ) + if not cls.network_id: + logger.warning( + "No guest network_id — deploy may fail in Advanced zone" + ) + return + + cmd = listRoutersAPI.listRoutersCmd() + cmd.zoneid = cls.zone.id + cmd.networkid = cls.network_id + cmd.listall = True + routers = cls.apiClient.listRouters(cmd) or [] + if not routers: + # Allocated isolated network: VR is created on first deploy. + logger.info( + "No VR yet for network %s — will be created on first deploy" + % cls.network_id + ) + return + + for router in routers: + state = (router.state or "").lower() + if state == "running": + continue + logger.info( + "Starting guest-network router %s (was %s)" + % (router.name, router.state) + ) + start = startRouterAPI.startRouterCmd() + start.id = router.id + cls.apiClient.startRouter(start) + deadline = time.time() + 300 + while time.time() < deadline: + cur = cls.apiClient.listRouters(cmd) or [] + match = [r for r in cur if r.id == router.id] + if match and (match[0].state or "").lower() == "running": + logger.info("Router %s is Running" % router.name) + break + time.sleep(10) + else: + raise RuntimeError( + "Guest-network router %s did not reach Running" + % router.name + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _unique_storage_tags(self): + """Per-test tags so leftover pools with shared tags cannot steal ROOT.""" + return "%s-neg-%d" % (self.storage_tags, random.randint(0, 99999)) + + def _create_pool_with(self, tags=None, capacitybytes=None, name_prefix=None): + """Create an ONTAP pool, optionally overriding tags / capacity.""" + ps = copy.deepcopy(self.testdata[TemplateCacheTestData.primaryStorage]) + if tags is not None: + ps[TemplateCacheTestData.tags] = tags + if capacitybytes is not None: + ps["capacitybytes"] = int(capacitybytes) + + storage_ip = self.testdata[TemplateCacheTestData.ontap][ + TemplateCacheTestData.DETAIL_STORAGE_IP + ] + prefix = name_prefix or self.POOL_NAME_PREFIX + pool_name = "%s_%d" % (prefix, random.randint(0, 99999)) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "%s://%s/ontap" % (self.POOL_URL_SCHEME, storage_ip) + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TemplateCacheTestData.scope] + cmd.provider = ps[TemplateCacheTestData.provider] + cmd.tags = ps[TemplateCacheTestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + pool = StoragePool(response.__dict__) + self.assertEqual( + pool.state, "Up", + "Pool %s should be Up after create, got %s" % (pool.name, pool.state), + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "FlexVol missing for pool %s" % pool.name) + self.assertEqual(ontap_vol.get("state"), "online") + return pool + + def _create_service_offering_with(self, tags): + """Create a compute offering with the given storage tags.""" + so_data = copy.deepcopy( + self.testdata[TemplateCacheTestData.computeOffering] + ) + so_data["tags"] = tags + so_data["name"] = "OntapTmplCacheNegSO_%d" % random.randint(0, 99999) + so_data["displaytext"] = so_data["name"] + so = ServiceOffering.create(self.apiClient, so_data) + self.assertIsNotNone(so.id, "Service offering create failed") + return so + + def _deploy_vm_with_offering(self, name_suffix, service_offering_id): + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-neg-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + return self._poll_vm_state(vm.id, "Running") + + def _deploy_vm_expect_failure(self, name_suffix, service_offering_id, + timeout=300): + """Assert deploy does not reach Running (API error or Error state).""" + vm = None + try: + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-neg-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + except Exception as ex: + logger.info( + "deployVirtualMachine failed as expected for %s: %s" + % (name_suffix, ex) + ) + return + + self.assertIsNotNone(vm, "deploy returned None without raising") + reached_running = False + try: + self._poll_vm_state(vm.id, "Running", timeout=timeout) + reached_running = True + except AssertionError as ex: + logger.info( + "Deploy %s did not reach Running as expected: %s" + % (name_suffix, ex) + ) + finally: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + if reached_running: + self.fail( + "Deploy %s unexpectedly reached Running (boundary should fail)" + % name_suffix + ) + + def _force_delete_pool(self, pool): + if pool is None: + return + pool_name = pool.name + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + self.apiClient.enableStorageMaintenance(mc) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + except Exception as ex: + logger.warning( + "enableStorageMaintenance for %s: %s" % (pool.id, ex) + ) + try: + self._delete_pool(pool.id, forced=True) + except Exception as ex: + logger.warning("force delete pool %s: %s" % (pool.id, ex)) + deadline = time.time() + 120 + while time.time() < deadline: + if self.ontap.get_volume(pool_name) is None: + break + time.sleep(5) + + def _delete_service_offering(self, offering): + if offering is None: + return + try: + cmd = deleteServiceOfferingAPI.deleteServiceOfferingCmd() + cmd.id = offering.id + self.apiClient.deleteServiceOffering(cmd) + except Exception as ex: + logger.warning("delete SO %s: %s" % (offering.id, ex)) + + def _delete_ontap_cache(self, pool, spool_ref): + """Remove ONTAP cache object while leaving CloudStack spool_ref.""" + if self.PROTOCOL.upper() == "ISCSI": + path = tcu.template_cache_lun_path( + pool.name, self.__class__.template_db_id + ) + deleted = self.ontap.delete_lun(self.svm_name, path) + self.assertTrue( + deleted, + "Failed to delete template cache LUN at %s" % path, + ) + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + else: + install_path = spool_ref.get("install_path") + self.assertTrue(install_path, "spool_ref.install_path required") + deleted = self.ontap.delete_file_in_volume( + pool.name, install_path + ) + self.assertTrue( + deleted, + "Failed to delete NFS cache file %s in %s" + % (install_path, pool.name), + ) + tcu.assert_no_nfs_template_cache_file( + self, self.ontap, pool.name, install_path + ) + + # ------------------------------------------------------------------ + # Negative cases + # ------------------------------------------------------------------ + + def step_01_tag_mismatch_does_not_seed_cache(self): + """ + Pool has template-cache tags; SO uses a different tag. + + Valid outcomes: + a) deploy fails (no alternate tagged pool) — common in lab + b) deploy succeeds elsewhere — ROOT must not be on ONTAP pool + + In both cases there must be no spool_ref / cache on the ONTAP pool. + """ + pool = None + so = None + vm = None + mismatch_tag = "ontap-tmpl-cache-mismatch-%d" % random.randint(0, 99999) + try: + pool = self._create_pool_with(tags=self.storage_tags) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=mismatch_tag) + + try: + vm = self._deploy_vm_with_offering("tag-mismatch", so.id) + except Exception as ex: + logger.info( + "Deploy with mismatched tags failed (acceptable if no " + "alternate pool matches): %s" % ex + ) + vm = None + + if vm is not None: + root = self._root_volume_for_vm(vm.id) + self.assertNotEqual( + str(root.storageid), str(pool.id), + "ROOT should not land on ONTAP pool when SO tags mismatch " + "(storageid=%s pool=%s)" % (root.storageid, pool.id), + ) + + tcu.assert_no_spool_ref( + self, self.dbConnection, pool_db_id, + self.__class__.template_db_id, + ) + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + finally: + if vm is not None: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + self._delete_service_offering(so) + self._force_delete_pool(pool) + + def step_02_undersized_pool_deploy_fails(self): + """ + Matching tags but capacitybytes far below template virtual size. + + Deploy must fail; no Ready spool_ref / cache object should remain. + """ + pool = None + so = None + tmpl_size = tcu.get_template_size_bytes( + self.dbConnection, self.__class__.template_db_id + ) + if not tmpl_size or tmpl_size <= 0: + self.skipTest( + "Cannot determine vm_template.size for template_db_id=%s" + % self.__class__.template_db_id + ) + + # Lab templates can be small (~100–200 MiB); keep capacity under size. + undersized = max(tmpl_size // 4, 8 * 1024 * 1024) + if undersized >= tmpl_size: + undersized = max(tmpl_size // 2, 1024 * 1024) + self.assertLess( + undersized, tmpl_size, + "undersized capacity %s must be < template size %s" + % (undersized, tmpl_size), + ) + + try: + exclusive_tags = self._unique_storage_tags() + pool = self._create_pool_with( + tags=exclusive_tags, + capacitybytes=undersized, + name_prefix=self.POOL_NAME_PREFIX + "Tiny", + ) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=exclusive_tags) + + self._deploy_vm_expect_failure("undersized", so.id, timeout=300) + + spool = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id + ) + if spool is not None: + self.assertNotEqual( + str(spool.get("state")), "Ready", + "spool_ref must not be Ready after undersized deploy failure", + ) + self.assertNotEqual( + str(spool.get("download_state", "")).upper(), + "DOWNLOADED", + "spool_ref must not be DOWNLOADED after undersized failure", + ) + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + finally: + self._delete_service_offering(so) + self._force_delete_pool(pool) + + def step_03_deleted_cache_blocks_reuse(self): + """ + Seed cache, destroy VM, delete ONTAP cache out-of-band, redeploy. + + CloudStack still has Ready spool_ref but backend object is gone — + second deploy must fail (clone / missing cache path). + """ + self._ensure_guest_network_router_running() + pool = None + so = None + vm1 = None + try: + exclusive_tags = self._unique_storage_tags() + pool = self._create_pool_with(tags=exclusive_tags) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=exclusive_tags) + + # Seed deploy can flake on lab planner/VR; retry once after + # re-ensuring the guest network router is Running. + last_ex = None + for attempt in range(2): + try: + self._ensure_guest_network_router_running() + vm1 = self._deploy_vm_with_offering("neg-seed", so.id) + last_ex = None + break + except Exception as ex: + last_ex = ex + logger.warning( + "Seed deploy attempt %s failed: %s" + % (attempt + 1, ex) + ) + self._reclaim_stale_template_cache_networks() + time.sleep(15) + if last_ex is not None: + # Seed must succeed — this case is not optional. Failure here + # is usually exhausted public IPs / stale VR from prior runs. + self.fail( + "Could not seed template cache for out-of-band delete " + "case after retries (check public IP capacity / leftover " + "VRs): %s" % last_ex + ) + self._assert_root_on_pool(vm1.id, pool) + spool = tcu.wait_for_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + timeout=600, + ) + tcu.assert_spool_ref_ready( + self, spool, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + self._assert_cache_on_ontap(pool, spool) + + self._destroy_vm_static(vm1.id) + vm1 = None + # Allow clone/ROOT LUN cleanup so the cache LUN is no longer a + # FlexClone parent before out-of-band delete. + time.sleep(30) + + self._delete_ontap_cache(pool, spool) + stale = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id + ) + tcu.assert_spool_ref_ready( + self, stale, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + + self._deploy_vm_expect_failure("stale-cache", so.id, timeout=300) + finally: + if vm1 is not None: + try: + self._destroy_vm_static(vm1.id) + except Exception: + pass + self._delete_service_offering(so) + self._force_delete_pool(pool) diff --git a/test/integration/plugins/ontap/helpers/template_cache_util.py b/test/integration/plugins/ontap/helpers/template_cache_util.py new file mode 100644 index 000000000000..e733924ee940 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_util.py @@ -0,0 +1,256 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Helpers for ONTAP primary template-cache Marvin assertions. + +Covers CloudStack DB (``template_spool_ref``) and ONTAP REST checks for the +cache LUN (iSCSI: ``/vol//cs_tmpl_``) or cache file (NFS). +""" + +from __future__ import print_function + +import logging +import time + +logger = logging.getLogger("template_cache_util") + +# Must stay in sync with OntapStorageConstants.TEMPLATE_LUN_PREFIX +TEMPLATE_LUN_PREFIX = "cs_tmpl_" + + +def get_db_id(db_connection, table, api_uuid): + """Resolve CloudStack API UUID to numeric DB id.""" + rows = db_connection.execute( + "SELECT id FROM `%s` WHERE uuid = '%s'" % (table, api_uuid) + ) + if not rows: + raise AssertionError( + "No row in %s for uuid=%s" % (table, api_uuid) + ) + return rows[0][0] + + +def get_template_spool_ref(db_connection, pool_db_id, template_db_id): + """ + Return template_spool_ref row as a dict, or None if absent. + + Columns: id, pool_id, template_id, download_state, local_path, + install_path, template_size, marked_for_gc, state + """ + rows = db_connection.execute( + "SELECT id, pool_id, template_id, download_state, local_path, " + "install_path, template_size, marked_for_gc, state " + "FROM template_spool_ref " + "WHERE pool_id = %s AND template_id = %s" + % (int(pool_db_id), int(template_db_id)) + ) + if not rows: + return None + row = rows[0] + return { + "id": row[0], + "pool_id": row[1], + "template_id": row[2], + "download_state": row[3], + "local_path": row[4], + "install_path": row[5], + "template_size": row[6], + "marked_for_gc": row[7], + "state": row[8], + } + + +def count_template_spool_refs(db_connection, pool_db_id, template_db_id): + """Return number of template_spool_ref rows for pool+template.""" + rows = db_connection.execute( + "SELECT COUNT(*) FROM template_spool_ref " + "WHERE pool_id = %s AND template_id = %s" + % (int(pool_db_id), int(template_db_id)) + ) + return int(rows[0][0]) if rows else 0 + + +def assert_spool_ref_ready(testcase, spool_ref, expect_local_path=True): + """Assert spool_ref is DOWNLOADED / Ready after a successful cache seed.""" + testcase.assertIsNotNone(spool_ref, "Expected template_spool_ref row") + testcase.assertEqual( + str(spool_ref["download_state"]).upper(), + "DOWNLOADED", + "download_state should be DOWNLOADED, got %s" + % spool_ref["download_state"], + ) + testcase.assertEqual( + str(spool_ref["state"]), + "Ready", + "state should be Ready, got %s" % spool_ref["state"], + ) + if expect_local_path: + testcase.assertTrue( + spool_ref.get("local_path"), + "local_path (LUN uuid / cache id) should be set", + ) + testcase.assertTrue( + spool_ref.get("install_path"), + "install_path should be set after template is cached", + ) + + +def assert_no_spool_ref(testcase, db_connection, pool_db_id, template_db_id): + """Assert there is no template_spool_ref for pool+template.""" + count = count_template_spool_refs( + db_connection, pool_db_id, template_db_id + ) + testcase.assertEqual( + count, 0, + "Expected no template_spool_ref for pool=%s template=%s, found %s" + % (pool_db_id, template_db_id, count), + ) + + +def template_cache_lun_path(flexvol_name, template_db_id): + """iSCSI cache LUN path: /vol//cs_tmpl_.""" + return "/vol/%s/%s%s" % ( + flexvol_name, TEMPLATE_LUN_PREFIX, int(template_db_id) + ) + + +def assert_iscsi_template_cache_lun(testcase, ontap, svm_name, flexvol_name, + template_db_id): + """Assert the deterministic template-cache LUN exists on ONTAP.""" + path = template_cache_lun_path(flexvol_name, template_db_id) + lun = ontap.get_lun(svm_name, path) + testcase.assertIsNotNone( + lun, + "Expected template cache LUN at %s on SVM %s" % (path, svm_name), + ) + return lun + + +def assert_no_iscsi_template_cache_lun(testcase, ontap, svm_name, flexvol_name, + template_db_id): + """Assert the template-cache LUN is absent.""" + path = template_cache_lun_path(flexvol_name, template_db_id) + lun = ontap.get_lun(svm_name, path) + testcase.assertIsNone( + lun, + "Template cache LUN should be absent at %s" % path, + ) + + +def count_iscsi_template_cache_luns(ontap, svm_name, flexvol_name, + template_db_id=None): + """Count cs_tmpl_* LUNs in the FlexVol (optionally for one template id).""" + luns = ontap.list_luns_in_volume(svm_name, flexvol_name) or [] + prefix = "/vol/%s/%s" % (flexvol_name, TEMPLATE_LUN_PREFIX) + matches = [l for l in luns if l.get("name", "").startswith(prefix)] + if template_db_id is not None: + exact = template_cache_lun_path(flexvol_name, template_db_id) + matches = [l for l in matches if l.get("name") == exact] + return len(matches) + + +def count_luns_excluding_template_cache(ontap, svm_name, flexvol_name): + """Count non-cache LUNs (volume clones / blank volumes) in the FlexVol.""" + luns = ontap.list_luns_in_volume(svm_name, flexvol_name) or [] + prefix = "/vol/%s/%s" % (flexvol_name, TEMPLATE_LUN_PREFIX) + return len([l for l in luns if not l.get("name", "").startswith(prefix)]) + + +def _normalize_nfs_path(install_path): + """Strip leading slash for OntapRestClient.list_files_in_volume paths.""" + if not install_path: + return "/" + path = install_path if install_path.startswith("/") else "/" + install_path + # Parent directory listing: if path is a file, list its parent + if path.endswith("/"): + return path.rstrip("/") or "/" + parent = path.rsplit("/", 1)[0] + return parent or "/" + + +def assert_nfs_template_cache_file(testcase, ontap, flexvol_name, install_path): + """ + Assert the NFS cache file referenced by install_path exists in the FlexVol. + + ``install_path`` comes from ``template_spool_ref.install_path`` (relative or + absolute path inside the FlexVol as reported by the KVM copy). + """ + testcase.assertTrue( + install_path, + "install_path is required to locate the NFS template cache file", + ) + file_name = install_path.rstrip("/").rsplit("/", 1)[-1] + parent = _normalize_nfs_path(install_path) + names = ontap.list_files_in_volume(flexvol_name, path=parent) + # Also try volume root if parent listing is empty (path styles differ) + if file_name not in names: + root_names = ontap.list_files_in_volume(flexvol_name, path="/") + testcase.assertIn( + file_name, root_names + names, + "Expected NFS template cache file '%s' under '%s' or '/' in " + "FlexVol '%s'; listed parent=%s root=%s" + % (file_name, parent, flexvol_name, names, root_names), + ) + return file_name + + +def wait_for_spool_ref(db_connection, pool_db_id, template_db_id, + timeout=600, interval=10): + """ + Poll until template_spool_ref exists and is Ready/DOWNLOADED. + + Returns the spool_ref dict, or None on timeout. + """ + deadline = time.time() + timeout + last = None + while time.time() < deadline: + last = get_template_spool_ref( + db_connection, pool_db_id, template_db_id + ) + if last and str(last.get("download_state", "")).upper() == "DOWNLOADED" \ + and str(last.get("state", "")) == "Ready": + return last + time.sleep(interval) + return last + + +def assert_no_nfs_template_cache_file(testcase, ontap, flexvol_name, install_path): + """Assert the NFS cache file referenced by install_path is absent.""" + testcase.assertTrue( + install_path, + "install_path is required to assert NFS cache file absence", + ) + file_name = install_path.rstrip("/").rsplit("/", 1)[-1] + parent = _normalize_nfs_path(install_path) + names = ontap.list_files_in_volume(flexvol_name, path=parent) or [] + root_names = ontap.list_files_in_volume(flexvol_name, path="/") or [] + testcase.assertNotIn( + file_name, names + root_names, + "NFS template cache file '%s' should be absent under '%s' or '/' in " + "FlexVol '%s'" % (file_name, parent, flexvol_name), + ) + + +def get_template_size_bytes(db_connection, template_db_id): + """Return vm_template.size (bytes) for the given DB id, or None.""" + rows = db_connection.execute( + "SELECT size FROM vm_template WHERE id = %s" % int(template_db_id) + ) + if not rows or rows[0][0] is None: + return None + return int(rows[0][0]) diff --git a/test/integration/plugins/ontap/iscsi/__init__.py b/test/integration/plugins/ontap/iscsi/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/instance/__init__.py b/test/integration/plugins/ontap/iscsi/instance/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/instance/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py new file mode 100644 index 000000000000..ab46b96e6a3b --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py @@ -0,0 +1,2030 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI data volume +lifecycle with a running virtual machine. + +Covers TDS (section 10) iSCSI VM volume scenarios: + + TDS Approach-1 SN 27 — Create CS Volume and allocate it to an Instance (iSCSI) + (attach data volume to running VM; verify LUN-map) + TDS VM Stop (iSCSI) — Stop running VM; verify LUN-maps are removed + TDS VM Start (iSCSI) — Start stopped VM; verify LUN-maps are re-created + TDS Detach (iSCSI) — Detach data volume; verify LUN-map removed + +Key iSCSI behaviour verified at each step via ONTAP REST API: + - createVolume → LUN is created inside the pool's FlexVol + - attachVolume → LUN-map is created linking the LUN to the host's igroup + - stopVirtualMachine → LUN-map is removed (LUN stays; just unmapped) + - startVirtualMachine → LUN-map is re-created + - detachVolume → LUN-map is removed + +Tests are numbered test_01 ... test_27, plus test_03a and test_03b, and must +run in that order. Each step builds on the shared state established by the +previous step. + +The VM is deployed with a compute offering whose storage tags match the pool, +so its ROOT volume lands on the ONTAP pool and exercises the primary +template cache (``cs_tmpl_`` LUN: seed / reuse / survive VM delete) +alongside the data-volume resize and snapshot workflow. LUN-map assertions are +filtered to the data volume's LUN, so the ROOT and cache LUNs in the same +FlexVol do not affect them. + +Workflow: + 01 Create iSCSI primary storage pool on ONTAP (tagged -tmpl-cache) + 02 Create and grow a detached CloudStack data volume (LUN on ONTAP) + 03 Deploy a VM — ROOT on ONTAP; seeds the cs_tmpl_* cache LUN + 03a Deploy a second VM — reuses the cache LUN (one more ROOT clone LUN only) + 03b Expunge the second VM — its ROOT LUN is removed; cache LUN survives + 04 Attach iSCSI data volume to running VM (LUN-map created) + 05 Stop VM — LUN-map for attached volume is removed from ONTAP + 06 Grow the attached volume and ONTAP LUN while the VM is stopped + 07 Verify shrinking the ONTAP LUN is rejected without changing its size + 08 Start VM — LUN-map is re-created on ONTAP + 09 Open an SSH port forward to the VM; write ROOT and DATA files + 10 Snapshot the ROOT volume of the running VM + 11 Snapshot the DATA volume of the running VM (kept for step 14) + 12 Revert a volume snapshot while the VM is Running — rejected + 13 ROOT volume revert: write before-snapshot, snapshot, write + after-snapshot, revert, file is back to before-snapshot + 14 DATA: snapshot again, revert to the step-11 snapshot then to this + one, then delete both while the VM is still running + 15 Write before-s1; VM snapshot of the running VM (kept for step 17) + 16 Write before-s2, stop, VM snapshot (kept for step 17) + 17 Revert to the step-15 snapshot, then to the step-16 snapshot, then + delete both while the VM is still running + 18 Two VM snapshot requests for the same VM at once — both succeed + 19 Deploy a second VM whose ROOT LUN is on the same pool + 20 Revert VM-A's ROOT; VM-B's file is unchanged + 21 VM snapshots of both VMs at the same time + 22 Volume snapshot of VM-A while VM-B's ROOT reverts + 23 VM snapshot of VM-A while VM-B reverts to its VM snapshot + 24 Detach data volume from running VM — LUN-map removed + 25 Grow the detached data volume + 26 Expunge the second VM while it still has a VM snapshot + 27 Destroy VM (cache LUN still survives), delete data volume, delete pool + +Snapshot steps only create and verify; tearDown deletes every snapshot the +step created and waits for its FlexVol snapshot to go. Step 11 keeps its +DATA snapshot so step 14 can revert to it and then delete both DATA +snapshots while the VM is running. Steps 15 and 16 keep their VM snapshots +so step 17 can revert to them and then delete both while the VM is running. + +Guest login for the CentOS 5.5 template comes from ontap.cfg +(cloudstack.templateUsername / templatePassword); see guest_ssh.py. + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI initiator configured (iqn.* IQN) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + - At least one KVM template must be fully downloaded and ready (isready=True) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_vm_volume_attach_iscsi.py -v +""" + +import base64 +import logging +import random +import re +import threading +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + startVirtualMachine as startVirtualMachineAPI, + stopVirtualMachine as stopVirtualMachineAPI, +) +from marvin.lib.base import ServiceOffering, Snapshot, StoragePool, VmSnapshot, Volume +from marvin.lib.common import list_storage_pools + +from guest_ssh import GuestSsh, guest_credentials, new_token, open_ssh_port +from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config +from helpers import template_cache_util as tcu +from helpers.template_cache_base import ( + find_ready_kvm_template, + tagged_compute_offering_data, + template_cache_capacity_bytes, + template_cache_tags, +) + +logger = logging.getLogger("TestOntapVMVolumeAttachISCSI") + +TAG = "iscsi_vm_workflow" + + +# --------------------------------------------------------------------------- +# Utility functions +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=300, + interval=10): + """Block until the VM reaches target_state or timeout expires.""" + deadline = time.time() + timeout + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms and vms[0].state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + return None + + +def _run_concurrent(*callables): + """Start all callables at the same moment; return [(result, exception)].""" + results = [(None, None)] * len(callables) + barrier = threading.Barrier(len(callables)) + + def worker(index, fn): + try: + barrier.wait(timeout=60) + results[index] = (fn(), None) + except Exception as exc: + results[index] = (None, exc) + + threads = [threading.Thread(target=worker, args=(i, fn)) + for i, fn in enumerate(callables)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return results + + +def _is_rejected(exc): + """True when CloudStack or ONTAP refused the call because another one was in progress.""" + text = str(exc).lower() + return any(token in text for token in ( + "internal error", + "failed to create snapshot", + "failed to create instance snapshot", + "failed to revert", + "another operation", + "other active", + "try again later", + "in progress", + "busy", + )) + + +def _snapshot_and_revert_with_retry(snapshot_fn, revert_fn, attempts=3, pause=15): + """Run a snapshot and a revert together. + + A call that is rejected is retried on its own. A call that already + succeeded is not repeated. Returns the snapshot from snapshot_fn. + """ + snap = None + reverted = False + last_errors = [] + for attempt in range(1, attempts + 1): + jobs = [] + if snap is None: + jobs.append(("snapshot", snapshot_fn)) + if not reverted: + jobs.append(("revert", revert_fn)) + results = _run_concurrent(*(fn for _, fn in jobs)) + last_errors = [] + retryable = True + for (kind, _), (result, exc) in zip(jobs, results): + if exc is None: + if kind == "snapshot": + snap = result + else: + reverted = True + continue + last_errors.append("%s: %s" % (kind, exc)) + if not _is_rejected(exc): + retryable = False + if snap is not None and reverted: + return snap + if not retryable or attempt == attempts: + break + logger.warning("snapshot/revert rejected, retry %d/%d: %s", + attempt, attempts, last_errors) + time.sleep(pause) + raise AssertionError( + "Snapshot-while-revert failed after %d attempt(s): %s" + % (attempt, last_errors)) + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + # Used when ontap.cfg has neither templateCacheCapacitybytes nor capacitybytes. + DEFAULT_POOL_SIZE = 50 * 1024 ** 3 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if not capacitybytes: + capacitybytes = TestData.DEFAULT_POOL_SIZE + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-vm@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-VM", + "username": "ontap_iscsi_vm_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIVM_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapVMVolumeAttachISCSI(OntapTestBase): + """ + Tests iSCSI ONTAP data volume lifecycle with a running CloudStack VM whose + ROOT volume is on the ONTAP pool (primary template cache). + All tests are sequential — state is carried on class attributes. + """ + + PROTOCOL = "ISCSI" + + # ---- extra shared state beyond OntapTestBase ----------------------- + vm = None + vm2 = None # second VM used only to verify template-cache reuse + vm2_baseline_luns = None # non-cache LUN count before the second deploy + template_id = None + template_db_id = None # numeric vm_template.id (template_spool_ref key) + pool_db_id = None # numeric storage_pool.id (template_spool_ref key) + service_offering_id = None + network_id = None + _created_network_id = None # network created by this suite for Advanced zones + initial_volume_size = None + target_volume_size = None + vm_ssh = None # (public_ip, public_port) forwarded to vm:22 + vm2_ssh = None + vm_snap_running = None # step 15 snapshot, reused by step 17 + vm_snap_stopped = None # step 16 snapshot, reused by step 17 + token_s1 = None + token_s2 = None + kept_ontap_snaps = None + data_snap_older = None # step 11 DATA snapshot, reused by step 14 + data_token_older = None + kept_data_ontap_snaps = None + guest_user = "root" + guest_password = "password" + + _vol_name_prefix = "OntapISCSIVM" + + # ---- setup --------------------------------------------------------- + + @classmethod + def setUpClass(cls): + super(TestOntapVMVolumeAttachISCSI, cls).setUpClass() + testclient = super( + TestOntapVMVolumeAttachISCSI, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + # Dedicated tag shared by the pool and the compute offering so the + # VM's ROOT volume is placed on this ONTAP pool (template cache path). + tags = template_cache_tags(iscsi_cfg, "iscsi") + capacitybytes = template_cache_capacity_bytes(pool_cfg) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + cls.guest_user, cls.guest_password = guest_credentials(config) + + # Discover a ready user KVM template (exclude SYSTEM type) + template = find_ready_kvm_template(cls.apiClient, cls.zone.id) + cls.template_id = template.id if template is not None else None + if cls.template_id is None: + logger.warning( + "No ready KVM user template found — VM tests will be skipped." + ) + else: + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) + + service_offering = ServiceOffering.create( + cls.apiClient, tagged_compute_offering_data(tags, "OntapISCSIVMSO") + ) + cls._cleanup.append(service_offering) + cls.service_offering_id = service_offering.id + # Network ID for VM deployment + cls.network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + # Find a network already accessible to the test account + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + # Create an Isolated guest network for the test account + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + offerings[0] if offerings else None + ) + if snat_offering: + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "ontap-iscsi-vm-net-%d" % random.randint( + 0, 9999) + cn_cmd.displaytext = "ONTAP iSCSI VM test network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def tearDownClass(cls): + """ + Safety-net cleanup: destroy VMs if still alive, delete the guest + network created for Advanced zones (if not already deleted by test_27), + then delegate pool/volume/account cleanup to the base class. + """ + cls._delete_kept_data_snapshot() + cls._delete_kept_vm_snapshots() + for vm in (cls.vm2, cls.vm): + if vm is None: + continue + try: + cls._stop_and_expunge_vm(vm.id) + except Exception as e: + logger.warning("tearDownClass: could not stop/destroy VM %s: " + "%s" % (vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as de: + logger.warning("tearDownClass: could not destroy VM %s: " + "%s" % (vm.id, de)) + cls.vm2 = None + cls.vm = None + + # Delete the guest network created for this account in Advanced zones. + # test_27 deletes it on the happy path; this is the fallback for + # mid-suite failures. + if cls._created_network_id is not None: + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + cls._created_network_id = None + except Exception as e: + logger.warning( + "tearDownClass: could not delete network %s: %s" + % (cls._created_network_id, e)) + + super(TestOntapVMVolumeAttachISCSI, cls).tearDownClass() + + @classmethod + def _stop_and_expunge_vm(cls, vm_id): + """Force-stop the VM if needed, then destroy it with expunge.""" + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + state = vms[0].state if vms else "unknown" + if state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=120) + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + + # ---- helpers ------------------------------------------------------- + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIVM_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _poll_volume_field(self, vol_id, field, target, timeout=120, + interval=5): + """Poll a volume field until it matches target; return the volume.""" + deadline = time.time() + timeout + while time.time() < deadline: + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) + if vols: + val = getattr(vols[0], field, None) + if val == target: + return vols[0] + time.sleep(interval) + return None + + def _data_lun_name(self, volume=None): + """ + Full ONTAP path of the LUN backing the CloudStack data volume. + + UnifiedSANStrategy names the LUN after the CloudStack volume with + hyphens replaced by underscores, inside the pool's FlexVol. + """ + volume = volume or self.__class__.volume + if volume is None or self.__class__.pool is None: + return None + return "/vol/%s/%s" % (self.__class__.pool.name, + volume.name.replace("-", "_")) + + def _lun_maps(self, volume=None): + """ + Return LUN-maps belonging to the data volume's LUN only. + + The same FlexVol also holds the VM's ROOT LUN whenever the allocator + places the root disk on this pool, so the unfiltered map list is not a + measure of the data volume's accessibility. + """ + lun_name = self._data_lun_name(volume) + if lun_name is None: + return [] + return [m for m in self.ontap.list_lun_maps_for_volume( + self.svm_name, self.__class__.pool.name) + if m.get("lun", {}).get("name") == lun_name] + + def _data_luns(self, volume=None): + """Return LUN records for the data volume's LUN only.""" + lun_name = self._data_lun_name(volume) + if lun_name is None: + return [] + return [lun for lun in self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + if lun.get("name") == lun_name] + + + def _data_lun_size(self): + """Return the ONTAP data LUN size in bytes.""" + lun_name = self._data_lun_name() + lun = self.ontap.get_lun(self.svm_name, lun_name) + self.assertIsNotNone( + lun, "ONTAP LUN '%s' was not found" % lun_name) + size = int(lun.get("space", {}).get("size", 0)) + self.assertGreater( + size, 0, "ONTAP LUN '%s' did not report space.size" % lun_name) + return size + + # ---- snapshot step helpers (steps 09-23) --------------------------- + + def setUp(self): + super(TestOntapVMVolumeAttachISCSI, self).setUp() + self._volume_snapshot_ids = [] + self._vm_snapshot_ids = [] + self._extra_volume_ids = [] + self._created_ontap_snaps = set() + + def tearDown(self): + """ + Delete the snapshots and extra volumes a step created, then check + their FlexVol snapshots are gone from ONTAP. + """ + for snap_id in self._vm_snapshot_ids: + try: + VmSnapshot.deleteVMSnapshot(self.apiClient, snap_id) + except Exception as exc: + logger.warning("tearDown: VM snapshot %s: %s" % (snap_id, exc)) + for snap_id in self._volume_snapshot_ids: + try: + Snapshot({"id": snap_id}).delete(self.apiClient) + except Exception as exc: + logger.warning("tearDown: volume snapshot %s: %s" % (snap_id, exc)) + for vol_id in self._extra_volume_ids: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol_id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + logger.warning("tearDown: volume %s: %s" % (vol_id, exc)) + if self._created_ontap_snaps and not self._wait_ontap_snaps_gone( + self._created_ontap_snaps, fail=False): + logger.warning("tearDown: FlexVol snapshots still present: %s" + % sorted(self._created_ontap_snaps)) + super(TestOntapVMVolumeAttachISCSI, self).tearDown() + + @classmethod + def _destroy_second_vm(cls): + if cls.vm2 is None: + return + try: + cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + cmd.id = cls.vm2.id + cmd.expunge = True + cls.apiClient.destroyVirtualMachine(cmd) + except Exception as exc: + logger.warning("Could not destroy second VM %s: %s" % (cls.vm2.id, exc)) + cls.vm2 = None + cls.vm2_ssh = None + + @classmethod + def _delete_kept_vm_snapshots(cls): + """Delete the step 15/16 snapshots when step 17 did not take them over.""" + for snap in (cls.vm_snap_running, cls.vm_snap_stopped): + if snap is None: + continue + try: + VmSnapshot.deleteVMSnapshot(cls.apiClient, snap.id) + except Exception as exc: + logger.warning("tearDownClass: VM snapshot %s: %s" % (snap.id, exc)) + cls.vm_snap_running = None + cls.vm_snap_stopped = None + cls.token_s1 = None + cls.token_s2 = None + cls.kept_ontap_snaps = set() + + @classmethod + def _delete_kept_data_snapshot(cls): + """Delete the step 11 DATA snapshot when step 14 did not take it over.""" + snap = cls.data_snap_older + if snap is None: + return + try: + Snapshot({"id": snap.id}).delete(cls.apiClient) + except Exception as exc: + logger.warning("tearDownClass: volume snapshot %s: %s" % (snap.id, exc)) + cls.data_snap_older = None + cls.data_token_older = None + cls.kept_data_ontap_snaps = set() + + def _keep_data_snapshot(self, snap, ontap_names, token): + """Leave the step 11 DATA snapshot in place for step 14.""" + if snap.id in self._volume_snapshot_ids: + self._volume_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + self.__class__.data_snap_older = snap + self.__class__.data_token_older = token + self.__class__.kept_data_ontap_snaps = names + + def _adopt_kept_data_snapshot(self): + """Hand the step 11 DATA snapshot to step 14.""" + snap = self.__class__.data_snap_older + token = self.__class__.data_token_older + if snap is None or not token: + self.skipTest("DATA snapshot from test_11 is required") + ontap_names = set(self.__class__.kept_data_ontap_snaps or []) + self._volume_snapshot_ids.append(snap.id) + self._created_ontap_snaps |= ontap_names + self.__class__.data_snap_older = None + self.__class__.data_token_older = None + self.__class__.kept_data_ontap_snaps = set() + return snap, token, ontap_names + + def _delete_volume_snapshot_while_running(self, vm, snap, ontap_names, token): + """Delete a volume snapshot while its VM is still running.""" + Snapshot({"id": snap.id}).delete(self.apiClient) + if snap.id in self._volume_snapshot_ids: + self._volume_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + self._wait_until_gone( + lambda: self._volume_snapshot_listed(snap.id), + "Volume snapshot %s still listed after delete" % snap.id) + self._wait_ontap_snaps_gone(names) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_marker(token, disk="DATA", msg="after volume snapshot delete") + + def _delete_vm_snapshot_while_running(self, vm, snap, ontap_names, token): + """Delete a VM snapshot while that VM is still running.""" + VmSnapshot.deleteVMSnapshot(self.apiClient, snap.id) + if snap.id in self._vm_snapshot_ids: + self._vm_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + self._wait_until_gone( + lambda: any(item.id == snap.id for item in self._vm_snapshots_of(vm.id)), + "VM snapshot %s still listed after delete" % snap.id) + self._wait_ontap_snaps_gone(names) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_marker(token, msg="after VM snapshot delete") + + def _ontap_names_for_snap(self, names, snap, kind): + prefix = "cs" if kind == "volume" else "vm" + table = "snapshots" if kind == "volume" else "vm_snapshots" + db_id = tcu.get_db_id(self.dbConnection, table, snap.id) + suffix = "_%s%s" % (prefix, db_id) + return {name for name in names if str(name).endswith(suffix)} + + def _keep_vm_snapshot(self, snap, ontap_names, which): + """Leave this VM snapshot in place for step 17.""" + if snap.id in self._vm_snapshot_ids: + self._vm_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + kept = self.__class__.kept_ontap_snaps + if kept is None: + kept = set() + self.__class__.kept_ontap_snaps = kept + kept.update(names) + setattr(self.__class__, which, snap) + + def _adopt_kept_vm_snapshots(self): + """Hand the step 15 and 16 snapshots to this step's tearDown.""" + s1 = self.__class__.vm_snap_running + s2 = self.__class__.vm_snap_stopped + token_a = self.__class__.token_s1 + token_b = self.__class__.token_s2 + if s1 is None or s2 is None or not token_a or not token_b: + self.skipTest("VM snapshots from test_15 and test_16 are required") + ontap_names = set(self.__class__.kept_ontap_snaps or []) + self._vm_snapshot_ids.extend([s1.id, s2.id]) + self._created_ontap_snaps |= ontap_names + self.__class__.vm_snap_running = None + self.__class__.vm_snap_stopped = None + self.__class__.token_s1 = None + self.__class__.token_s2 = None + self.__class__.kept_ontap_snaps = set() + return s1, s2, token_a, token_b, ontap_names + + def _require_snapshot_vm(self, ssh=True): + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + if ssh and self.__class__.vm_ssh is None: + self.skipTest("Guest SSH not set up — test_09 must pass first") + self._start_vm(self.__class__.vm.id) + return self.__class__.vm + + def _require_second_vm(self): + self._require_snapshot_vm() + if self.__class__.vm2 is None or self.__class__.vm2_ssh is None: + self.skipTest("Second VM not deployed — test_19 must pass first") + self._start_vm(self.__class__.vm2.id) + return self.__class__.vm2 + + def _vm_state(self, vm_id): + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + return vms[0].state if vms else None + + def _stop_vm(self, vm_id): + if (self._vm_state(vm_id) or "").lower() == "stopped": + return + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = vm_id + cmd.forced = True + self.apiClient.stopVirtualMachine(cmd) + self._poll_vm_state(vm_id, "Stopped", timeout=600) + + def _start_vm(self, vm_id): + if (self._vm_state(vm_id) or "").lower() == "running": + return + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = vm_id + self.apiClient.startVirtualMachine(cmd) + self._poll_vm_state(vm_id, "Running", timeout=900) + + def _root_volume(self, vm_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.type = "ROOT" + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + self.assertTrue(vols, "No ROOT volume found for VM %s" % vm_id) + return vols[0] + + def _guest(self, which="vm"): + host, port = getattr(self.__class__, "%s_ssh" % which) + return GuestSsh(host, port, self.guest_user, self.guest_password) + + def _write_marker(self, token, which="vm", disks=("ROOT",)): + with self._guest(which) as guest: + if "ROOT" in disks: + guest.write_root_marker(token) + if "DATA" in disks: + guest.write_data_marker(token) + + def _assert_marker(self, token, which="vm", disk="ROOT", msg=""): + with self._guest(which) as guest: + found = guest.read_data_marker() if disk == "DATA" else guest.read_root_marker() + self.assertEqual(found, token, "%s: %s marker on %s is %r, expected %r" + % (msg, disk, which, found, token)) + + def _ontap_snap_names(self): + return {s.get("name") for s in self.ontap.list_snapshots(self.__class__.pool.name)} + + def _assert_new_ontap_snaps(self, before, snaps, kind, msg): + """Return the FlexVol snapshot created for each CloudStack snapshot. + + A volume snapshot is stored as ``..._cs`` and a VM snapshot as + ``..._vm``. ONTAP snapshots from its own schedule, such as + ``hourly.*`` or ``Anti_ransomware_*``, are ignored. + """ + new = self._ontap_snap_names() - before + prefix = "cs" if kind == "volume" else "vm" + table = "snapshots" if kind == "volume" else "vm_snapshots" + matched = set() + missing = [] + for snap in snaps: + db_id = tcu.get_db_id(self.dbConnection, table, snap.id) + suffix = "_%s%s" % (prefix, db_id) + hits = sorted(name for name in new if name.endswith(suffix)) + if len(hits) != 1: + missing.append("%s (suffix %s, matches %s)" % (snap.id, suffix, hits)) + else: + matched.add(hits[0]) + self.assertFalse( + missing, + "%s: FlexVol snapshot missing for %s; new FlexVol snapshots: %s" + % (msg, missing, sorted(new))) + ignored = new - matched + if ignored: + logger.info("%s: ignoring FlexVol snapshots not created by this step: %s", + msg, sorted(ignored)) + self._created_ontap_snaps |= matched + return matched + + def _assert_ontap_snaps_present(self, names, msg): + missing = set(names) - self._ontap_snap_names() + self.assertFalse(missing, "%s: FlexVol snapshots missing: %s" % (msg, sorted(missing))) + + def _wait_ontap_snaps_gone(self, names, timeout=180, fail=True): + names = set(names) + deadline = time.time() + timeout + left = names + while time.time() < deadline: + left = names & self._ontap_snap_names() + if not left: + return True + time.sleep(5) + if fail: + self.fail("FlexVol snapshots %s still present after %ds" % (sorted(left), timeout)) + return False + + def _take_volume_snapshot(self, volume_id): + snap = Snapshot.create(self.apiClient, volume_id, + account=self.account.name, domainid=self.domain.id) + self._volume_snapshot_ids.append(snap.id) + self.assertEqual(snap.state, "BackedUp", + "Volume snapshot %s is %s, expected BackedUp" % (snap.id, snap.state)) + return snap + + def _take_vm_snapshot(self, vm_id): + snap = VmSnapshot.create(self.apiClient, vm_id, snapshotmemory="false", + name="ontap_%d" % random.randint(0, 999999)) + self._vm_snapshot_ids.append(snap.id) + self.assertEqual(snap.state, "Ready", + "VM snapshot %s is %s, expected Ready" % (snap.id, snap.state)) + return snap + + def _vm_snapshots_of(self, vm_id): + return VmSnapshot.list(self.apiClient, virtualmachineid=vm_id, listall=True) or [] + + def _volume_snapshot_listed(self, snap_id): + listed = Snapshot.list(self.apiClient, id=snap_id, listall=True) or [] + return any(getattr(item, "id", None) == snap_id for item in listed) + + def _wait_until_gone(self, still_there, msg, timeout=180): + deadline = time.time() + timeout + while time.time() < deadline and still_there(): + time.sleep(5) + self.assertFalse(still_there(), msg) + + def _volume_revert_flow(self, disk): + """Write A, snapshot, write B, stop, revert, start; expect A.""" + vm = self._require_snapshot_vm() + volume_id = (self.__class__.volume.id if disk == "DATA" + else self._root_volume(vm.id).id) + token_a = new_token("before-volume-snapshot") + token_b = new_token("after-volume-snapshot") + self._write_marker(token_a, disks=(disk,)) + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(volume_id) + new = self._assert_new_ontap_snaps(before, [snap], "volume", "%s volume snapshot" % disk) + self._write_marker(token_b, disks=(disk,)) + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, snap.id) + self._assert_ontap_snaps_present(new, "after revert") + self._start_vm(vm.id) + self._assert_marker(token_a, disk=disk, + msg="after revert (write after snapshot was %s)" % token_b) + + # ================================================================== + # Test steps + # ================================================================== + + # ------------------------------------------------------------------ + # Step 01 — Create iSCSI ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_01_create_iscsi_pool(self): + """ + Create an iSCSI primary storage pool on ONTAP. + Verifies: + - Pool reaches 'Up' state; type is 'OntapiSCSI' + - ONTAP: FlexVol is online + - ONTAP: igroup exists for every host in the cluster that has an IQN + """ + pool = self._create_pool() + self.__class__.pool = pool + self.__class__.pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + + self.assertEqual(pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state) + self.assertEqual(pool.type, "OntapiSCSI", + "Pool type should be 'OntapiSCSI', got '%s'" % pool.type) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name) + self.assertEqual(ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online'") + + # ------------------------------------------------------------------ + # Step 02 — Create iSCSI data volume (LUN on ONTAP) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_02_create_ontap_data_volume(self): + """ + Allocate and grow a detached CloudStack data volume on the iSCSI + ONTAP pool. + Verifies: + - createVolume returns a volume object + - ONTAP: at least one LUN is created inside the pool's FlexVol + - detached resize updates CloudStack API, DB, and LUN space.size + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + luns = self._data_luns() + self.assertTrue( + len(luns) > 0, + "Expected the data volume's LUN '%s' in ONTAP FlexVol '%s' after " + "volume creation, found none" + % (self._data_lun_name(), self.__class__.pool.name) + ) + + volume = self._poll_volume_size(vol.id, int(vol.size)) + detached_target = ( + (int(volume.size) // (1024 ** 3)) + 1 + ) * (1024 ** 3) + self._resize_volume(vol.id, detached_target // (1024 ** 3)) + self._poll_volume_size(vol.id, detached_target) + self.assertEqual( + self._get_db_volume_size(vol.id), detached_target, + "DB size does not match detached iSCSI grow target") + self.assertEqual( + self._data_lun_size(), detached_target, + "ONTAP LUN size does not match detached grow target") + + + # ------------------------------------------------------------------ + # Step 03 — Deploy a VM + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03_deploy_vm(self): + """ + Deploy a VM using the first available ready KVM template and the + compute offering tagged to match the ONTAP pool. + Verifies: + - VM reaches Running state + - ROOT volume is placed on the ONTAP pool + - template_spool_ref for pool + template is Ready / DOWNLOADED + (local_path = cache LUN uuid) + - ONTAP: exactly one cache LUN /vol//cs_tmpl_ + - ONTAP: the iSCSI data volume's LUN is NOT yet mapped (no VM + attachment has been performed yet) + """ + if self.__class__.template_id is None: + self.skipTest( + "No ready KVM user template available — waiting for template " + "download to complete" + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.__class__.vm = vm + + result = self._poll_vm_state(vm.id, "Running") + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after deploy, got '%s'" % result.state + ) + + pool = self.__class__.pool + self._assert_root_on_pool(vm.id, pool) + spool = self._wait_for_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # Data volume LUN-map must not exist yet (volume not yet attached) + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 data-volume LUN-maps before volume attach, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ------------------------------------------------------------------ + # Step 03a — Deploy a second VM — reuses the template cache LUN + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03a_deploy_second_vm_reuses_template_cache(self): + """ + Deploy a second VM from the same template and offering. + Verifies: + - VM reaches Running; its ROOT volume is on the ONTAP pool + - Still exactly one template_spool_ref (cache reused, not re-seeded) + - ONTAP: still exactly one cs_tmpl_* cache LUN + - ONTAP: exactly one additional non-cache LUN (the VM-2 ROOT clone) + """ + self.assertIsNotNone(self.__class__.vm, + "VM absent — test_03 must pass first") + pool = self.__class__.pool + baseline = self._count_non_cache_luns(pool) + self.__class__.vm2_baseline_luns = baseline + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm2 = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm2, "deployVirtualMachine returned None") + self.__class__.vm2 = vm2 + self._poll_vm_state(vm2.id, "Running") + + self._assert_root_on_pool(vm2.id, pool) + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + self.assertEqual( + self._count_non_cache_luns(pool), baseline + 1, + "Expected one additional non-cache LUN for the VM-2 ROOT clone", + ) + + # ------------------------------------------------------------------ + # Step 03b — Expunge the second VM — template cache LUN survives + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03b_expunge_second_vm_template_cache_survives(self): + """ + Destroy (expunge) the second VM. The primary template cache is only + removed by storage GC / pool deletion, never by VM lifecycle. + Verifies: + - ONTAP: the VM-2 ROOT LUN is removed (non-cache LUN count back to + the pre-deploy baseline) + - template_spool_ref is still Ready / DOWNLOADED + - ONTAP: the cs_tmpl_* cache LUN is still present + """ + self.assertIsNotNone(self.__class__.vm2, + "Second VM absent — test_03a must pass first") + pool = self.__class__.pool + + self._stop_and_expunge_vm(self.__class__.vm2.id) + self.__class__.vm2 = None + + self._wait_for_non_cache_lun_count( + pool, self.__class__.vm2_baseline_luns) + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # ------------------------------------------------------------------ + # Step 04 — Attach iSCSI volume to running VM (TDS SN 27 iSCSI) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_04_attach_volume_to_vm(self): + """ + Attach the iSCSI data volume to the running VM. + Covers TDS Approach-1 SN 27 (iSCSI): + - attachVolume completes successfully + - CloudStack: volume shows virtualmachineid set + - ONTAP: a LUN-map is created linking the data LUN to the host's + igroup (the LUN is now accessible to the VM's KVM host) + """ + if self.__class__.vm is None: + self.skipTest( + "VM not deployed — test_03 was skipped (no ready template)" + ) + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + cmd = attachVolumeAPI.attachVolumeCmd() + cmd.id = self.__class__.volume.id + cmd.virtualmachineid = self.__class__.vm.id + self.apiClient.attachVolume(cmd) + + # Poll until virtualmachineid is set on the volume + result = self._poll_volume_field( + self.__class__.volume.id, "virtualmachineid", + self.__class__.vm.id, timeout=120) + self.assertIsNotNone( + result, + "Volume virtualmachineid was not set after attachVolume" + ) + + # ONTAP: at least one LUN-map must exist for the pool's FlexVol + lun_maps = self._lun_maps() + self.assertGreater( + len(lun_maps), 0, + "Expected ≥1 LUN-map after volume attach, found 0 — " + "LUN is not accessible to the VM's host" + ) + + initial_size = int(result.size) + self.__class__.initial_volume_size = initial_size + self.__class__.target_volume_size = ( + (initial_size // (1024 ** 3)) + 1 + ) * (1024 ** 3) + self.assertEqual( + self._data_lun_size(), initial_size, + "Initial ONTAP LUN size should match the CloudStack volume size") + + + # ------------------------------------------------------------------ + # Step 05 — Stop VM — LUN-maps should be removed + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_05_stop_vm_lun_unmapped(self): + """ + Stop the running VM while the iSCSI data volume is still attached. + Covers TDS VM Stop (iSCSI): 'Luns for the volumes under this VM + should be unmapped.' + Verifies: + - VM reaches Stopped state + - ONTAP: LUN-map is removed (LUN itself stays in the FlexVol) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = self.__class__.vm.id + self.apiClient.stopVirtualMachine(cmd) + + result = self._poll_vm_state(self.__class__.vm.id, "Stopped", + timeout=300) + self.assertEqual( + result.state, "Stopped", + "VM should be 'Stopped', got '%s'" % result.state + ) + + # ONTAP: LUN-map must be removed once VM is stopped + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 data-volume LUN-maps after VM stop, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ONTAP: LUN itself must still exist in the FlexVol + luns = self._data_luns() + self.assertTrue( + len(luns) > 0, + "Data volume LUN should still exist in ONTAP FlexVol after VM stop" + ) + + # ------------------------------------------------------------------ + + # ------------------------------------------------------------------ + # Step 06 — Grow attached volume while VM is stopped + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_06_grow_stopped_vm_volume(self): + """ + Grow the attached iSCSI volume while its managed KVM VM is stopped. + Verifies: + - CloudStack reports the target size and Ready state + - ONTAP reports the target LUN space.size + - The LUN remains unmapped while the VM is stopped + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + target_size = self.__class__.target_volume_size + self._resize_volume( + self.__class__.volume.id, target_size // (1024 ** 3)) + + volume = self._poll_volume_size( + self.__class__.volume.id, target_size) + self.assertIsNotNone( + volume, "CloudStack did not report the grown volume size") + + # DB: cloud.volumes.size must be updated by the driver's volumeDao call + db_size = self._get_db_volume_size(self.__class__.volume.id) + self.assertEqual( + db_size, target_size, + "cloud DB volumes.size (%s) does not match target (%s) after grow" + % (db_size, target_size)) + + # ONTAP: LUN space.size must reflect the new allocation + self.assertEqual( + self._data_lun_size(), target_size, + "ONTAP LUN space.size does not match the requested size") + self.assertEqual( + len(self._lun_maps()), 0, + "The data LUN should remain unmapped while the VM is stopped") + + # ------------------------------------------------------------------ + # Step 07 — Shrink must fail + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_07_shrink_rejected(self): + """ + Attempt to shrink the grown iSCSI volume back to its initial size. + Verifies: + - The ONTAP driver rejects shrink even with shrinkok=true + - CloudStack and ONTAP LUN sizes remain at the grown value + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + self.assertEqual( + self._get_db_volume_size(self.__class__.volume.id), + self.__class__.target_volume_size, + "Volume was not grown to the target size — test_06 must pass " + "before the shrink check") + + initial_gib = self.__class__.initial_volume_size // (1024 ** 3) + with self.assertRaises(Exception) as context: + self._resize_volume( + self.__class__.volume.id, initial_gib, shrink_ok=True) + self.assertIn( + "Unable to shrink volume", str(context.exception), + "Shrink failed for an unexpected reason") + + volume = self._poll_volume_size( + self.__class__.volume.id, self.__class__.target_volume_size) + self.assertIsNotNone( + volume, "CloudStack volume size changed after rejected shrink") + + # DB: size must remain at grown value after rejected shrink + db_size = self._get_db_volume_size(self.__class__.volume.id) + self.assertEqual( + db_size, self.__class__.target_volume_size, + "cloud DB volumes.size did not remain at the grown target after " + "the rejected iSCSI shrink") + + self.assertEqual( + self._data_lun_size(), self.__class__.target_volume_size, + "ONTAP LUN size changed after rejected shrink") + + # Step 08 — Start VM — LUN-maps should be re-created + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_08_start_vm_lun_remapped(self): + """ + Start the stopped VM. + Covers TDS VM Start (iSCSI): 'luns should be re-mapped again to + provide access.' + Verifies: + - VM reaches Running state + - ONTAP: LUN-map is re-created (LUN accessible to VM's host) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = self.__class__.vm.id + self.apiClient.startVirtualMachine(cmd) + + result = self._poll_vm_state(self.__class__.vm.id, "Running", + timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after start, got '%s'" % result.state + ) + + # ONTAP: LUN-map must be re-created after VM starts + lun_maps = self._lun_maps() + self.assertGreater( + len(lun_maps), 0, + "Expected ≥1 LUN-map after VM start (re-map), found 0" + ) + + # ------------------------------------------------------------------ + # Step 09 — Guest SSH access for snapshot checks + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_09_open_ssh_and_write_root_data_files(self): + """ + Prepare the running VM for snapshot checks. + Verifies: + - ROOT and DATA LUNs are both on the ONTAP test pool + - SSH login (ontap.cfg templateUsername/templatePassword) works + through a port forward on the network's source-NAT IP + - The DATA disk can be formatted, mounted and written + """ + vm = self._require_snapshot_vm(ssh=False) + if not self.__class__.network_id: + self.skipTest("Snapshot steps need an Advanced zone guest network for SSH") + pool_id = self.__class__.pool.id + root = self._root_volume(vm.id) + self.assertEqual(getattr(root, "storageid", None), pool_id, + "ROOT volume is on '%s', not the ONTAP test pool" + % getattr(root, "storage", None)) + vol_cmd = listVolumesAPI.listVolumesCmd() + vol_cmd.id = self.__class__.volume.id + vols = self.apiClient.listVolumes(vol_cmd) or [] + self.assertTrue(vols and getattr(vols[0], "virtualmachineid", None) == vm.id, + "DATA volume must be attached before the snapshot steps") + + host, port, _ = open_ssh_port(self.apiClient, vm.id, self.__class__.network_id) + self.__class__.vm_ssh = (host, port) + token = new_token("ssh-access-test") + self._write_marker(token, disks=("ROOT", "DATA")) + self._assert_marker(token, disk="ROOT", msg="guest access") + self._assert_marker(token, disk="DATA", msg="guest access") + + # ------------------------------------------------------------------ + # Steps 08-12 — Volume snapshots + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_10_root_volume_snapshot_create(self): + """ + VS-01: snapshot the ROOT volume of the running VM. + Verifies: state BackedUp; VM stays Running; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(self._root_volume(vm.id).id) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_new_ontap_snaps(before, [snap], "volume", "ROOT volume snapshot") + + @attr(tags=[TAG], required_hardware=True) + def test_11_data_volume_snapshot_create(self): + """ + VS-03: snapshot the attached DATA volume. + The snapshot is kept for step 14. + Verifies: state BackedUp; VM stays Running; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + token = new_token("data-s1") + self._write_marker(token, disks=("DATA",)) + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(self.__class__.volume.id) + self.assertEqual(self._vm_state(vm.id), "Running") + new = self._assert_new_ontap_snaps(before, [snap], "volume", "DATA volume snapshot") + self._keep_data_snapshot(snap, new, token) + + @attr(tags=[TAG], required_hardware=True) + def test_12_volume_revert_running_vm_rejected(self): + """ + VS-20: reverting a volume snapshot while the VM is Running fails. + Verifies: error says the Instance is not shut down; VM stays Running; + the FlexVol snapshot remains. + """ + vm = self._require_snapshot_vm() + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(self._root_volume(vm.id).id) + new = self._assert_new_ontap_snaps(before, [snap], "volume", "ROOT volume snapshot") + with self.assertRaises(Exception) as ctx: + Volume.revertToSnapshot(self.apiClient, snap.id) + self.assertIn("shutdown", str(ctx.exception).lower(), + "Unexpected error: %s" % ctx.exception) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_ontap_snaps_present(new, "after rejected revert") + + @attr(tags=[TAG], required_hardware=True) + def test_13_root_volume_revert(self): + """ + VS-12: write A, snapshot ROOT, write B, stop, revert, start. + Verifies: ROOT marker is A; the FlexVol snapshot survives the revert. + """ + self._volume_revert_flow("ROOT") + + @attr(tags=[TAG], required_hardware=True) + def test_14_data_volume_revert(self): + """ + Take a second DATA snapshot, revert to the step 11 snapshot, then to + this later one, and delete both while the VM is still running. + Verifies: + - After revert to the older snapshot, the DATA marker is the step 11 + token and the later snapshot is still listed + - After revert to the later snapshot, the DATA marker is that token + - Both snapshots and their FlexVol snapshots are gone; the VM stays + Running and the later marker is unchanged + """ + vm = self._require_snapshot_vm() + older, token_older, ontap_older = self._adopt_kept_data_snapshot() + token_latest = new_token("data-s2") + self._write_marker(token_latest, disks=("DATA",)) + before = self._ontap_snap_names() + latest = self._take_volume_snapshot(self.__class__.volume.id) + ontap_latest = self._assert_new_ontap_snaps( + before, [latest], "volume", "latest DATA snapshot") + self._write_marker(new_token("data-after-both"), disks=("DATA",)) + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, older.id) + self._start_vm(vm.id) + self._assert_marker(token_older, disk="DATA", + msg="after revert to the older snapshot") + self.assertTrue( + self._volume_snapshot_listed(latest.id), + "The later volume snapshot should still exist") + self._assert_ontap_snaps_present( + ontap_older | ontap_latest, "after revert to the older snapshot") + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, latest.id) + self._start_vm(vm.id) + self._assert_marker(token_latest, disk="DATA", + msg="after revert to the latest snapshot") + + self._delete_volume_snapshot_while_running( + vm, latest, ontap_latest, token_latest) + self._delete_volume_snapshot_while_running( + vm, older, ontap_older, token_latest) + + # ------------------------------------------------------------------ + # Steps 13-16 — VM snapshots + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_15_vm_snapshot_create_running(self): + """ + VM snapshot of the running VM (ROOT + DATA LUNs on one FlexVol). + Writes before-s1, then snapshots. The snapshot is kept for step 17. + Verifies: state Ready; VM stays Running; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + self.__class__.token_s1 = new_token("before-s1") + self._write_marker(self.__class__.token_s1, disks=("ROOT", "DATA")) + before = self._ontap_snap_names() + snap = self._take_vm_snapshot(vm.id) + self.assertEqual(self._vm_state(vm.id), "Running") + new = self._assert_new_ontap_snaps(before, [snap], "vm", "VM snapshot") + self._keep_vm_snapshot(snap, new, "vm_snap_running") + + @attr(tags=[TAG], required_hardware=True) + def test_16_vm_snapshot_create_stopped(self): + """ + VM snapshot of the stopped VM. + Writes before-s2 while Running, then stops and snapshots. + The snapshot is kept for step 17. + Verifies: state Ready; VM stays Stopped; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + self.__class__.token_s2 = new_token("before-s2") + self._write_marker(self.__class__.token_s2, disks=("ROOT", "DATA")) + self._stop_vm(vm.id) + before = self._ontap_snap_names() + snap = self._take_vm_snapshot(vm.id) + self.assertEqual(self._vm_state(vm.id), "Stopped") + new = self._assert_new_ontap_snaps(before, [snap], "vm", "VM snapshot (Stopped)") + self._keep_vm_snapshot(snap, new, "vm_snap_stopped") + + @attr(tags=[TAG], required_hardware=True) + def test_17_vm_snapshot_s1_running_s2_stopped_revert(self): + """ + VMS-09 (iSCSI ROOT + iSCSI DATA on the same pool): revert to the + running snapshot from step 15, then to the stopped snapshot from + step 16. + Verifies: + - After revert to S1: ROOT and DATA hold before-s1 + - After revert to S2: ROOT and DATA hold before-s2 + - Both FlexVol snapshots survive the reverts + - Both VM snapshots are then deleted while the VM is still running + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + if self.__class__.vm_ssh is None: + self.skipTest("Guest SSH not set up — test_09 must pass first") + s1, s2, token_a, token_b, ontap_names = self._adopt_kept_vm_snapshots() + vm = self.__class__.vm + + self._stop_vm(vm.id) + VmSnapshot.revertToSnapshot(self.apiClient, s1.id) + self._start_vm(vm.id) + self._assert_marker(token_a, disk="ROOT", msg="after revert to S1") + self._assert_marker(token_a, disk="DATA", msg="after revert to S1") + self.assertEqual(len(self._vm_snapshots_of(vm.id)), 2, "S2 should still be listed") + + self._stop_vm(vm.id) + VmSnapshot.revertToSnapshot(self.apiClient, s2.id) + self._start_vm(vm.id) + self._assert_marker(token_b, disk="ROOT", msg="after revert to S2") + self._assert_marker(token_b, disk="DATA", msg="after revert to S2") + self._assert_ontap_snaps_present(ontap_names, "after both reverts") + + self._delete_vm_snapshot_while_running( + vm, s1, self._ontap_names_for_snap(ontap_names, s1, "vm"), token_b) + self._delete_vm_snapshot_while_running( + vm, s2, self._ontap_names_for_snap(ontap_names, s2, "vm"), token_b) + + @attr(tags=[TAG], required_hardware=True) + def test_18_vm_snapshot_duplicate_requests(self): + """ + VMS-53: two VM snapshot requests for the same VM at the same time. + A rejected request is retried on its own; one that already succeeded + is not repeated. + Verifies: both succeed and are Ready; two new FlexVol snapshots. + """ + vm = self._require_snapshot_vm() + before = self._ontap_snap_names() + pending = [ + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_dup_a_%d" % random.randint(0, 99999)), + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_dup_b_%d" % random.randint(0, 99999)), + ] + ok = [] + last_errors = [] + attempts = 3 + for attempt in range(1, attempts + 1): + results = _run_concurrent(*pending) + last_errors = [] + still_pending = [] + retryable = True + for fn, (result, exc) in zip(pending, results): + if exc is None and result is not None: + ok.append(result) + self._vm_snapshot_ids.append(result.id) + continue + last_errors.append(exc) + if exc is None or not _is_rejected(exc): + retryable = False + still_pending.append(fn) + pending = still_pending + if not pending: + break + if not retryable or attempt == attempts: + break + logger.warning("duplicate VM snapshot rejected, retry %d/%d: %s", + attempt, attempts, last_errors) + time.sleep(15) + self.assertFalse(pending, + "Duplicate VM snapshot requests failed: %s" % last_errors) + self.assertEqual(len(ok), 2, "Expected both VM snapshots to succeed; " + "got %d" % len(ok)) + for snap in ok: + self.assertEqual(snap.state, "Ready") + self.assertEqual(snap.virtualmachineid, vm.id) + self._assert_new_ontap_snaps(before, ok, "vm", "duplicate VM snapshot requests") + + # ------------------------------------------------------------------ + # Steps 17-21 — Second VM on the same pool: isolation and concurrency + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_19_deploy_second_vm(self): + """ + Deploy a second VM whose ROOT LUN is on the same ONTAP pool. + Verifies: VM Running; ROOT on the test pool; SSH works. + """ + self._require_snapshot_vm() + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.networkids = self.__class__.network_id + vm2 = self.apiClient.deployVirtualMachine(cmd) + self.__class__.vm2 = vm2 + self._poll_vm_state(vm2.id, "Running", timeout=900) + root = self._root_volume(vm2.id) + self.assertEqual(getattr(root, "storageid", None), self.__class__.pool.id, + "Second VM ROOT is not on the ONTAP test pool") + host, port, _ = open_ssh_port(self.apiClient, vm2.id, self.__class__.network_id) + self.__class__.vm2_ssh = (host, port) + token = new_token("second-vm-ssh-access-test") + self._write_marker(token, which="vm2") + self._assert_marker(token, which="vm2", msg="second VM access") + + @attr(tags=[TAG], required_hardware=True) + def test_20_volume_revert_leaves_other_vm_untouched(self): + """ + VS-28: two VMs on one FlexVol; revert VM-A's ROOT to its snapshot. + Verifies: VM-A is back at its snapshot; VM-B keeps data written after it. + """ + vm = self._require_snapshot_vm() + self._require_second_vm() + a1 = new_token("vm-a-before-snapshot") + a2 = new_token("vm-a-after-snapshot") + b1 = new_token("vm-b-before-snapshot") + b2 = new_token("vm-b-after-snapshot") + self._write_marker(a1) + self._write_marker(b1, which="vm2") + snap = self._take_volume_snapshot(self._root_volume(vm.id).id) + self._write_marker(a2) + self._write_marker(b2, which="vm2") + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, snap.id) + self._start_vm(vm.id) + self._assert_marker(a1, msg="VM-A after revert") + self._assert_marker(b2, which="vm2", msg="VM-B after VM-A's revert") + + @attr(tags=[TAG], required_hardware=True) + def test_21_concurrent_vm_snapshots_two_vms(self): + """ + VMS-55: VM snapshots on two VMs of the same iSCSI pool at the same time. + Verifies: both Ready and owned by their VMs; two distinct new FlexVol + snapshots. + """ + vm = self._require_snapshot_vm() + vm2 = self._require_second_vm() + before = self._ontap_snap_names() + results = _run_concurrent( + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_cc_a_%d" % random.randint(0, 99999)), + lambda: VmSnapshot.create(self.apiClient, vm2.id, snapshotmemory="false", + name="ontap_cc_b_%d" % random.randint(0, 99999)), + ) + self._vm_snapshot_ids.extend(s.id for s, _ in results if s is not None) + errors = [e for _, e in results if e is not None] + self.assertFalse(errors, "Concurrent VM snapshots failed: %s" % errors) + snap_a, snap_b = results[0][0], results[1][0] + self.assertEqual(snap_a.state, "Ready") + self.assertEqual(snap_b.state, "Ready") + self.assertEqual(snap_a.virtualmachineid, vm.id) + self.assertEqual(snap_b.virtualmachineid, vm2.id) + self._assert_new_ontap_snaps(before, [snap_a, snap_b], "vm", "concurrent VM snapshots") + + @attr(tags=[TAG], required_hardware=True) + def test_22_concurrent_volume_snapshot_and_revert(self): + """ + Volume snapshot of VM-A's ROOT while VM-B's ROOT reverts, at the same time. + A rejected snapshot or revert is retried on its own. + Verifies: VM-A snapshot BackedUp with its own FlexVol snapshot; VM-B is + back at its snapshot; VM-A data unchanged. + """ + vm = self._require_snapshot_vm() + vm2 = self._require_second_vm() + a1 = new_token("vm-a-current-data") + b1 = new_token("vm-b-before-snapshot") + b2 = new_token("vm-b-after-snapshot") + self._write_marker(a1) + self._write_marker(b1, which="vm2") + snap_b = self._take_volume_snapshot(self._root_volume(vm2.id).id) + self._write_marker(b2, which="vm2") + self._stop_vm(vm2.id) + before = self._ontap_snap_names() + root_a = self._root_volume(vm.id).id + + snap_a = _snapshot_and_revert_with_retry( + lambda: Snapshot.create(self.apiClient, root_a, + account=self.account.name, domainid=self.domain.id), + lambda: Volume.revertToSnapshot(self.apiClient, snap_b.id), + ) + self._volume_snapshot_ids.append(snap_a.id) + self.assertEqual(snap_a.state, "BackedUp") + self._assert_new_ontap_snaps(before, [snap_a], "volume", "VM-A snapshot during VM-B revert") + + self._start_vm(vm2.id) + self._assert_marker(b1, which="vm2", msg="VM-B after revert") + self._assert_marker(a1, msg="VM-A during VM-B revert") + + @attr(tags=[TAG], required_hardware=True) + def test_23_concurrent_vm_snapshot_and_revert(self): + """ + VM snapshot of VM-A while VM-B reverts to its VM snapshot, at the same time. + A rejected snapshot or revert is retried on its own. + Verifies: VM-A snapshot Ready with its own FlexVol snapshot; VM-B is + back at its snapshot; VM-A data unchanged. + """ + vm = self._require_snapshot_vm() + vm2 = self._require_second_vm() + a1 = new_token("vm-a-current-data") + b1 = new_token("vm-b-before-snapshot") + b2 = new_token("vm-b-after-snapshot") + self._write_marker(a1) + self._write_marker(b1, which="vm2") + snap_b = self._take_vm_snapshot(vm2.id) + self._write_marker(b2, which="vm2") + self._stop_vm(vm2.id) + before = self._ontap_snap_names() + + snap_a = _snapshot_and_revert_with_retry( + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_ccr_%d" % random.randint(0, 99999)), + lambda: VmSnapshot.revertToSnapshot(self.apiClient, snap_b.id), + ) + self._vm_snapshot_ids.append(snap_a.id) + self.assertEqual(snap_a.state, "Ready") + self._assert_new_ontap_snaps(before, [snap_a], "vm", "VM-A snapshot during VM-B revert") + + self._start_vm(vm2.id) + self._assert_marker(b1, which="vm2", msg="VM-B after revert") + self._assert_marker(a1, msg="VM-A during VM-B revert") + + # ------------------------------------------------------------------ + # Step 24 — Detach volume from running VM + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_24_detach_volume_from_vm(self): + """ + Detach the iSCSI data volume from the running VM. + Verifies: + - detachVolume completes successfully + - CloudStack: volume virtualmachineid cleared + - ONTAP: LUN-map is removed (LUN stays in FlexVol) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = self.__class__.volume.id + + max_timeout = 180 + interval = 10 + deadline = time.time() + max_timeout + last_exc = None + while True: + try: + self.apiClient.detachVolume(cmd) + last_exc = None + break + except Exception as exc: + last_exc = exc + remaining = deadline - time.time() + if remaining <= 0: + break + time.sleep(min(interval, remaining)) + interval = min(interval * 2, max_timeout) + if last_exc is not None: + raise last_exc + + # Poll until virtualmachineid is cleared + result = self._poll_volume_field( + self.__class__.volume.id, "virtualmachineid", None, timeout=120) + self.assertIsNotNone( + result, + "Volume virtualmachineid was not cleared after detachVolume" + ) + + # ONTAP: LUN-map must be removed after detach + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 data-volume LUN-maps after volume detach, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ONTAP: LUN still exists in FlexVol + luns = self._data_luns() + self.assertTrue( + len(luns) > 0, + "Data volume LUN should still exist in ONTAP FlexVol after detach" + ) + + # ------------------------------------------------------------------ + + # ------------------------------------------------------------------ + # Step 25 — Grow detached data volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_25_grow_detached_volume(self): + """Grow the materialized LUN after it has been detached.""" + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + detached_target = self.__class__.target_volume_size + (1024 ** 3) + self._resize_volume( + self.__class__.volume.id, detached_target // (1024 ** 3)) + + self._poll_volume_size(self.__class__.volume.id, detached_target) + self.assertEqual( + self._get_db_volume_size(self.__class__.volume.id), + detached_target, + "DB size does not match detached iSCSI grow target") + self.assertEqual( + self._data_lun_size(), detached_target, + "ONTAP LUN size does not match detached grow target") + self.__class__.target_volume_size = detached_target + + # ------------------------------------------------------------------ + # Step 26 — Destroy the second VM while it still has a VM snapshot + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_26_destroy_vm_with_vm_snapshot(self): + """ + Destroy (expunge) the second VM while it still has a VM snapshot. + Verifies: destroy succeeds; the VM snapshot is no longer listed; its + FlexVol snapshot is removed from ONTAP. + """ + self._require_snapshot_vm() + vm2 = self._require_second_vm() + before = self._ontap_snap_names() + snap = self._take_vm_snapshot(vm2.id) + new = self._assert_new_ontap_snaps(before, [snap], "vm", "second VM snapshot") + + cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + cmd.id = vm2.id + cmd.expunge = True + self.apiClient.destroyVirtualMachine(cmd) + self.__class__.vm2 = None + self.__class__.vm2_ssh = None + self._vm_snapshot_ids.remove(snap.id) + + deadline = time.time() + 300 + while time.time() < deadline and self._vm_snapshots_of(vm2.id): + time.sleep(10) + self.assertFalse(self._vm_snapshots_of(vm2.id), + "VM snapshot still listed after destroying its VM") + self._wait_ontap_snaps_gone(new) + + # ------------------------------------------------------------------ + # Step 27 — Destroy VM and clean up pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_27_destroy_vm_and_cleanup(self): + """ + Destroy the VM (with expunge), delete the data volume, force-delete + the ONTAP pool, and delete the guest network created for this suite. + This test leaves no entities behind in either CloudStack or ONTAP. + Verifies: + - VM is destroyed and expunged from CloudStack + - Template cache survives VM delete: template_spool_ref still Ready + and the cs_tmpl_* cache LUN is still in the FlexVol + - deleteVolume removes the LUN from the ONTAP FlexVol + - deleteStoragePool(forced=True) removes the pool from CS + - ONTAP: FlexVol deleted + - ONTAP: all per-host igroups deleted + - CloudStack: guest network deleted (Advanced zones only) + """ + self._destroy_second_vm() + pool = self.__class__.pool + vol = self.__class__.volume + + if self.__class__.vm is not None: + # Ensure VM is stopped before destroying + vms = self.apiClient.listVirtualMachines( + _list_vms_cmd(self.__class__.vm.id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = self.__class__.vm.id + stop_cmd.forced = True + self.apiClient.stopVirtualMachine(stop_cmd) + self._poll_vm_state(self.__class__.vm.id, "Stopped", + timeout=120) + + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = self.__class__.vm.id + dest_cmd.expunge = True + self.apiClient.destroyVirtualMachine(dest_cmd) + self.__class__.vm = None + + if pool is not None: + spool = self._assert_single_ready_spool_ref( + self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + if vol is not None and pool is not None: + pool_name = pool.name + + # Delete the data volume + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + + # ONTAP: LUN must be removed after volume deletion + luns = self._data_luns(vol) + self.assertEqual( + len(luns), 0, + "Expected the data volume's LUN to be gone from FlexVol '%s' " + "after volume delete, found %d" % (pool_name, len(luns)) + ) + + # Enter maintenance and force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after force deletion" + % pool_name + ) + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool force deletion" + % pool_name + ) + + # ONTAP: per-host igroups must be deleted by the pool force-delete + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + short = host.name.split(".")[0] + igroup_name = "cs_%s_%s" % ( + self.svm_name, + re.sub(r"[^a-zA-Z0-9_-]", "_", short), + ) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool force deletion" + % igroup_name + ) + + # Delete the guest network created by setUpClass for Advanced zones. + # Doing this inside the test (rather than only in tearDownClass) makes + # the full sequence self-contained when all tests pass. + if self.__class__._created_network_id is not None: + net_id = self.__class__._created_network_id + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = net_id + # The management server may briefly drop the connection after the + # heavy teardown above; retry deleteNetwork up to 3× with 15s gaps. + last_net_exc = None + for attempt in range(3): + try: + self.apiClient.deleteNetwork(dn_cmd) + last_net_exc = None + break + except Exception as exc: + last_net_exc = exc + if attempt < 2: + time.sleep(15) + if last_net_exc is not None: + raise last_net_exc + self.__class__._created_network_id = None + self.__class__.network_id = None + + # CloudStack: guest network must be gone + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.id = net_id + net_cmd.listall = True + remaining_nets = self.apiClient.listNetworks(net_cmd) or [] + self.assertFalse( + remaining_nets, + "Guest network %s still listed in CloudStack after deletion" + % net_id + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/__init__.py b/test/integration/plugins/ontap/iscsi/pool/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py new file mode 100644 index 000000000000..85663d6370d0 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -0,0 +1,1751 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage +pool lifecycle. + +Tests are numbered test_01 ... test_21 and must run in that order. test_01 +and test_02 are isolated create rejections. test_04 reuses the FlexVol created +by test_03. test_20 and test_21 are isolated delete cases. + +Workflow: + 01 Reject create when no online assigned aggregate has enough free space + 02 Reject create when capacitybytes has invalid values + 03 Create primary storage pool + 04 Reject create when a FlexVol of that name already exists on ONTAP + 05 Increase storage pool capacity to the maximum supported size + 06 Reject a grow beyond the maximum supported size + 07 Safely shrink storage pool capacity + 08 Disable storage pool + 09 Grow and shrink the pool while it is disabled + 10 Enable storage pool + 11 Enter maintenance mode + 12 Grow and shrink the pool while it is in maintenance mode + 13 Cancel maintenance mode + 14 Enter maintenance mode and delete the storage pool + 15 Create a new pool and allocate a CloudStack data volume (LUN created) + 16 Deploy a VM and attach the ONTAP volume (LUN) to it + 17 Reject shrink below the ONTAP used capacity of the attached LUN + 18 Grow and shrink the pool while the VM holds the volume (LUN) + 19 Detach/destroy the VM, delete the volume (LUN removed), enter + maintenance, force-delete pool + 20 Delete an empty pool whose FlexVol was deleted directly on ONTAP + 21 Delete an empty pool whose igroups were deleted on ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - At least one ready user KVM template in the zone (test_16 skips without one) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py -v + +Note: Tests 03-19 share class-level state (sequential). test_04 reuses the +FlexVol created by test_03. Running a single test with -m "test_NN" will +invoke setUpClass but the guard assertion will fail immediately if earlier +steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import ( + OntapRestClient, OntapTestBase, _wait_for_vm_state, get_datacenter_config, + log_progress, run_invalid_capacitybytes_rejections, +) + +logger = logging.getLogger("TestOntapISCSIPoolLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + # Maximum FlexVol size on AFF/FAS, raised from 100 TB in ONTAP 9.12.1P2. + # ONTAP writes this as "300 TB" but means binary, hence 1024**4. The + # value has to be hardcoded: no REST field reports the platform ceiling. + # Against an ONTAP older than 9.12.1P2 the real limit is 100 TB and the + # grow step will fail. CloudStack enforces no maximum of its own (the + # plugin validates only a 20 MB floor), so it is ONTAP that refuses + # anything above this. + ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 + ONTAP_GROW_TARGET_SIZE = ONTAP_MAX_VOLUME_SIZE + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wf@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WF", + "username": "ontap_iscsi_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSI_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPoolLifecycle(OntapTestBase): + + # ---- iSCSI-specific state (set/cleared by individual tests) -------- + _vol_name_prefix = "OntapISCSIVol" + resize_original_size = None + + # VM state (vm/template_id/network_id) lives in OntapTestBase. + _vm_network_name_prefix = "ontap-iscsi-lifecycle-net" + + @classmethod + def setUpClass(cls): + super(TestOntapISCSIPoolLifecycle, cls).setUpClass() + testclient = super( + TestOntapISCSIPoolLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + cls._discover_vm_deploy_resources() + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self, pool_name=None, capacitybytes=None): + """Create a pool; name and capacity default to the suite values.""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + if pool_name is None: + pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + if capacitybytes is None: + capacitybytes = ps["capacitybytes"] + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = capacitybytes + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + def _throwaway_pool_name(self, suffix): + return "OntapISCSI%s_%d" % (suffix, random.randint(0, 99999)) + + def _cleanup_throwaway_pool(self, pool, flexvol_name=None): + """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + + Never raises, so a failed assertion in the test body is the error + that surfaces. + """ + if pool is not None: + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + if listed: + try: + if listed[0].state != "Maintenance": + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not put pool '%s' into Maintenance: %s", + pool.name, exc, + ) + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "cleanup: could not delete pool '%s': %s", + pool.name, exc, + ) + if flexvol_name is None: + flexvol_name = pool.name + if flexvol_name: + try: + if self.ontap.get_volume(flexvol_name) is not None: + self.ontap.offline_and_delete_volume(flexvol_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete ONTAP FlexVol '%s': %s", + flexvol_name, exc, + ) + if pool is None: + return + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + if not remaining: + self.__class__.pool2 = None + + def _assert_only_original_pool(self, pool): + """Assert the rejected create did not add a second pool of this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool.name) or [] + except CloudstackAPIException: + listed = [] + same_name = [ + item for item in listed if getattr(item, "name", None) == pool.name + ] + self.assertEqual( + [item.id for item in same_name], + [pool.id], + "CloudStack should still list only pool '%s' after the rejected " + "create, found: %s" % (pool.name, same_name), + ) + current = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + current, + "Original pool '%s' disappeared after the rejected create" + % pool.name, + ) + self.assertEqual( + current[0].state, "Up", + "Original pool '%s' should still be Up, got '%s'" + % (pool.name, current[0].state), + ) + + def _warn_if_duplicate_pool(self, duplicate, pool): + """Leave a stray duplicate in place. + + Deleting it would remove the FlexVol that the rest of the suite + still uses. + """ + if duplicate is None or getattr(duplicate, "id", None) == pool.id: + return + logger.warning( + "Duplicate pool '%s' (id=%s) was created against the shared " + "FlexVol; leaving it in place so cleanup does not delete the " + "pool the rest of the suite uses", + pool.name, duplicate.id, + ) + + def _assert_no_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except CloudstackAPIException: + listed = None + self.assertFalse( + listed, + "CloudStack should hold no pool named '%s' after a rejected " + "create, found: %s" % (pool_name, listed), + ) + + def _assert_pool_gone_from_cs(self, pool_id, pool_name): + try: + remaining = list_storage_pools(self.apiClient, id=pool_id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after deletion" % pool_name, + ) + + def _sweep_tracked_pool2(self): + """Clean a prior leftover before reusing the shared pool2 slot.""" + if self.__class__.pool2 is None: + return + self._cleanup_throwaway_pool(self.__class__.pool2) + if self.__class__.pool2 is not None: + self.skipTest("A previously tracked pool could not be cleaned up") + + def _create_throwaway_pool(self, suffix): + """Create an isolated pool, park it in pool2, and return it.""" + self._sweep_tracked_pool2() + pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) + self.__class__.pool2 = pool + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (pool.name, pool.state), + ) + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ) + return pool + + # ------------------------------------------------------------------ + # Step 01 - reject create when no aggregate space + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_01_reject_create_when_no_aggregate_space(self): + """ + Ask for 1 GiB more than the largest online aggregate assigned to the + SVM can provide, so no aggregate qualifies and the plugin refuses + before creating anything. + + Verifies: + - createStoragePool raises CloudstackAPIException + - the error names the aggregate shortage rather than some other + failure ('No suitable aggregates') + - no pool is left in CloudStack and no FlexVol on ONTAP + """ + self._sweep_tracked_pool2() + max_free = self.ontap.max_online_aggregate_available_bytes( + self.svm_name + ) + if not max_free: + self.skipTest( + "No online aggregate with reported free space is assigned to " + "SVM '%s'; cannot build an unsatisfiable request" + % self.svm_name + ) + requested = int(max_free) + 1024 ** 3 + if requested > TestData.ONTAP_MAX_VOLUME_SIZE: + self.skipTest( + "Largest aggregate free space (%d B) + 1 GiB exceeds the " + "ONTAP FlexVol maximum (%d B); the request would be refused " + "for the size limit rather than the aggregate shortage" + % (max_free, TestData.ONTAP_MAX_VOLUME_SIZE) + ) + + pool_name = self._throwaway_pool_name("NoSpace") + log_progress( + logger, "info", + "Requesting pool '%s' of %d B; largest online aggregate on SVM " + "'%s' has %d B free (expect reject)", + pool_name, requested, self.svm_name, max_free, + ) + try: + with self.assertRaises(CloudstackAPIException) as caught: + self.__class__.pool2 = self._create_pool( + pool_name=pool_name, capacitybytes=requested + ) + error_text = str(caught.exception) + log_progress( + logger, "info", + "Rejected no-space create for '%s': %s", pool_name, error_text, + ) + self.assertIn( + "No suitable aggregates", error_text, + "Expected the rejection to report 'No suitable aggregates', " + "got: %s" % error_text, + ) + self._assert_no_pool_named(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' was created despite the rejected pool " + "create" % pool_name, + ) + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool_name + ) + + # ------------------------------------------------------------------ + # Step 02 - reject create with invalid capacitybytes values + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_reject_create_with_invalid_capacitybytes_values(self): + """ + Negative case: reject pool creation when capacitybytes has invalid + values. + + Verifies, for each invalid value: + - createStoragePool raises CloudstackAPIException + - zero and negative report that capacity must be at least the + ONTAP minimum + - special characters and a decimal are rejected as an invalid + capacitybytes value + - no pool is left in CloudStack and no FlexVol on ONTAP + """ + def cleanup(pool, pool_name): + self._cleanup_throwaway_pool(pool, flexvol_name=pool_name) + + run_invalid_capacitybytes_rejections( + self, + lambda name, capacity: self._create_pool( + pool_name=name, capacitybytes=capacity + ), + cleanup, + logger, + ) + + # ------------------------------------------------------------------ + # Step 03 - create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_create_primary_storage_pool(self): + """ + Create an iSCSI primary storage pool and verify: + - CloudStack state is Up, type is OntapiSCSI + - ONTAP: FlexVol exists and is online + - ONTAP: one igroup per cluster host exists with the correct IQN initiator + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "OntapiSCSI", + "Pool type should be 'OntapiSCSI', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue # host not iSCSI-enabled; skip igroup check for it + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "pool-created") + + + # ------------------------------------------------------------------ + # Step 04 - reject create when flexvol name exists + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_reject_create_when_flexvol_name_exists(self): + """ + The pool from test_03 already has a FlexVol of that name on ONTAP. + A second createStoragePool with the same name must be rejected, and + the existing pool and FlexVol must be left untouched. + + Verifies: + - createStoragePool raises CloudstackAPIException + - CloudStack still lists only the pool from test_03 + - the existing ONTAP FlexVol is still online + """ + pool = self.__class__.pool + self.assertIsNotNone(pool, "Pool absent - test_03 must pass first") + pool_name = pool.name + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' from test_03 is missing" % pool_name, + ) + log_progress( + logger, "info", + "Pool '%s' already owns a FlexVol; requesting another pool of " + "the same name (expect reject)", pool_name, + ) + duplicate = None + try: + with self.assertRaises(CloudstackAPIException) as caught: + duplicate = self._create_pool(pool_name=pool_name) + log_progress( + logger, "info", + "Rejected duplicate-name create for '%s': %s", + pool_name, caught.exception, + ) + self._assert_only_original_pool(pool) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "Existing ONTAP FlexVol '%s' was removed by the failed " + "pool create" % pool_name, + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "Existing ONTAP FlexVol '%s' should still be online, got '%s'" + % (pool_name, ontap_vol.get("state")), + ) + finally: + self._warn_if_duplicate_pool(duplicate, pool) + + # ------------------------------------------------------------------ + # Step 05 - grow storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_05_grow_storage_pool(self): + """Grow the original pool to 300 TiB and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = TestData.ONTAP_GROW_TARGET_SIZE + self.assertGreater( + requested_size, original_size, + "Grow target %d B must exceed the current pool size %d B" + % (requested_size, original_size) + ) + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after increase" % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 06 - reject grow above max size + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_06_reject_grow_above_max_size(self): + """ + Request a capacity above the 300 TiB ONTAP FlexVol maximum and verify + it is refused, leaving the pool at the size test_05 grew it to. + ONTAP reports this as an autosize-maximum violation rather than an + oversized-volume error. + Verifies: + - updateStoragePool raises CloudstackAPIException + - the error text is about the size limit, not about some other + failure such as the aggregate running out of space + - CloudStack capacity and ONTAP FlexVol size are both unchanged + - Pool stays 'Up' + - ONTAP: per-host igroups are untouched + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_05 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before over-max resize") + self.assertEqual( + int(getattr(listed[0], "capacitybytes", 0) or 0), + TestData.ONTAP_MAX_VOLUME_SIZE, + "Pool should sit at the 300 TiB maximum - test_05 must pass first" + ) + + # Overshoot by only 1 GiB: at 0.0003% of the limit this probes the + # 300 TiB boundary itself rather than asking for something absurd. + above_max = self._align_flexvol_bytes( + TestData.ONTAP_MAX_VOLUME_SIZE + 1024 ** 3 + ) + log_progress( + logger, "info", + "Requesting %d B for pool '%s' (ONTAP maximum is %d B, " + "expect reject)", + above_max, pool.name, TestData.ONTAP_MAX_VOLUME_SIZE, + ) + + # ONTAP enforces the platform cap through autosize rather than + # rejecting the size outright, reporting "Volume maximum autosize + # must be greater than or equal to the current volume size 300.0TB". + # Matching on that keeps the test from passing when the resize fails + # for an unrelated reason, such as the aggregate running out of space. + self._assert_capacity_update_rejected( + pool.id, above_max, "grow-above-max-flexvol", + pool.id, pool.name, + expected_error=( + "maximum autosize", + "maximum size", + "exceeds", + ), + ) + + after = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(after, "Pool disappeared after rejected over-max grow") + self.assertEqual( + after[0].state, "Up", + "Pool should stay 'Up' after a rejected resize, got '%s'" + % after[0].state + ) + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after a rejected " + "over-max grow" % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 07 - shrink storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_07_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_05 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent - test_05 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after safe shrink" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 08 - disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_08_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online (disable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + # ONTAP: disable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 09 - grow and shrink storage pool while disabled + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_09_grow_and_shrink_storage_pool_while_disabled(self): + """ + Grow and shrink the disabled pool and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - The pool remains Disabled throughout + - The FlexVol remains online and per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before disabled resize") + self.assertEqual( + listed[0].state, "Disabled", + "Pool must be Disabled - test_08 must pass first, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before disabled resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s disabled iSCSI pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Disabled", + "Pool should remain Disabled after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared during disabled resize" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 10 - enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_10_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online (enable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + # ONTAP: enable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 11 - enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_11_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + result = self._enter_maintenance(self.__class__.pool.id) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: maintenance must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 12 - grow and shrink storage pool in maintenance + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_12_grow_and_shrink_storage_pool_in_maintenance(self): + """ + Grow and shrink the pool while it is in maintenance mode and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - The pool remains in Maintenance throughout + - The FlexVol remains online and per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before maintenance resize") + self.assertEqual( + listed[0].state, "Maintenance", + "Pool must be in Maintenance - test_11 must pass first, got '%s'" + % listed[0].state, + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before maintenance resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s iSCSI pool '%s' while in Maintenance: target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Maintenance", + "Pool should remain in Maintenance after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared during maintenance resize" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 13 - cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_13_cancel_maintenance_mode(self): + """ + Cancel maintenance and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + result = self._exit_maintenance(self.__class__.pool.id) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 14 - enter maintenance and delete pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_14_enter_maintenance_and_delete_pool(self): + """ + Enter maintenance mode then delete the pool. + Verifies the pool is removed from CloudStack and the backing ONTAP + FlexVol is deleted. + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + pool = self.__class__.pool + pool_name = pool.name + + self._enter_maintenance(pool.id) + + self._delete_pool(pool.id) + self.__class__.pool = None + self.__class__.resize_original_size = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + + # ------------------------------------------------------------------ + # Step 15 - create volume on pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_15_create_volume_on_pool(self): + """ + Create a new iSCSI pool and allocate a CloudStack data volume. + For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. + Verifies: + - pool.state is Up, type is OntapiSCSI + - createVolume returns a non-None volume object + - ONTAP: FlexVol is still online + - ONTAP: at least one LUN is present in the FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + log_progress( + logger, "info", + "test_15: created storage pool name='%s' id=%s state=%s type=%s", + pool.name, pool.id, pool.state, pool.type, + ) + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "OntapiSCSI", + "Pool type should be 'OntapiSCSI', got '%s'" % pool.type + ) + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + log_progress( + logger, "info", + "test_15: created CloudStack volume name='%s' id=%s state=%s " + "on pool='%s' (id=%s) account='%s' domain='%s' — " + "switch to this account in the UI to see the volume", + getattr(vol, "name", "?"), getattr(vol, "id", "?"), + getattr(vol, "state", "?"), pool.name, pool.id, + self.account.name, self.domain.name, + ) + + # ONTAP: FlexVol must still be online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: at least one LUN must be present in the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" % pool.name + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "volume-allocated") + + + # ------------------------------------------------------------------ + # Step 16 - create vm and attach volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_16_create_vm_and_attach_volume(self): + """ + Deploy a VM and attach the ONTAP data volume from test_15 to it. + Verifies: + - VM reaches 'Running' + - attachVolume sets the volume's virtualmachineid (on ONTAP managed + storage CloudStack may leave the volume state at 'Ready', so + virtualmachineid is the reliable attach signal) + - VM is still 'Running' after the attach + - ONTAP: FlexVol stays online and the LUN is still present + - ONTAP: per-host igroups survive the attach + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template in the zone - cannot deploy a VM" + ) + self.assertIsNotNone( + self.__class__.service_offering_id, + "No service offering available - check the CloudStack setup" + ) + + pool = self.__class__.pool + vol = self.__class__.volume + + log_progress( + logger, "info", + "Deploying VM for iSCSI pool '%s' (template=%s, offering=%s)", + pool.name, self.__class__.template_id, + self.__class__.service_offering_id, + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=600) + self.assertIsNotNone( + vm_obj, "VM %s never appeared in listVirtualMachines" % vm.id + ) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running' before attach, got '%s'" % vm_obj.state + ) + + log_progress( + logger, "info", + "Attaching volume '%s' to VM '%s'", vol.id, vm.id, + ) + attach_cmd = attachVolumeAPI.attachVolumeCmd() + attach_cmd.id = vol.id + attach_cmd.virtualmachineid = vm.id + self.assertIsNotNone( + self.apiClient.attachVolume(attach_cmd), + "attachVolume returned None" + ) + + vol_vmid = self._poll_volume_attached(vol.id, timeout=180) + self.assertEqual( + vol_vmid, vm.id, + "Volume %s should report virtualmachineid=%s after attach, got %s" + % (vol.id, vm.id, vol_vmid) + ) + + vm_after = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=60) + self.assertEqual( + vm_after.state, "Running", + "VM should still be 'Running' after attach, got '%s'" + % vm_after.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing after volume attach" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach, got '%s'" + % ontap_vol.get("state") + ) + + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "No LUN present in FlexVol '%s' after attach" % pool.name + ) + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after volume attach" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 17 - reject shrink below used capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_17_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the pool below ONTAP used space and verify the + request is rejected while the volume and ONTAP LUNs remain unchanged. + + Runs after test_16, so the LUN is mapped to a running VM. The LUN is + thin-provisioned, though, and in practice leaves used space under the + 20 MiB FlexVol floor, so incompressible data is written through the + ONTAP files API to get above it and removed before this test returns. + The fill is skipped whenever used space is already high enough. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_16 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_16 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs in FlexVol '%s' before rejected shrink" % pool.name, + ) + luns_before_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in luns + ) + + # The attached volume is thin on both protocols, so it usually does + # not put 20 MiB on disk by itself. This writes incompressible data + # only when used space is still under the FlexVol floor, and removes + # it before returning. + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking iSCSI pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + # ONTAP's wording: "Selected volume size is too small to + # hold the current volume data. New volume size must be at + # least 34.3MB ...". Matching the specific phrase stops an + # unrelated resize failure from counting as a pass. + expected_error=( + "too small to hold the current volume data", + "too small to hold", + "cannot reduce", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + luns_after_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in self.ontap.list_luns_in_volume( + self.svm_name, pool.name + ) + ) + self.assertEqual( + luns_after_used, luns_before_used, + "LUN list in FlexVol '%s' changed after rejected " + "used-capacity shrink" % pool.name, + ) + self.assertTrue( + len(luns_after_used) > 0, + "LUN disappeared from FlexVol '%s' after rejected shrink" + % pool.name, + ) + finally: + self._delete_filler_file(pool.name, filler_name) + + + # ------------------------------------------------------------------ + # Step 18 - grow and shrink pool with vm attached + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_18_grow_and_shrink_pool_with_vm_attached(self): + """ + Grow the pool and shrink it straight back while the volume is + attached to the running VM from test_16, so resize is exercised + against a pool that is genuinely in use rather than an idle one. + Verifies: + - grow: CloudStack and ONTAP both reach the requested size + - shrink back: both return to the starting size + - the VM stays 'Running' and keeps the volume across both resizes + - ONTAP: FlexVol stays online + - ONTAP: the LUN backing the volume is still present + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.vm is None: + self.skipTest( + "No VM was deployed (test_16 skipped) - nothing to resize " + "a pool underneath" + ) + + pool = self.__class__.pool + vm = self.__class__.vm + vol = self.__class__.volume + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before the in-use resize") + original_size = int(getattr(listed[0], "capacitybytes", 0) or 0) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Growing in-use pool '%s' from %d B to %d B (VM %s holds the " + "volume)", pool.name, original_size, grow_target, vm.id, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = grow_target + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, grow_target, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing under a VM, got '%s'" + % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, grow_target, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "grow-with-vm") + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "LUN disappeared from FlexVol '%s' during the in-use resize" + % pool.name, + ) + + log_progress( + logger, "info", + "Shrinking in-use pool '%s' back from %d B to %d B", + pool.name, grow_target, original_size, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = original_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, original_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking under a VM, got '%s'" + % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, original_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "shrink-with-vm") + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "LUN disappeared from FlexVol '%s' during the in-use resize" + % pool.name, + ) + + + # ------------------------------------------------------------------ + # Step 19 - delete volume and pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_19_delete_volume_and_pool(self): + """ + Detach the volume and destroy the VM from test_16, then delete the + volume, enter maintenance, and force-delete the pool. + Verifies: + - deleteVolume removes the LUN from ONTAP + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: igroups for all cluster hosts deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_15 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_15 must pass first") + + # A volume still attached to a VM cannot be deleted, so unwind test_16 + # first. Both steps are no-ops when test_16 skipped. + self._detach_volume_if_attached(self.__class__.volume.id) + self.__class__._destroy_vm_if_present() + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume — LUN is removed from ONTAP + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume deletion, found %d" + % (pool_name, len(luns)) + ) + + # ONTAP: FlexVol must still be online (pool not yet deleted) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion" + ) + + # Capacity reporting: capacity stable after volume deletion + self._assert_pool_capacity(pool, "volume-deleted") + + # Enter maintenance then force-delete the pool + self._enter_maintenance(pool.id) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 20 - delete pool with flexvol predeleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_20_delete_pool_with_flexvol_predeleted(self): + """ + Delete the backing FlexVol directly on ONTAP, then delete the empty + pool through CloudStack. Deletion must tolerate the missing volume + rather than leaving an undeletable pool behind. + + Verifies: + - deleteStoragePool succeeds with the FlexVol already gone + - the pool is removed from CloudStack + """ + pool = self._create_throwaway_pool("PreDelVol") + try: + self._enter_maintenance(pool.id) + log_progress( + logger, "info", + "Deleting ONTAP FlexVol '%s' behind CloudStack's back", + pool.name, + ) + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still present after direct deletion" + % pool.name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' reappeared after pool deletion" + % pool.name, + ) + self.__class__.pool2 = None + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool.name + ) + + # ------------------------------------------------------------------ + # Step 21 - delete pool with igroups predeleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_21_delete_pool_with_igroups_predeleted(self): + """ + Delete the per-host igroups directly on ONTAP, then delete the empty + pool through CloudStack. Deletion must tolerate the missing igroups + and still remove the FlexVol. + + The igroup name is keyed off the host UUID and the SVM, not the pool, + so the igroups are shared by every ONTAP pool on that SVM. This test + therefore assumes no other pool is in use on the SVM, which holds + here because test_11 removed the workflow pools. + + The plugin only creates an igroup when a host is first granted access + to a LUN, so a freshly created empty pool has none. The test seeds + them on ONTAP under the exact names the plugin would use, which both + proves the naming scheme still matches and makes the pre-deletion a + real precondition rather than a no-op. + + Verifies: + - deleteStoragePool succeeds with the igroups already gone + - the pool is removed from CloudStack + - ONTAP: the FlexVol is deleted + """ + other_pools = self._other_ontap_pools_on_svm(None) + if other_pools: + self.skipTest( + "Pre-deleting SVM-wide igroups requires exclusive SVM use; " + "found other ONTAP pool(s): %s" + % ", ".join(str(getattr(p, "name", p)) for p in other_pools) + ) + specs = self._iscsi_host_specs() + if not specs: + self.skipTest( + "No cluster host advertises an iSCSI IQN, so no igroup name " + "can be derived" + ) + pool = self._create_throwaway_pool("PreDelIgroup") + seeded = [] + try: + for igroup_name, iqn in specs: + if self.ontap.get_igroup(self.svm_name, igroup_name) is None: + log_progress( + logger, "info", + "Seeding ONTAP igroup '%s' with initiator '%s'", + igroup_name, iqn, + ) + self.ontap.create_igroup(self.svm_name, igroup_name, iqn) + seeded.append(igroup_name) + present = [name for name, _ in specs] + for igroup_name in present: + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' should exist before the pre-deletion" + % igroup_name, + ) + + self._enter_maintenance(pool.id) + log_progress( + logger, "info", + "Deleting ONTAP igroups %s behind CloudStack's back", present, + ) + for igroup_name in present: + self.ontap.delete_igroup(self.svm_name, igroup_name) + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' still present after direct deletion" + % igroup_name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still exists after pool deletion" + % pool.name, + ) + self.__class__.pool2 = None + finally: + for igroup_name in seeded: + try: + self.ontap.delete_igroup(self.svm_name, igroup_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete seeded igroup '%s': %s", + igroup_name, exc, + ) + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool.name + ) + + + # ------------------------------------------------------------------ diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py new file mode 100644 index 000000000000..78182a7dca84 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py @@ -0,0 +1,1354 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI pool lifecycle tests with a CloudStack data volume present throughout. + +Covers the TDS (section 10) scenarios that require a data volume to already +exist on the pool during pool state transitions — the iSCSI variants of those +scenarios: + + TDS Approach-1 SN 11 — Disable iSCSI pool WITH volumes + TDS Approach-1 SN 15 — Enable iSCSI pool WITH volumes + TDS Approach-1 SN 19 — Enter maintenance WITH volumes + TDS Approach-1 SN 23 — Cancel maintenance WITH volumes + TDS Negative SN 5 — Delete iSCSI pool that has volumes; forced=False rejected + TDS Approach-1 SN 7 — Force-delete iSCSI pool (volume deleted first from + Maintenance — allowed on iSCSI unlike NFS3) + +Key iSCSI difference from NFS3: cancelStorageMaintenance works on iSCSI because +the KVM agent can unmount/remount iSCSI LUNs correctly. This allows the full +maintenance-cancel-maintenance lifecycle and proper volume cleanup while pool +is in Maintenance state. + +Tests are numbered test_01 ... test_07 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI pool and allocate a CloudStack data volume (LUN on ONTAP) + 02 Disable pool — volume survives; ONTAP LUN still exists (SN 11) + 03 Re-enable pool — volume intact; ONTAP LUN accessible (SN 15) + 04 Enter maintenance with volume — pool Maintenance; LUN exists (SN 19) + 05 Cancel maintenance with volume — pool Up; LUN accessible (SN 23) + 06 Re-enter maintenance; forced=False delete rejected (Neg SN 5) + 07 Delete volume from Maintenance, then force-delete pool (SN 7) + +Isolated tests (test_08 onwards) run after that workflow and share no state +with it. Each one builds its own pool plus CloudStack volume, breaks a single +ONTAP object behind CloudStack's back, and force-deletes the pool: + + 08 FlexVol pre-deleted on ONTAP, then force-delete pool with CS volume + 09 Host igroups pre-deleted on ONTAP, then force-delete pool with CS volume + 10 Enter maintenance with CS volume after LUN maps are pre-deleted + 11 Log out only the isolated test iSCSI session, then enter maintenance + 12 Delete the test volume while that session is already logged in + 13 Replace the test LUN by-path with a regular file, then delete the volume + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI initiator configured + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py -v +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config + +logger = logging.getLogger("TestOntapISCSIPoolWithVolumes") + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wv@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WV", + "username": "ontap_iscsi_wv_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIWV_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPoolWithVolumes(OntapTestBase): + """ + iSCSI pool lifecycle tests with a CloudStack data volume present throughout. + Tests 01-07 are sequential; tests 08-10 use isolated throwaway resources. + """ + + _vol_name_prefix = "OntapISCSIWV" + + @classmethod + def setUpClass(cls): + super(TestOntapISCSIPoolWithVolumes, cls).setUpClass() + testclient = super( + TestOntapISCSIPoolWithVolumes, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + cls._capture_igroup_baseline() + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIWV_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _assert_lun_exists(self, pool_name, msg_context=""): + """Assert that at least one LUN exists in the pool's ONTAP FlexVol.""" + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertTrue( + len(luns) > 0, + "Expected ≥1 LUN in ONTAP FlexVol '%s'%s, found 0" + % (pool_name, " (%s)" % msg_context if msg_context else "") + ) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create pool and allocate a data volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create an iSCSI primary storage pool and allocate a CloudStack data + volume on it. + Verifies: + - Pool state is Up; pool type is OntapiSCSI + - ONTAP: FlexVol is online + - ONTAP: at least one igroup exists (one per cluster host with IQN) + - ONTAP: after createVolume, a LUN exists in the FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "OntapiSCSI", + "Pool type should be 'OntapiSCSI', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: the plugin creates an igroup only when a host is first + # granted access to a LUN, so the empty pool must not change the + # suite-start igroup baseline. + self._assert_igroup_baseline_unchanged( + "after creating an empty pool" + ) + + # Allocate a CloudStack data volume on this pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: a LUN must exist in the FlexVol after volume creation + self._assert_lun_exists(pool.name, "after volume creation") + self._assert_igroup_baseline_unchanged( + "after creating an unattached volume" + ) + + # Capacity reporting: LUN allocated but FlexVol size unchanged + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 02 — Disable pool with volume present (TDS SN 11) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_02_disable_pool_volume_survives(self): + """ + Disable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 11 (iSCSI): + - Pool transitions to Disabled + - Existing CS volume still listed + - ONTAP: FlexVol remains online; LUN still exists + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual( + result.state, "Disabled", + "Pool should be 'Disabled', got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool disable" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after pool disable" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after pool disable") + + # ------------------------------------------------------------------ + # Step 03 — Re-enable pool with volume present (TDS SN 15) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_03_enable_pool_volume_intact(self): + """ + Re-enable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 15 (iSCSI): + - Pool transitions back to Up + - CS volume still listed + - ONTAP: FlexVol online; LUN still exists + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after re-enable, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool re-enable" + ) + + # ONTAP: FlexVol online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool re-enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after pool re-enable" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after pool re-enable") + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance with volume present (TDS SN 19) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_04_enter_maintenance_volume_present(self): + """ + Enter maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 19 (iSCSI): + - Pool transitions to Maintenance + - CS volume still listed (not destroyed) + - ONTAP: FlexVol remains online (maintenance is a CloudStack state) + - ONTAP: LUN still exists in the FlexVol + + Note: the TDS additionally expects VMs using this pool to stop and their + LUN maps to be removed. This suite uses a standalone data volume (not + attached to any VM), so the VM stop behaviour is not exercised here — it + is covered by the VM lifecycle test suite. + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + result = self._enter_maintenance(self.__class__.pool.id) + self.assertEqual( + result.state, "Maintenance", + "Pool should be 'Maintenance', got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool entered Maintenance" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after entering Maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' in Maintenance" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after entering Maintenance") + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance with volume present (TDS SN 23) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_05_cancel_maintenance_volume_present(self): + """ + Cancel maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 23 (iSCSI): + - cancelStorageMaintenance works on iSCSI (unlike the NFS3 variant) + - Pool transitions back to Up + - CS volume still listed + - ONTAP: FlexVol online; LUN still present in FlexVol + + Note: when VMs are attached to volumes on this pool, ONTAP would + re-create the LUN-maps (igroup bindings) at cancel-maintenance time. + This suite has no VMs attached, so LUN-map re-creation is not verified + here; it is covered by the VM lifecycle test suite. + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + result = self._exit_maintenance(self.__class__.pool.id) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after cancel maintenance" + ) + + # ONTAP: FlexVol online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after cancel maintenance") + + # ------------------------------------------------------------------ + # Step 06 — forced=False delete rejected (negative) (TDS Neg SN 5) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_06_forced_false_delete_rejected(self): + """ + Enter maintenance then attempt deleteStoragePool(forced=False) while + a CloudStack volume exists on the pool. The operation must be rejected. + Covers TDS Negative Scenario SN 5 (iSCSI): + - CloudstackAPIException is raised + - Pool remains in Maintenance state + - CS volume still exists + - ONTAP: FlexVol and LUN unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + # Re-enter Maintenance (pool is Up from test_05) + self._enter_maintenance(self.__class__.pool.id) + + # Attempt forced=False delete — must raise + with self.assertRaises(Exception, + msg="deleteStoragePool(forced=False) with a live " + "volume should raise an exception"): + self._delete_pool(self.__class__.pool.id, forced=False) + + # Pool must still be listed (in Maintenance) + try: + remaining = list_storage_pools(self.apiClient, id=self.__class__.pool.id) + except Exception: + remaining = None + self.assertTrue( + remaining, + "Pool was deleted even though forced=False delete should have failed" + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume was deleted after rejected pool deletion" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol should still exist after rejected pool deletion" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after rejected deletion" + ) + + # ------------------------------------------------------------------ + # Step 07 — Delete volume from Maintenance, then force-delete pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_07_delete_volume_and_force_delete_pool(self): + """ + Delete the CloudStack volume (while pool is in Maintenance) then + force-delete the pool. + Covers TDS Approach-1 SN 7 (iSCSI): + - On iSCSI, deleteVolume succeeds even when pool is in Maintenance + (unlike NFS3 where the KVM agent raises NPE) + - After volume deletion, the LUN is removed from the ONTAP FlexVol + - force-delete pool removes pool, FlexVol, and all igroups + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume while pool is in Maintenance + # (this works on iSCSI — no KVM NPE unlike NFS3) + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol after volume deletion + luns_after = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns_after), 0, + "Expected 0 LUNs in ONTAP FlexVol '%s' after volume deletion, " + "found %d: %s" % (pool_name, len(luns_after), luns_after) + ) + + # ONTAP: FlexVol must still be online (pool deletion removes the FlexVol) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after CS volume deletion" + % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after CS volume deletion" + ) + + # Capacity reporting: capacity fields stable after LUN removal + self._assert_pool_capacity(pool, "volume-deleted") + + # Force-delete the pool (no live volumes remain; pool is in Maintenance) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after force deletion" % pool_name + ) + + # ONTAP: FlexVol must be deleted + ontap_vol_after = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol_after, + "ONTAP FlexVol '%s' still exists after pool force deletion" % pool_name + ) + + self._assert_no_lun_maps_for_volume( + pool_name, "after pool force deletion" + ) + self._assert_igroup_baseline_unchanged("after pool force deletion") + + # ================================================================== + # Isolated tests — appended after the sequential workflow above. + # + # Each one creates its own pool and CloudStack volume in the pool2 / + # volume2 slots (which OntapTestBase.tearDownClass also sweeps), runs a + # single scenario, and cleans up in a finally block. They never reuse a + # pool destroyed by another test. + # ================================================================== + + def _host_igroup_names(self): + """igroup names the plugin creates, one per cluster host with an IQN. + + Built from the host UUID so the names match the plugin. + """ + return [name for name, _ in self._host_igroup_specs()] + + def _host_igroup_specs(self): + """(igroup name, initiator IQN) per cluster host that reports an IQN.""" + return self._iscsi_host_specs() + + def _create_isolated_pool_with_volume(self, label): + """Build a fresh pool plus CS volume for one isolated scenario. + + Returns ``(pool, volume)``. Any pool a previous isolated test could + not clean up is swept first so its ONTAP objects are never orphaned by + the overwrite of the pool2 slot. + """ + if self.__class__.pool2 is not None: + self._cleanup_isolated_pool( + self.__class__.pool2, "leftover-from-previous-isolated-test" + ) + + pool = self._create_pool() + self.__class__.pool2 = pool + logger.info("[%s] created isolated pool '%s'", label, pool.name) + + self.assertEqual( + pool.state, "Up", + "[%s] new pool state should be 'Up', got '%s'" % (label, pool.state) + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol not found for new pool '%s'" % (label, pool.name) + ) + + vol = self._create_volume(pool.id) + self.__class__.volume2 = vol + self.assertIsNotNone(vol, "[%s] createVolume returned None" % label) + self._assert_lun_exists(pool.name, "%s: after volume creation" % label) + return pool, vol + + def _assert_pool_absent(self, pool, label, delete_error=None): + """Assert CloudStack no longer lists the pool.""" + try: + remaining = list_storage_pools(self.apiClient, name=pool.name) + except Exception: + remaining = None + self.assertFalse( + remaining, + "[%s] pool '%s' is still listed after deleteStoragePool(forced=True)%s" + % (label, pool.name, + "; the API raised: %s" % delete_error if delete_error else "") + ) + + def _cleanup_isolated_pool(self, pool, label): + """Best-effort teardown of one isolated pool and its ONTAP FlexVol. + + Igroups are intentionally left alone: their names carry no pool + identity, so the pool delete owns their removal. + """ + if pool is None: + return + try: + listed = list_storage_pools(self.apiClient, name=pool.name) + except Exception: + listed = None + if listed: + self._remove_cs_volume(pool, self.__class__.volume2, label) + try: + listed = list_storage_pools( + self.apiClient, name=pool.name + ) or listed + if listed[0].state != "Maintenance": + self._enter_maintenance(pool.id) + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning("[%s] could not force-delete pool '%s': %s", + label, pool.name, exc) + try: + self.ontap.offline_and_delete_volume(pool.name) + except Exception as exc: + logger.warning("[%s] ONTAP FlexVol cleanup for '%s' failed: %s", + label, pool.name, exc) + try: + listed = list_storage_pools(self.apiClient, name=pool.name) + except Exception: + listed = None + if not listed: + self.__class__.pool2 = None + + # ------------------------------------------------------------------ + # Step 08 — FlexVol deleted on ONTAP before the pool delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_08_delete_pool_with_volume_flexvol_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after its + ONTAP FlexVol — and with it the volume's LUN — has been removed behind + CloudStack's back. + + Uses its own pool and volume, so the pool is known to be healthy up to + the point the FlexVol is destroyed. Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing FlexVol + - the CloudStack pool record is removed + - the leftover CS volume record can still be cleaned up + """ + label = "flexvol-missing" + pool, vol = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool.id) + + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should be gone before the pool delete" + % (label, pool.name) + ) + self.assertEqual( + len(self.ontap.list_luns_in_volume(self.svm_name, pool.name)), 0, + "[%s] LUNs should have gone with the FlexVol '%s'" + % (label, pool.name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + try: + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning("[%s] could not enter maintenance on '%s': %s", + label, pool.name, exc) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate a missing " + "FlexVol, but raised: %s" % (label, delete_error) + ) + + self._assert_igroup_baseline_unchanged( + "[%s] after pool delete with missing FlexVol" % label + ) + + self._purge_cs_volume_record(vol, label) + finally: + self._cleanup_isolated_pool(pool, label) + + # ------------------------------------------------------------------ + # Step 09 — Host igroups deleted on ONTAP before the delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_09_delete_pool_with_volume_igroups_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after the host + igroups have been removed behind CloudStack's back. + + Uses its own pool and volume. The volume is not attached to any VM, so + no LUN maps reference the igroups and they delete cleanly. Unlike + test_08 the FlexVol is still present, so the plugin is expected to + remove it as part of the delete. Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing igroups + - the CloudStack pool record is removed + - the ONTAP FlexVol is deleted and no igroup is left behind + """ + other_pools = self._other_ontap_pools_on_svm(None) + if other_pools: + self.skipTest( + "Pre-deleting SVM-wide igroups requires exclusive SVM use; " + "found other ONTAP pool(s): %s" + % ", ".join(str(getattr(p, "name", p)) for p in other_pools) + ) + label = "igroups-missing" + pool, vol = self._create_isolated_pool_with_volume(label) + seeded = [] + try: + igroup_specs = self._host_igroup_specs() + self.assertTrue( + igroup_specs, + "[%s] no cluster host reports an IQN, so there is no igroup " + "to remove" % label + ) + igroup_names = [name for name, _ in igroup_specs] + + # The volume is not attached to a VM, so the plugin has never + # granted a host access and created no igroups. Seed them under + # the plugin's own names so the pre-deletion is a real one. + for name, iqn in igroup_specs: + if self.ontap.get_igroup(self.svm_name, name) is None: + logger.info("[%s] seeding igroup '%s' with initiator '%s'", + label, name, iqn) + self.ontap.create_igroup(self.svm_name, name, iqn) + seeded.append(name) + + self._enter_maintenance(pool.id) + + deleted = [] + for name in igroup_names: + if self.ontap.get_igroup(self.svm_name, name) is None: + continue + self.ontap.delete_igroup(self.svm_name, name) + deleted.append(name) + logger.info("[%s] deleted %d of %d host igroup(s): %s", + label, len(deleted), len(igroup_names), deleted) + + for name in igroup_names: + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, name), + "[%s] igroup '%s' should be gone before the pool delete" + % (label, name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + try: + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning("[%s] could not enter maintenance on '%s': %s", + label, pool.name, exc) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate missing " + "igroups, but raised: %s" % (label, delete_error) + ) + + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should have been deleted with the pool" + % (label, pool.name) + ) + for name in igroup_names: + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, name), + "[%s] igroup '%s' reappeared during the pool delete" + % (label, name) + ) + + self._purge_cs_volume_record(vol, label) + finally: + for name in seeded: + try: + self.ontap.delete_igroup(self.svm_name, name) + except Exception as exc: + logger.warning( + "[%s] cleanup: could not delete seeded igroup '%s': %s", + label, name, exc) + self._cleanup_isolated_pool(pool, label) + + # ------------------------------------------------------------------ + # Step 10 — Enter maintenance after LUN maps are pre-deleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_10_enter_maintenance_lun_maps_predeleted(self): + """ + Enter maintenance with a CloudStack volume after its ONTAP LUN maps + have been deleted behind CloudStack's back. Verifies: + - enableStorageMaintenance tolerates already-absent LUN maps + - the pool reaches Maintenance and the CS volume remains present + - the LUN remains online and its maps remain absent + """ + label = "maintenance-lun-maps-missing" + pool, vol = self._create_isolated_pool_with_volume(label) + seeded_igroup = None + try: + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + luns, + "[%s] no LUN found in FlexVol '%s'" % (label, pool.name) + ) + lun_path = luns[0].get("name") + + igroup_specs = self._host_igroup_specs() + self.assertTrue( + igroup_specs, + "[%s] no cluster host reports an IQN" % label + ) + igroup_name, initiator_iqn = igroup_specs[0] + if self.ontap.get_igroup(self.svm_name, igroup_name) is None: + self.ontap.create_igroup( + self.svm_name, igroup_name, initiator_iqn + ) + seeded_igroup = igroup_name + + self.ontap.create_lun_map( + self.svm_name, lun_path, igroup_name + ) + maps = self.ontap.list_lun_maps_for_volume( + self.svm_name, pool.name + ) + self.assertTrue( + maps, + "[%s] failed to seed a LUN map for '%s'" % (label, lun_path) + ) + + for lun_map in maps: + self.ontap.delete_lun_map(lun_map) + self.assertEqual( + self.ontap.list_lun_maps_for_volume( + self.svm_name, pool.name + ), + [], + "[%s] LUN maps should be absent before maintenance" % label + ) + + self._enter_maintenance(pool.id) + self.assertTrue( + self._volume_exists_in_cs(vol.id), + "[%s] CS volume disappeared after entering maintenance" % label + ) + self._assert_lun_exists( + pool.name, "after entering Maintenance with maps pre-deleted" + ) + self.assertEqual( + self.ontap.list_lun_maps_for_volume( + self.svm_name, pool.name + ), + [], + "[%s] LUN maps unexpectedly reappeared during maintenance" + % label + ) + finally: + self._cleanup_isolated_pool(pool, label) + if seeded_igroup: + try: + self.ontap.delete_igroup( + self.svm_name, seeded_igroup + ) + except Exception as exc: + logger.warning( + "[%s] cleanup: could not delete seeded igroup '%s': %s", + label, seeded_igroup, exc + ) + + def _test_iscsi_endpoint(self, host_ip=None): + """Return a reachable data LIF and target IQN for the suite SVM.""" + lifs = self.ontap.get_iscsi_data_lifs(self.svm_name) + target = self.ontap.get_iscsi_target_iqn(self.svm_name) + if not lifs or not target: + self.skipTest( + "SVM '%s' has no iSCSI data LIF or target IQN" % self.svm_name + ) + if host_ip: + for lif in lifs: + quoted_lif = lif.replace("'", "'\"'\"'") + reachable = self._kvm_run( + host_ip, + "timeout 3 bash -c " + "'exec 3<>/dev/tcp/%s/3260' >/dev/null 2>&1 " + "&& echo OPEN || true" % quoted_lif + ) + if any(line.strip() == "OPEN" for line in reachable): + return lif, target + self.skipTest( + "No iSCSI data LIF for SVM '%s' is reachable from %s" + % (self.svm_name, host_ip) + ) + return lifs[0], target + + def _iscsi_session_lines(self, host_ip, target): + quoted = target.replace("'", "'\"'\"'") + return self._kvm_run( + host_ip, + "iscsiadm -m session 2>/dev/null | grep -F '%s' || true" % quoted + ) + + def _login_test_iscsi_session(self, host_ip, portal, target): + """Log in only this target. Skip when the target is already in use.""" + if self._iscsi_session_lines(host_ip, target): + self.skipTest( + "iSCSI target '%s' is already logged in on %s; logging it " + "out would drop other LUNs" % (target, host_ip) + ) + portal_port = "%s:3260" % portal + quoted_target = target.replace("'", "'\"'\"'") + quoted_portal = portal_port.replace("'", "'\"'\"'") + self._kvm_run( + host_ip, + "iscsiadm -m node -T '%s' -p '%s' -o new >/dev/null 2>&1 || true; " + "timeout 30 iscsiadm -m node -T '%s' -p '%s' " + "--login >/dev/null 2>&1 || true" + % (quoted_target, quoted_portal, quoted_target, quoted_portal) + ) + if not self._iscsi_session_lines(host_ip, target): + self.skipTest( + "Could not log in test iSCSI target '%s' on %s" + % (target, host_ip) + ) + + def _logout_test_iscsi_session(self, host_ip, portal, target): + portal_port = "%s:3260" % portal + quoted_target = target.replace("'", "'\"'\"'") + quoted_portal = portal_port.replace("'", "'\"'\"'") + self._kvm_run( + host_ip, + "iscsiadm -m node -T '%s' -p '%s' --logout >/dev/null 2>&1 || true" + % (quoted_target, quoted_portal) + ) + + def _map_isolated_lun(self, pool, label): + """Map the isolated pool's LUN and return path, igroup and LUN id.""" + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue(luns, "[%s] no LUN found in '%s'" % (label, pool.name)) + lun_path = luns[0].get("name") + specs = self._host_igroup_specs() + self.assertTrue(specs, "[%s] no host IQN is available" % label) + igroup_name, initiator = specs[0] + seeded = None + if self.ontap.get_igroup(self.svm_name, igroup_name) is None: + self.ontap.create_igroup(self.svm_name, igroup_name, initiator) + seeded = igroup_name + self.ontap.create_lun_map(self.svm_name, lun_path, igroup_name) + maps = self.ontap.list_lun_maps_for_volume(self.svm_name, pool.name) + self.assertTrue(maps, "[%s] LUN map was not created" % label) + lun_id = maps[0].get("logical_unit_number") + return lun_path, igroup_name, lun_id, seeded + + # ------------------------------------------------------------------ + # Step 11 — Test iSCSI session logged out on one host + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_11_iscsi_session_logged_out(self): + """ + Log out only the isolated test target on one KVM host, then enter + maintenance. The session is restored before the pool is deleted. + """ + label = "iscsi-session-logout" + host_ip = self._kvm_host_ip() + portal, target = self._test_iscsi_endpoint(host_ip) + pool, vol = self._create_isolated_pool_with_volume(label) + seeded = None + logged_in = False + try: + _, _, _, seeded = self._map_isolated_lun(pool, label) + self._login_test_iscsi_session(host_ip, portal, target) + logged_in = True + self._logout_test_iscsi_session(host_ip, portal, target) + logged_in = False + self.assertFalse( + self._iscsi_session_lines(host_ip, target), + "[%s] test iSCSI session still present after logout" % label + ) + + self._enter_maintenance(pool.id) + self.assertTrue( + self._volume_exists_in_cs(vol.id), + "[%s] volume disappeared after maintenance" % label + ) + self._assert_lun_exists(pool.name, label) + finally: + if logged_in: + try: + self._logout_test_iscsi_session(host_ip, portal, target) + except Exception as exc: + logger.warning( + "[%s] could not log out test session: %s", label, exc + ) + self._cleanup_isolated_pool(pool, label) + if seeded: + try: + self.ontap.delete_igroup(self.svm_name, seeded) + except Exception as exc: + logger.warning( + "[%s] could not delete seeded igroup: %s", label, exc + ) + + # ------------------------------------------------------------------ + # Step 12 — Volume delete while the test session already exists + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_12_delete_volume_with_existing_iscsi_session(self): + """ + Use the existing target session while CloudStack deletes the volume. + Delete must still remove the LUN and must not create another session. + """ + label = "iscsi-session-exists" + host_ip = self._kvm_host_ip() + portal, target = self._test_iscsi_endpoint(host_ip) + existing_sessions = self._iscsi_session_lines(host_ip, target) + created_session = False + if not existing_sessions: + self._login_test_iscsi_session(host_ip, portal, target) + created_session = True + existing_sessions = self._iscsi_session_lines(host_ip, target) + pool, vol = self._create_isolated_pool_with_volume(label) + seeded = None + try: + _, _, _, seeded = self._map_isolated_lun(pool, label) + before = len(existing_sessions) + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume2 = None + self.assertFalse( + self._volume_exists_in_cs(vol.id), + "[%s] volume remained after deleteVolume" % label + ) + self.assertEqual( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + [], + "[%s] LUN remained after deleteVolume" % label + ) + after = len(self._iscsi_session_lines(host_ip, target)) + self.assertLessEqual( + after, before, + "[%s] deleteVolume created an extra iSCSI session" % label + ) + finally: + self._cleanup_isolated_pool(pool, label) + if seeded: + try: + self.ontap.delete_igroup(self.svm_name, seeded) + except Exception as exc: + logger.warning( + "[%s] could not delete seeded igroup: %s", label, exc + ) + if created_session: + try: + self._logout_test_iscsi_session(host_ip, portal, target) + except Exception as exc: + logger.warning( + "[%s] could not log out test session: %s", label, exc + ) + + # ------------------------------------------------------------------ + # Step 13 — Regular file planted at the test LUN by-path + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_13_corrupt_iscsi_by_path(self): + """ + Replace only the isolated LUN's by-path symlink with a regular file. + Deleting the unattached volume must still remove the ONTAP LUN and + must leave that regular file in place. + """ + label = "iscsi-by-path-file" + host_ip = self._kvm_host_ip() + portal, target = self._test_iscsi_endpoint(host_ip) + existing_sessions = self._iscsi_session_lines(host_ip, target) + created_session = False + if not existing_sessions: + self._login_test_iscsi_session(host_ip, portal, target) + created_session = True + existing_sessions = self._iscsi_session_lines(host_ip, target) + portal_field = existing_sessions[0].split()[2] + portal = portal_field.rsplit(":3260,", 1)[0] + if portal == portal_field: + self.skipTest( + "[%s] could not parse portal from session: %s" + % (label, existing_sessions[0]) + ) + pool, vol = self._create_isolated_pool_with_volume(label) + seeded = None + by_path = None + try: + _, _, lun_id, seeded = self._map_isolated_lun(pool, label) + if lun_id is None: + self.skipTest("[%s] ONTAP did not report a LUN number" % label) + by_path = ( + "/dev/disk/by-path/ip-%s:3260-iscsi-%s-lun-%s" + % (portal, target, lun_id) + ) + quoted = by_path.replace("'", "'\"'\"'") + quoted_target = target.replace("'", "'\"'\"'") + quoted_portal = ("%s:3260" % portal).replace("'", "'\"'\"'") + self._kvm_run( + host_ip, + "iscsiadm -m node -T '%s' -p '%s' --rescan" + % (quoted_target, quoted_portal) + ) + self.assertTrue( + self._kvm_run( + host_ip, + "if [ -L '%s' ]; then echo SYMLINK; fi" % quoted + ), + "[%s] expected iSCSI by-path symlink was not found at %s" + % (label, by_path) + ) + planted = self._kvm_run( + host_ip, + "if [ -L '%s' ]; then mv '%s' '%s.bak'; fi; " + "rm -f '%s'; : > '%s'; " + "if [ -f '%s' ] && [ ! -L '%s' ]; then echo FILE; fi" + % (quoted, quoted, quoted, quoted, quoted, quoted, quoted) + ) + self.assertTrue( + any(line.strip() == "FILE" for line in planted), + "[%s] could not plant a regular file at %s" % (label, by_path) + ) + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume2 = None + self.assertEqual( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + [], + "[%s] LUN remained after deleteVolume" % label + ) + still_file = self._kvm_run( + host_ip, + "if [ -f '%s' ] && [ ! -L '%s' ]; then echo FILE; fi" + % (quoted, quoted) + ) + self.assertTrue( + any(line.strip() == "FILE" for line in still_file), + "[%s] CloudStack replaced the planted file at %s" + % (label, by_path) + ) + finally: + if by_path: + quoted = by_path.replace("'", "'\"'\"'") + try: + self._kvm_run( + host_ip, + "rm -f '%s' '%s.bak'" % (quoted, quoted) + ) + except Exception as exc: + logger.warning( + "[%s] could not clean up %s: %s", label, by_path, exc + ) + self._cleanup_isolated_pool(pool, label) + if seeded: + try: + self.ontap.delete_igroup(self.svm_name, seeded) + except Exception as exc: + logger.warning( + "[%s] could not delete seeded igroup: %s", label, exc + ) + if created_session: + try: + self._logout_test_iscsi_session(host_ip, portal, target) + except Exception as exc: + logger.warning( + "[%s] could not log out test session: %s", label, exc + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py new file mode 100644 index 000000000000..52b4fcaabff2 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py @@ -0,0 +1,413 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Zone-scoped primary storage lifecycle tests for NetApp ONTAP (iSCSI). + +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). Host igroups are +shared by host and SVM and are created only when a LUN is granted to a host, +not when an empty pool is created. + +Test order — sequential workflow that must run in order: + 01 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; + pre-existing shared igroups unchanged + 02 Grow and shrink zone-scoped pool — CloudStack and ONTAP agree + 03 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 04 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 05 Delete zone-scoped pool — pool gone; FlexVol deleted; baseline restored + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM hosts with iSCSI registered in the zone + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import ( + OntapRestClient, + OntapTestBase, + get_datacenter_config, + log_progress, +) + +logger = logging.getLogger("TestOntapISCSIZoneScopedPool") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + provider="NetApp ONTAP", tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-zone@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-Zone", + "username": "ontap_iscsi_zone_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapZoneISCSI_%d" % random.randint(0, 9999), + TestData.scope: "ZONE", + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIZoneScopedPool(OntapTestBase): + + _vol_name_prefix = "OntapISCSIZoneVol" + + @classmethod + def setUpClass(cls): + super(TestOntapISCSIZoneScopedPool, cls).setUpClass() + testclient = super( + TestOntapISCSIZoneScopedPool, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + provider=provider, tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + cls._capture_igroup_baseline() + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_zone_pool(self): + """Create a zone-scoped iSCSI pool (no clusterid / podid).""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapZoneISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + # Intentionally omit clusterid and podid — zone-scoped pool + cmd.scope = "ZONE" + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ------------------------------------------------------------------ + # Step 01 — Create zone-scoped iSCSI pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_01_create_zone_scoped_pool(self): + """ + Create a zone-scoped iSCSI primary storage pool (no clusterid/podid). + Verifies: + - pool.state is Up, type is OntapiSCSI + - ONTAP: FlexVol is online + - ONTAP: pre-existing shared igroups are unchanged + """ + pool = self._create_zone_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "OntapiSCSI", + "Pool type should be 'OntapiSCSI', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: the plugin creates an igroup only when a host is first + # granted access to a LUN, so creating an empty pool must not make + # new ones. + self._assert_igroup_baseline_unchanged( + "after creating an empty zone-scoped pool" + ) + + # ------------------------------------------------------------------ + # Step 02 — Grow and shrink zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_02_grow_and_shrink_zone_scoped_pool(self): + """Grow the zone-scoped pool and shrink it back to its original size.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Zone-scoped pool missing before resize") + self.assertEqual( + listed[0].state, "Up", + "Zone-scoped pool must be Up before resize, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before zone-scoped resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s zone-scoped iSCSI pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Up", + "Zone-scoped pool should remain Up after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + self._assert_igroup_baseline_unchanged( + "after %s of empty zone-scoped pool" % operation + ) + + # ------------------------------------------------------------------ + # Step 03 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_03_disable_zone_scoped_pool(self): + """ + Disable the zone-scoped iSCSI pool. + Verifies: + - pool.state is Disabled + - ONTAP: FlexVol still online; igroups unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_02 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable" + ) + + # The pool has no volumes, so shared igroups must remain unchanged. + self._assert_igroup_baseline_unchanged( + "after disabling an empty zone-scoped pool" + ) + + # ------------------------------------------------------------------ + # Step 04 — Enable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_04_enable_zone_scoped_pool(self): + """ + Re-enable the zone-scoped iSCSI pool. + Verifies: + - pool.state is Up + - ONTAP: FlexVol still online; igroups unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable" + ) + + # The pool has no volumes, so shared igroups must remain unchanged. + self._assert_igroup_baseline_unchanged( + "after re-enabling an empty zone-scoped pool" + ) + + # ------------------------------------------------------------------ + # Step 05 — Delete zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_05_delete_zone_scoped_pool(self): + """ + Enter maintenance then delete the zone-scoped iSCSI pool. + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol deleted + - ONTAP: this pool's maps are gone and shared igroups are restored + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_04 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + + self._enter_maintenance(pool.id) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + self._assert_no_lun_maps_for_volume( + pool_name, "after zone-scoped pool deletion" + ) + self._assert_igroup_baseline_unchanged( + "after zone-scoped pool deletion" + ) diff --git a/test/integration/plugins/ontap/iscsi/template/__init__.py b/test/integration/plugins/ontap/iscsi/template/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py b/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py new file mode 100644 index 000000000000..33ee6f7a5a38 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI primary template-cache negative / boundary suite (Marvin). + +Independent of the sequential happy-path suite. + + 01 Tag mismatch — no spool_ref / cache on ONTAP pool + 02 Undersized pool — deploy fails + 03 Out-of-band cache delete — reuse deploy fails + +Running: + bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache_negative +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_negative_workflow import ( + OntapTemplateCacheNegativeWorkflow, +) + + +class TestOntapIscsiTemplateCacheNegative(OntapTemplateCacheNegativeWorkflow): + PROTOCOL = "ISCSI" + NOSE_TAG = "iscsi_template_cache_negative" + PROTOCOL_CFG_KEY = "iscsi" + POOL_URL_SCHEME = "iscsi" + POOL_NAME_PREFIX = "OntapIscsiTmplNeg" + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_01_tag_mismatch_does_not_seed_cache(self): + self.step_01_tag_mismatch_does_not_seed_cache() + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_02_undersized_pool_deploy_fails(self): + self.step_02_undersized_pool_deploy_fails() + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_03_deleted_cache_blocks_reuse(self): + self.step_03_deleted_cache_blocks_reuse() diff --git a/test/integration/plugins/ontap/iscsi/volume/__init__.py b/test/integration/plugins/ontap/iscsi/volume/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/volume/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py b/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py new file mode 100644 index 000000000000..3056f36c422c --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py @@ -0,0 +1,417 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI data volume +lifecycle (LUN create / delete / negative-delete / force-delete). + +Tests are numbered test_01 ... test_05 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI primary storage pool (infrastructure) and allocate a + CloudStack data volume — LUN is created inside the pool's ONTAP FlexVol + 02 Delete the volume — LUN is removed from the FlexVol + 03 Recreate volume — LUN is present again (setup for negative delete tests) + 04 Put pool in Maintenance; attempt forced=False deleteStoragePool — must be + rejected because volumes exist; pool stays in Maintenance + 05 Delete volume from Maintenance; forced=True deleteStoragePool — FlexVol, + igroups, and all LUNs are removed from ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/volume/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +The pool is cleaned up in test_05 on the happy path; OntapTestBase tearDownClass +provides a safety net for mid-run failures. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config + +logger = logging.getLogger("TestOntapISCSIVolumeLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-vol@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-Vol", + "username": "ontap_iscsi_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIVol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIVolumeLifecycle(OntapTestBase): + + _vol_name_prefix = "OntapISCSIVol" + + @classmethod + def setUpClass(cls): + super(TestOntapISCSIVolumeLifecycle, cls).setUpClass() + testclient = super( + TestOntapISCSIVolumeLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIVol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ------------------------------------------------------------------ + # Step 01 - Create pool (infrastructure) and allocate a volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create a new iSCSI pool and allocate a CloudStack data volume on it. + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: at least one LUN exists in the pool's FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: at least one LUN must be present in the pool FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" % pool.name + ) + + # ------------------------------------------------------------------ + # Step 02 - Delete volume; LUN must be removed from ONTAP + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_02_delete_volume(self): + """ + Delete the volume created in test_01. + Verifies: + - deleteVolume completes without error + - ONTAP: LUN is removed from the pool's FlexVol + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_01 must pass first") + + pool = self.__class__.pool + vol = self.__class__.volume + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume deletion, found %d: %s" + % (pool.name, len(luns), luns) + ) + + # ------------------------------------------------------------------ + # Step 03 - Recreate volume for negative delete tests + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_03_recreate_volume_for_delete_tests(self): + """ + Recreate a volume on the existing pool (setup for tests 04-05). + Verifies: + - volume created successfully + - ONTAP: LUN present in pool FlexVol + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + luns = self.ontap.list_luns_in_volume(self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume re-creation" + % self.__class__.pool.name + ) + + # ------------------------------------------------------------------ + # Step 04 - Forced=False delete with live volume must fail + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_04_forced_false_delete_with_volume_fails(self): + """ + Put pool in Maintenance then attempt deleteStoragePool(forced=False). + With a live volume present CloudStack must reject the request. + Verifies: + - CloudstackAPIException is raised + - Pool is still listed in CloudStack (in Maintenance state) + - ONTAP: FlexVol still exists and is online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + + # Enter maintenance mode + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise exception because volumes exist + with self.assertRaises(Exception): + self._delete_pool(pool.id, forced=False) + + # Pool must still be listed in CloudStack + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + listed, + "Pool should still exist in CloudStack after failed forced=False delete" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after failed delete" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Delete volume then force-delete pool from Maintenance + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_05_delete_volume_and_force_delete_pool(self): + """ + Delete the live volume then force-delete the pool while it is still + in Maintenance state (pool is in Maintenance from test_04). + Verifies: + - Volume can be deleted while pool is in Maintenance + - Pool is removed from CloudStack using forced=True from Maintenance + - ONTAP: FlexVol deleted + - ONTAP: igroups deleted for all cluster hosts + """ + self.assertIsNotNone( + self.__class__.pool, + "Pool absent - test_04 must not have cleaned up the pool" + ) + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume first (pool is in Maintenance — volume deletion is allowed) + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # Force-delete the pool from Maintenance (no live volumes remaining) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after force deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after force deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/nfs3/__init__.py b/test/integration/plugins/ontap/nfs3/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/instance/__init__.py b/test/integration/plugins/ontap/nfs3/instance/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/instance/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py new file mode 100644 index 000000000000..d3450949baf2 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py @@ -0,0 +1,2018 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP data volume lifecycle +with a running virtual machine. + +Tests are numbered test_01 ... test_27, plus test_03a and test_03b, and must +run in that order. Each step builds on the shared state established by the +previous step. + +The VM is deployed with a compute offering whose storage tags match the pool, +so its ROOT volume lands on the ONTAP pool and exercises the primary +template cache (seed / reuse / survive VM delete) alongside the data-volume +resize and snapshot workflow. + +Workflow: + 01 Create NFS3 primary storage pool on ONTAP (tagged -tmpl-cache) + 02 Create a CloudStack data volume on the ONTAP pool + 03 Deploy a VM — ROOT on ONTAP; seeds template cache (template_spool_ref + Ready + cache file on the FlexVol) + 03a Deploy a second VM — reuses the cache (still one template_spool_ref) + 03b Expunge the second VM — cache and template_spool_ref survive + 04 Attach the ONTAP data volume to the running VM + 05 Stop the VM — export policy stays; volume remains attached in CS + 06 Grow the attached volume and its qcow2 file while the VM is stopped + 07 Verify shrinking the volume is rejected without changing its size + 08 Start the VM — VM Running; volume still attached; FlexVol online + 09 Open an SSH port forward to the VM; write ROOT and DATA files + 10 Snapshot the ROOT volume of the running VM + 11 Snapshot the DATA volume of the running VM (kept for step 14) + 12 Revert a volume snapshot while the VM is Running — rejected + 13 ROOT volume revert: write before-snapshot, snapshot, write + after-snapshot, revert, file is back to before-snapshot + 14 DATA: snapshot again, revert to the step-11 snapshot then to this + one, then delete both while the VM is still running + 15 Write before-s1; VM snapshot of the running VM (kept for step 17) + 16 Write before-s2, stop, VM snapshot (kept for step 17) + 17 Revert to the step-15 snapshot, then to the step-16 snapshot, then + delete both while the VM is still running + 18 Two VM snapshot requests for the same VM at once — both succeed + 19 Deploy a second VM whose ROOT disk is on the same pool + 20 Revert VM-A's ROOT; VM-B's file is unchanged + 21 VM snapshots of both VMs at the same time + 22 Volume snapshot of VM-A while VM-B's ROOT reverts + 23 VM snapshot of VM-A while VM-B reverts to its VM snapshot + 24 Detach the ONTAP data volume from the VM + 25 Grow the detached data volume + 26 Expunge the second VM while it still has a VM snapshot + 27 Destroy VM (cache still survives); delete ONTAP volume; enter + maintenance; force-delete pool + +Snapshot steps only create and verify; tearDown deletes every snapshot the +step created and waits for its FlexVol snapshot to go. Step 11 keeps its +DATA snapshot so step 14 can revert to it and then delete both DATA +snapshots while the VM is running. Steps 15 and 16 keep their VM snapshots +so step 17 can revert to them and then delete both while the VM is running. + +Guest login for the CentOS 5.5 template comes from ontap.cfg +(cloudstack.templateUsername / templatePassword); see guest_ssh.py. + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack with at least one executable template + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import threading +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + startVirtualMachine as startVirtualMachineAPI, + stopVirtualMachine as stopVirtualMachineAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import ServiceOffering, Snapshot, StoragePool, VmSnapshot, Volume +from marvin.lib.common import list_storage_pools + +from guest_ssh import GuestSsh, guest_credentials, new_token, open_ssh_port +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config +from helpers import template_cache_util as tcu +from helpers.template_cache_base import ( + find_ready_kvm_template, + tagged_compute_offering_data, + template_cache_capacity_bytes, + template_cache_tags, +) + +logger = logging.getLogger("TestOntapVMVolumeAttach") + +TAG = "vm_volume_workflow" + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + # Used when ontap.cfg has neither templateCacheCapacitybytes nor capacitybytes. + DEFAULT_POOL_SIZE = 50 * 1024 ** 3 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if not capacitybytes: + capacitybytes = TestData.DEFAULT_POOL_SIZE + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-vm-vol@test.com", + "firstname": "ONTAP", + "lastname": "VMVol", + "username": "ontap_vm_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapVMVol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapVMVolumeAttach(OntapTestBase): + """ + Tests ONTAP data volume lifecycle with a running CloudStack VM whose ROOT + volume is on the ONTAP pool (primary template cache). + + All tests are sequential — state is carried on class attributes. + """ + + PROTOCOL = "NFS3" + + # ---- extra shared state beyond OntapTestBase ----------------------- + vm = None # running VirtualMachine + vm2 = None # second VM used only to verify template-cache reuse + template_id = None # KVM template ID discovered at setup + template_db_id = None # numeric vm_template.id (template_spool_ref key) + pool_db_id = None # numeric storage_pool.id (template_spool_ref key) + service_offering_id = None + network_id = None # None for Basic zones + _created_network_id = None # network created by this suite for Advanced zones + initial_volume_size = None + target_volume_size = None + volume_path = None + vm_ssh = None # (public_ip, public_port) forwarded to vm:22 + vm2_ssh = None + vm_snap_running = None # step 15 snapshot, reused by step 17 + vm_snap_stopped = None # step 16 snapshot, reused by step 17 + token_s1 = None + token_s2 = None + kept_ontap_snaps = None + data_snap_older = None # step 11 DATA snapshot, reused by step 14 + data_token_older = None + kept_data_ontap_snaps = None + guest_user = "root" + guest_password = "password" + + _vol_name_prefix = "OntapVMVol" + + # ---- setup --------------------------------------------------------- + + @classmethod + def setUpClass(cls): + super(TestOntapVMVolumeAttach, cls).setUpClass() + testclient = super( + TestOntapVMVolumeAttach, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + # Dedicated tag shared by the pool and the compute offering so the + # VM's ROOT volume is placed on this ONTAP pool (template cache path). + tags = template_cache_tags(nfs3_cfg, "nfs3") + capacitybytes = template_cache_capacity_bytes(pool_cfg) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + cls.guest_user, cls.guest_password = guest_credentials(config) + + # Discover a suitable user KVM template in the zone (must be fully + # downloaded; system-type templates are excluded as they cannot be + # deployed as user VMs). + template = find_ready_kvm_template(cls.apiClient, cls.zone.id) + if template is not None: + cls.template_id = template.id + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) + else: + logger.warning( + "No ready user KVM template found in zone '%s'. " + "Tests that deploy VMs will be skipped until a template " + "finishes downloading." % cls.zone.name + ) + cls.template_id = None + + service_offering = ServiceOffering.create( + cls.apiClient, tagged_compute_offering_data(tags, "OntapNFS3VMSO") + ) + cls._cleanup.append(service_offering) + cls.service_offering_id = service_offering.id + + # Detect zone type; resolve network ID for Advanced zones + cls.network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + # Find a network already accessible to the test account + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + # Create an Isolated guest network for the test account + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + no_offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in no_offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + no_offerings[0] if no_offerings else None + ) + if snat_offering: + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "ontap-nfs3-vm-net-%d" % random.randint( + 0, 9999) + cn_cmd.displaytext = "ONTAP NFS3 VM test network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def tearDownClass(cls): + """Destroy the VMs first, then delegate pool/volume cleanup to super.""" + cls._delete_kept_data_snapshot() + cls._delete_kept_vm_snapshots() + for vm in (cls.vm2, cls.vm): + if vm is None: + continue + try: + cls._stop_and_expunge_vm(vm.id) + except Exception as e: + logger.warning("tearDownClass: could not stop/destroy VM %s: " + "%s" % (vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as de: + logger.warning("tearDownClass: could not destroy VM %s: " + "%s" % (vm.id, de)) + cls.vm2 = None + cls.vm = None + + # Delete the guest network created for this account in Advanced zones. + if cls._created_network_id is not None: + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + cls._created_network_id = None + except Exception as e: + logger.warning( + "tearDownClass: could not delete network %s: %s" + % (cls._created_network_id, e)) + + super(TestOntapVMVolumeAttach, cls).tearDownClass() + + @classmethod + def _stop_and_expunge_vm(cls, vm_id): + """Force-stop the VM if needed, then destroy it with expunge.""" + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=120) + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + + # ---- pool creation helper ----------------------------------------- + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapVMVol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ---- VM state helpers ---------------------------------------------- + + def _volume_state(self, vol_id): + """Return the current CloudStack state string for a volume.""" + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + vols = self.apiClient.listVolumes(cmd) + return vols[0].state if vols else "unknown" + + # ---- snapshot step helpers (steps 09-23) --------------------------- + + def setUp(self): + super(TestOntapVMVolumeAttach, self).setUp() + self._volume_snapshot_ids = [] + self._vm_snapshot_ids = [] + self._extra_volume_ids = [] + self._created_ontap_snaps = set() + + def tearDown(self): + """ + Delete the snapshots and extra volumes a step created, then check + their FlexVol snapshots are gone from ONTAP. + """ + for snap_id in self._vm_snapshot_ids: + try: + VmSnapshot.deleteVMSnapshot(self.apiClient, snap_id) + except Exception as exc: + logger.warning("tearDown: VM snapshot %s: %s" % (snap_id, exc)) + for snap_id in self._volume_snapshot_ids: + try: + Snapshot({"id": snap_id}).delete(self.apiClient) + except Exception as exc: + logger.warning("tearDown: volume snapshot %s: %s" % (snap_id, exc)) + for vol_id in self._extra_volume_ids: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol_id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + logger.warning("tearDown: volume %s: %s" % (vol_id, exc)) + if self._created_ontap_snaps and not self._wait_ontap_snaps_gone( + self._created_ontap_snaps, fail=False): + logger.warning("tearDown: FlexVol snapshots still present: %s" + % sorted(self._created_ontap_snaps)) + super(TestOntapVMVolumeAttach, self).tearDown() + + @classmethod + def _destroy_second_vm(cls): + if cls.vm2 is None: + return + try: + cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + cmd.id = cls.vm2.id + cmd.expunge = True + cls.apiClient.destroyVirtualMachine(cmd) + except Exception as exc: + logger.warning("Could not destroy second VM %s: %s" % (cls.vm2.id, exc)) + cls.vm2 = None + cls.vm2_ssh = None + + @classmethod + def _delete_kept_vm_snapshots(cls): + """Delete the step 15/16 snapshots when step 17 did not take them over.""" + for snap in (cls.vm_snap_running, cls.vm_snap_stopped): + if snap is None: + continue + try: + VmSnapshot.deleteVMSnapshot(cls.apiClient, snap.id) + except Exception as exc: + logger.warning("tearDownClass: VM snapshot %s: %s" % (snap.id, exc)) + cls.vm_snap_running = None + cls.vm_snap_stopped = None + cls.token_s1 = None + cls.token_s2 = None + cls.kept_ontap_snaps = set() + + @classmethod + def _delete_kept_data_snapshot(cls): + """Delete the step 11 DATA snapshot when step 14 did not take it over.""" + snap = cls.data_snap_older + if snap is None: + return + try: + Snapshot({"id": snap.id}).delete(cls.apiClient) + except Exception as exc: + logger.warning("tearDownClass: volume snapshot %s: %s" % (snap.id, exc)) + cls.data_snap_older = None + cls.data_token_older = None + cls.kept_data_ontap_snaps = set() + + def _keep_data_snapshot(self, snap, ontap_names, token): + """Leave the step 11 DATA snapshot in place for step 14.""" + if snap.id in self._volume_snapshot_ids: + self._volume_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + self.__class__.data_snap_older = snap + self.__class__.data_token_older = token + self.__class__.kept_data_ontap_snaps = names + + def _adopt_kept_data_snapshot(self): + """Hand the step 11 DATA snapshot to step 14.""" + snap = self.__class__.data_snap_older + token = self.__class__.data_token_older + if snap is None or not token: + self.skipTest("DATA snapshot from test_11 is required") + ontap_names = set(self.__class__.kept_data_ontap_snaps or []) + self._volume_snapshot_ids.append(snap.id) + self._created_ontap_snaps |= ontap_names + self.__class__.data_snap_older = None + self.__class__.data_token_older = None + self.__class__.kept_data_ontap_snaps = set() + return snap, token, ontap_names + + def _delete_volume_snapshot_while_running(self, vm, snap, ontap_names, token): + """Delete a volume snapshot while its VM is still running.""" + Snapshot({"id": snap.id}).delete(self.apiClient) + if snap.id in self._volume_snapshot_ids: + self._volume_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + self._wait_until_gone( + lambda: self._volume_snapshot_listed(snap.id), + "Volume snapshot %s still listed after delete" % snap.id) + self._wait_ontap_snaps_gone(names) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_marker(token, disk="DATA", msg="after volume snapshot delete") + + def _delete_vm_snapshot_while_running(self, vm, snap, ontap_names, token): + """Delete a VM snapshot while that VM is still running.""" + VmSnapshot.deleteVMSnapshot(self.apiClient, snap.id) + if snap.id in self._vm_snapshot_ids: + self._vm_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + self._wait_until_gone( + lambda: any(item.id == snap.id for item in self._vm_snapshots_of(vm.id)), + "VM snapshot %s still listed after delete" % snap.id) + self._wait_ontap_snaps_gone(names) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_marker(token, msg="after VM snapshot delete") + + def _ontap_names_for_snap(self, names, snap, kind): + prefix = "cs" if kind == "volume" else "vm" + table = "snapshots" if kind == "volume" else "vm_snapshots" + db_id = tcu.get_db_id(self.dbConnection, table, snap.id) + suffix = "_%s%s" % (prefix, db_id) + return {name for name in names if str(name).endswith(suffix)} + + def _keep_vm_snapshot(self, snap, ontap_names, which): + """Leave this VM snapshot in place for step 17.""" + if snap.id in self._vm_snapshot_ids: + self._vm_snapshot_ids.remove(snap.id) + names = set(ontap_names) + self._created_ontap_snaps -= names + kept = self.__class__.kept_ontap_snaps + if kept is None: + kept = set() + self.__class__.kept_ontap_snaps = kept + kept.update(names) + setattr(self.__class__, which, snap) + + def _adopt_kept_vm_snapshots(self): + """Hand the step 15 and 16 snapshots to this step's tearDown.""" + s1 = self.__class__.vm_snap_running + s2 = self.__class__.vm_snap_stopped + token_a = self.__class__.token_s1 + token_b = self.__class__.token_s2 + if s1 is None or s2 is None or not token_a or not token_b: + self.skipTest("VM snapshots from test_15 and test_16 are required") + ontap_names = set(self.__class__.kept_ontap_snaps or []) + self._vm_snapshot_ids.extend([s1.id, s2.id]) + self._created_ontap_snaps |= ontap_names + self.__class__.vm_snap_running = None + self.__class__.vm_snap_stopped = None + self.__class__.token_s1 = None + self.__class__.token_s2 = None + self.__class__.kept_ontap_snaps = set() + return s1, s2, token_a, token_b, ontap_names + + def _require_snapshot_vm(self, ssh=True): + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + if ssh and self.__class__.vm_ssh is None: + self.skipTest("Guest SSH not set up — test_09 must pass first") + self._start_vm(self.__class__.vm.id) + return self.__class__.vm + + def _require_second_vm(self): + self._require_snapshot_vm() + if self.__class__.vm2 is None or self.__class__.vm2_ssh is None: + self.skipTest("Second VM not deployed — test_19 must pass first") + self._start_vm(self.__class__.vm2.id) + return self.__class__.vm2 + + def _vm_state(self, vm_id): + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + return vms[0].state if vms else None + + def _stop_vm(self, vm_id): + if (self._vm_state(vm_id) or "").lower() == "stopped": + return + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = vm_id + cmd.forced = True + self.apiClient.stopVirtualMachine(cmd) + self._poll_vm_state(vm_id, "Stopped", timeout=600) + + def _start_vm(self, vm_id): + if (self._vm_state(vm_id) or "").lower() == "running": + return + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = vm_id + self.apiClient.startVirtualMachine(cmd) + self._poll_vm_state(vm_id, "Running", timeout=900) + + def _root_volume(self, vm_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.type = "ROOT" + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + self.assertTrue(vols, "No ROOT volume found for VM %s" % vm_id) + return vols[0] + + def _guest(self, which="vm"): + host, port = getattr(self.__class__, "%s_ssh" % which) + return GuestSsh(host, port, self.guest_user, self.guest_password) + + def _write_marker(self, token, which="vm", disks=("ROOT",)): + with self._guest(which) as guest: + if "ROOT" in disks: + guest.write_root_marker(token) + if "DATA" in disks: + guest.write_data_marker(token) + + def _assert_marker(self, token, which="vm", disk="ROOT", msg=""): + with self._guest(which) as guest: + found = guest.read_data_marker() if disk == "DATA" else guest.read_root_marker() + self.assertEqual(found, token, "%s: %s marker on %s is %r, expected %r" + % (msg, disk, which, found, token)) + + def _ontap_snap_names(self): + return {s.get("name") for s in self.ontap.list_snapshots(self.__class__.pool.name)} + + def _assert_new_ontap_snaps(self, before, snaps, kind, msg): + """Return the FlexVol snapshot created for each CloudStack snapshot. + + A volume snapshot is stored as ``..._cs`` and a VM snapshot as + ``..._vm``. ONTAP snapshots from its own schedule, such as + ``hourly.*`` or ``Anti_ransomware_*``, are ignored. + """ + new = self._ontap_snap_names() - before + prefix = "cs" if kind == "volume" else "vm" + table = "snapshots" if kind == "volume" else "vm_snapshots" + matched = set() + missing = [] + for snap in snaps: + db_id = tcu.get_db_id(self.dbConnection, table, snap.id) + suffix = "_%s%s" % (prefix, db_id) + hits = sorted(name for name in new if name.endswith(suffix)) + if len(hits) != 1: + missing.append("%s (suffix %s, matches %s)" % (snap.id, suffix, hits)) + else: + matched.add(hits[0]) + self.assertFalse( + missing, + "%s: FlexVol snapshot missing for %s; new FlexVol snapshots: %s" + % (msg, missing, sorted(new))) + ignored = new - matched + if ignored: + logger.info("%s: ignoring FlexVol snapshots not created by this step: %s", + msg, sorted(ignored)) + self._created_ontap_snaps |= matched + return matched + + def _assert_ontap_snaps_present(self, names, msg): + missing = set(names) - self._ontap_snap_names() + self.assertFalse(missing, "%s: FlexVol snapshots missing: %s" % (msg, sorted(missing))) + + def _wait_ontap_snaps_gone(self, names, timeout=180, fail=True): + names = set(names) + deadline = time.time() + timeout + left = names + while time.time() < deadline: + left = names & self._ontap_snap_names() + if not left: + return True + time.sleep(5) + if fail: + self.fail("FlexVol snapshots %s still present after %ds" % (sorted(left), timeout)) + return False + + def _take_volume_snapshot(self, volume_id): + snap = Snapshot.create(self.apiClient, volume_id, + account=self.account.name, domainid=self.domain.id) + self._volume_snapshot_ids.append(snap.id) + self.assertEqual(snap.state, "BackedUp", + "Volume snapshot %s is %s, expected BackedUp" % (snap.id, snap.state)) + return snap + + def _take_vm_snapshot(self, vm_id): + snap = VmSnapshot.create(self.apiClient, vm_id, snapshotmemory="false", + name="ontap_%d" % random.randint(0, 999999)) + self._vm_snapshot_ids.append(snap.id) + self.assertEqual(snap.state, "Ready", + "VM snapshot %s is %s, expected Ready" % (snap.id, snap.state)) + return snap + + def _vm_snapshots_of(self, vm_id): + return VmSnapshot.list(self.apiClient, virtualmachineid=vm_id, listall=True) or [] + + def _volume_snapshot_listed(self, snap_id): + listed = Snapshot.list(self.apiClient, id=snap_id, listall=True) or [] + return any(getattr(item, "id", None) == snap_id for item in listed) + + def _wait_until_gone(self, still_there, msg, timeout=180): + deadline = time.time() + timeout + while time.time() < deadline and still_there(): + time.sleep(5) + self.assertFalse(still_there(), msg) + + def _volume_revert_flow(self, disk): + """Write A, snapshot, write B, stop, revert, start; expect A.""" + vm = self._require_snapshot_vm() + volume_id = (self.__class__.volume.id if disk == "DATA" + else self._root_volume(vm.id).id) + token_a = new_token("before-volume-snapshot") + token_b = new_token("after-volume-snapshot") + self._write_marker(token_a, disks=(disk,)) + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(volume_id) + new = self._assert_new_ontap_snaps(before, [snap], "volume", "%s volume snapshot" % disk) + self._write_marker(token_b, disks=(disk,)) + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, snap.id) + self._assert_ontap_snaps_present(new, "after revert") + self._start_vm(vm.id) + self._assert_marker(token_a, disk=disk, + msg="after revert (write after snapshot was %s)" % token_b) + + # ================================================================== + # Test steps + # ================================================================== + + # ------------------------------------------------------------------ + # Step 01 - Create NFS3 ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_01_create_nfs3_pool(self): + """ + Create an NFS3 ONTAP primary storage pool. + Verifies: + - Pool reaches 'Up' state in CloudStack + - ONTAP: FlexVol is created and online + """ + pool = self._create_pool() + self.__class__.pool = pool + self.__class__.pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not created for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 02 - Create CloudStack data volume on ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_02_create_ontap_data_volume(self): + """ + Allocate a CloudStack data volume on the ONTAP NFS3 pool. + Verifies: + - Volume is created and in 'Allocated' or 'Ready' state + - ONTAP: FlexVol remains online + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + pool = self.__class__.pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + vol_state = self._volume_state(vol.id) + self.assertIn( + vol_state.lower(), ("allocated", "ready"), + "Volume should be 'Allocated' or 'Ready', got '%s'" % vol_state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after data volume creation" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume creation" + ) + + # ------------------------------------------------------------------ + # Step 03 - Deploy a VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03_deploy_vm(self): + """ + Deploy a VM using the first available KVM template and the compute + offering tagged to match the ONTAP pool. + Verifies: + - VM reaches 'Running' state in CloudStack + - ROOT volume is placed on the ONTAP pool + - template_spool_ref for pool + template is Ready / DOWNLOADED + - ONTAP: template cache file exists at the spool install_path + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template available — " + "waiting for template download to complete" + ) + self.assertIsNotNone(self.__class__.service_offering_id, + "No service offering available — check setup") + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = self._poll_vm_state(vm.id, "Running") + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running', got '%s'" % vm_obj.state + ) + + pool = self.__class__.pool + self._assert_root_on_pool(vm.id, pool) + spool = self._wait_for_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # ------------------------------------------------------------------ + # Step 03a - Deploy a second VM — reuses the template cache + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03a_deploy_second_vm_reuses_template_cache(self): + """ + Deploy a second VM from the same template and offering. + Verifies: + - VM reaches 'Running'; its ROOT volume is on the ONTAP pool + - Still exactly one template_spool_ref (cache reused, not re-seeded) + - ONTAP: the same template cache file is still present + """ + self.assertIsNotNone(self.__class__.vm, + "VM absent — test_03 must pass first") + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm2 = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm2, "deployVirtualMachine returned None") + self.__class__.vm2 = vm2 + self._poll_vm_state(vm2.id, "Running") + + pool = self.__class__.pool + self._assert_root_on_pool(vm2.id, pool) + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # ------------------------------------------------------------------ + # Step 03b - Expunge the second VM — template cache survives + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03b_expunge_second_vm_template_cache_survives(self): + """ + Destroy (expunge) the second VM. The primary template cache is only + removed by storage GC / pool deletion, never by VM lifecycle. + Verifies: + - template_spool_ref is still Ready / DOWNLOADED + - ONTAP: template cache file is still present + """ + self.assertIsNotNone(self.__class__.vm2, + "Second VM absent — test_03a must pass first") + + self._stop_and_expunge_vm(self.__class__.vm2.id) + self.__class__.vm2 = None + + pool = self.__class__.pool + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # ------------------------------------------------------------------ + # Step 04 - Attach ONTAP data volume to the running VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_04_attach_volume_to_vm(self): + """ + Attach the ONTAP data volume to the running VM. + Verifies: + - Volume virtualmachineid is set to the VM's ID in CloudStack + (Note: on ONTAP/NFS shared storage the volume state remains 'Ready'; + attachment is signalled by virtualmachineid being populated) + - ONTAP: FlexVol remains online + - ONTAP: NFS3 volume data file created in FlexVol after attach (lazy creation) + - VM remains 'Running' + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = attachVolumeAPI.attachVolumeCmd() + cmd.id = vol.id + cmd.virtualmachineid = vm.id + attached = self.apiClient.attachVolume(cmd) + self.assertIsNotNone(attached, "attachVolume returned None") + + # On ONTAP/NFS shared storage CloudStack does not transition the volume + # state to 'In Use' — attachment is indicated by virtualmachineid being + # set on the volume record. Poll on that field instead of state. + deadline = time.time() + 120 + vol_vmid = None + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if vol_vmid: + break + time.sleep(5) + + self.assertEqual( + vol_vmid, vm.id, + "Volume should be attached to VM %s after attach, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # VM must still be Running + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=30) + self.assertEqual(vm_obj.state, "Running", + "VM should still be 'Running' after volume attach") + + # ONTAP FlexVol must remain online + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol not found after attach") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach" + ) + + # ONTAP: NFS3 uses lazy file creation — the volume data file is + # materialised on the FlexVol only when CloudStack calls createAsync + # during attachVolume. Verify that the file now exists. + files = self.ontap.list_files_in_volume(pool.name) + vol_file = next((f for f in files if vol.id in f), None) + self.assertIsNotNone( + vol_file, + "No data file matching volume UUID '%s' found in FlexVol '%s' " + "after attach; files present: %s" % (vol.id, pool.name, files) + ) + + volumes = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) or [] + self.assertTrue(volumes, "Attached volume is not listed in CloudStack") + initial_size = int(volumes[0].size) + self.__class__.initial_volume_size = initial_size + self.__class__.target_volume_size = ( + (initial_size // (1024 ** 3)) + 1 + ) * (1024 ** 3) + self.__class__.volume_path = vol_file + + backend_size = self._get_qcow2_virtual_size(pool.id, vol_file) + self.assertEqual( + backend_size, initial_size, + "Initial qcow2 virtual size should match the CloudStack volume size" + ) + + + # ------------------------------------------------------------------ + # Step 05 - Stop VM — export policy must be retained + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_05_stop_vm_export_retained(self): + """ + Stop the running VM while the NFS3 data volume is still attached. + Unlike iSCSI (where LUN-maps are removed on VM stop), NFS3 export + policies are not torn down when a VM stops — the FlexVol stays + accessible on the same mount. + Verifies: + - VM reaches Stopped state + - ONTAP: FlexVol still online + - CloudStack: volume virtualmachineid still set (volume stays attached) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = vm.id + self.apiClient.stopVirtualMachine(cmd) + + result = self._poll_vm_state(vm.id, "Stopped", timeout=300) + self.assertEqual( + result.state, "Stopped", + "VM should be 'Stopped', got '%s'" % result.state + ) + + # ONTAP: FlexVol must remain online — NFS export is not torn down on VM stop + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' disappeared after VM stop" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after VM stop, " + "got '%s'" % ontap_vol.get("state") + ) + + # CloudStack: volume must still be attached (virtualmachineid set) + cmd_list = listVolumesAPI.listVolumesCmd() + cmd_list.id = vol.id + vols = self.apiClient.listVolumes(cmd_list) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + self.assertEqual( + vol_vmid, vm.id, + "Volume should still be attached to VM %s after stop, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # ------------------------------------------------------------------ + + # ------------------------------------------------------------------ + # Step 06 - Grow attached volume while VM is stopped + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_06_grow_stopped_vm_volume(self): + """ + Grow the attached NFS3 volume while its managed KVM VM is stopped. + Verifies: + - CloudStack reports the target size and Ready state + - KVM reports the target qcow2 virtual size + - The backing file remains present in the ONTAP FlexVol + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + target_size = self.__class__.target_volume_size + self._resize_volume( + self.__class__.volume.id, target_size // (1024 ** 3)) + + volume = self._poll_volume_size( + self.__class__.volume.id, target_size) + self.assertIsNotNone( + volume, "CloudStack did not report the grown volume size") + + # KVM: qcow2 virtual size must reflect the new allocation + backend_size = self._get_qcow2_virtual_size( + self.__class__.pool.id, self.__class__.volume_path) + self.assertEqual( + backend_size, target_size, + "KVM qcow2 virtual size does not match the requested size") + + # DB: cloud.volumes.size must be updated by the driver's volumeDao call + db_size = self._get_db_volume_size(self.__class__.volume.id) + self.assertEqual( + db_size, target_size, + "cloud DB volumes.size (%s) does not match target (%s) after grow" + % (db_size, target_size)) + + # ONTAP REST: backing file must still be present in the FlexVol. + # Note: ONTAP's file-API returns bytes_used (actual disk blocks), NOT + # the qcow2 virtual size, so the host-side qemu-img check above is the + # correct NFS3 backend verification for virtual size. + files = self.ontap.list_files_in_volume(self.__class__.pool.name) + self.assertIn( + self.__class__.volume_path, files, + "NFS3 backing file disappeared from ONTAP after resize") + + # ------------------------------------------------------------------ + # Step 07 - Shrink must fail + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_07_shrink_rejected(self): + """ + Attempt to shrink the grown NFS3 volume back to its initial size. + Verifies: + - CloudStack rejects qcow2 shrink even with shrinkok=true + - CloudStack and qcow2 sizes remain at the grown value + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + self.assertEqual( + self._get_db_volume_size(self.__class__.volume.id), + self.__class__.target_volume_size, + "Volume was not grown to the target size — test_06 must pass " + "before the shrink check") + + initial_gib = self.__class__.initial_volume_size // (1024 ** 3) + with self.assertRaises(Exception) as context: + self._resize_volume( + self.__class__.volume.id, initial_gib, shrink_ok=True) + self.assertIn( + "Unable to shrink volumes of type QCOW2", str(context.exception), + "Shrink failed for an unexpected reason") + + volume = self._poll_volume_size( + self.__class__.volume.id, self.__class__.target_volume_size) + self.assertIsNotNone( + volume, "CloudStack volume size changed after rejected shrink") + + # DB: size must remain at grown value after rejected shrink + db_size = self._get_db_volume_size(self.__class__.volume.id) + self.assertEqual( + db_size, self.__class__.target_volume_size, + "cloud DB volumes.size did not remain at the grown target after " + "the rejected NFS3 shrink") + + backend_size = self._get_qcow2_virtual_size( + self.__class__.pool.id, self.__class__.volume_path) + self.assertEqual( + backend_size, self.__class__.target_volume_size, + "qcow2 virtual size changed after rejected shrink") + + # Step 08 - Start VM — volume accessible; FlexVol online + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_08_start_vm_volume_accessible(self): + """ + Start the stopped VM. + Verifies: + - VM reaches Running state + - ONTAP: FlexVol still online + - CloudStack: volume virtualmachineid still set (volume remains attached) + - VM remains 'Running' after start + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = vm.id + self.apiClient.startVirtualMachine(cmd) + + result = self._poll_vm_state(vm.id, "Running", timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after start, got '%s'" % result.state + ) + + # ONTAP: FlexVol must remain online after VM start + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after VM start" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after VM start, " + "got '%s'" % ontap_vol.get("state") + ) + + # CloudStack: volume must still be attached to the VM + cmd_list = listVolumesAPI.listVolumesCmd() + cmd_list.id = vol.id + vols = self.apiClient.listVolumes(cmd_list) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + self.assertEqual( + vol_vmid, vm.id, + "Volume should still be attached to VM %s after start, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + instance_name = getattr(result, "instancename", None) + self.assertIsNotNone( + instance_name, "Running VM response did not include instancename") + self.assertEqual( + self._get_vm_data_disk_capacity(instance_name), + self.__class__.target_volume_size, + "KVM data-disk capacity does not reflect the NFS3 resize") + + + # ------------------------------------------------------------------ + # Step 09 - Guest SSH access for snapshot checks + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_09_open_ssh_and_write_root_data_files(self): + """ + Prepare the running VM for snapshot checks. + Verifies: + - ROOT and DATA volumes are both on the ONTAP test pool + - SSH login (ontap.cfg templateUsername/templatePassword) works + through a port forward on the network's source-NAT IP + - The DATA disk can be formatted, mounted and written + """ + vm = self._require_snapshot_vm(ssh=False) + if not self.__class__.network_id: + self.skipTest("Snapshot steps need an Advanced zone guest network for SSH") + pool_id = self.__class__.pool.id + root = self._root_volume(vm.id) + self.assertEqual(getattr(root, "storageid", None), pool_id, + "ROOT volume is on '%s', not the ONTAP test pool" + % getattr(root, "storage", None)) + vols = self.apiClient.listVolumes(_list_vols_cmd(self.__class__.volume.id)) or [] + self.assertTrue(vols and getattr(vols[0], "virtualmachineid", None) == vm.id, + "DATA volume must be attached before the snapshot steps") + + host, port, _ = open_ssh_port(self.apiClient, vm.id, self.__class__.network_id) + self.__class__.vm_ssh = (host, port) + token = new_token("ssh-access-test") + self._write_marker(token, disks=("ROOT", "DATA")) + self._assert_marker(token, disk="ROOT", msg="guest access") + self._assert_marker(token, disk="DATA", msg="guest access") + + # ------------------------------------------------------------------ + # Steps 08-12 - Volume snapshots + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_10_root_volume_snapshot_create(self): + """ + VS-01: snapshot the ROOT volume of the running VM. + Verifies: state BackedUp; VM stays Running; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(self._root_volume(vm.id).id) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_new_ontap_snaps(before, [snap], "volume", "ROOT volume snapshot") + + @attr(tags=[TAG], required_hardware=True) + def test_11_data_volume_snapshot_create(self): + """ + VS-03: snapshot the attached DATA volume. + The snapshot is kept for step 14. + Verifies: state BackedUp; VM stays Running; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + token = new_token("data-s1") + self._write_marker(token, disks=("DATA",)) + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(self.__class__.volume.id) + self.assertEqual(self._vm_state(vm.id), "Running") + new = self._assert_new_ontap_snaps(before, [snap], "volume", "DATA volume snapshot") + self._keep_data_snapshot(snap, new, token) + + @attr(tags=[TAG], required_hardware=True) + def test_12_volume_revert_running_vm_rejected(self): + """ + VS-19: reverting a volume snapshot while the VM is Running fails. + Verifies: error says the Instance is not shut down; VM stays Running; + the FlexVol snapshot remains. + """ + vm = self._require_snapshot_vm() + before = self._ontap_snap_names() + snap = self._take_volume_snapshot(self._root_volume(vm.id).id) + new = self._assert_new_ontap_snaps(before, [snap], "volume", "ROOT volume snapshot") + with self.assertRaises(Exception) as ctx: + Volume.revertToSnapshot(self.apiClient, snap.id) + self.assertIn("shutdown", str(ctx.exception).lower(), + "Unexpected error: %s" % ctx.exception) + self.assertEqual(self._vm_state(vm.id), "Running") + self._assert_ontap_snaps_present(new, "after rejected revert") + + @attr(tags=[TAG], required_hardware=True) + def test_13_root_volume_revert(self): + """ + VS-11: write A, snapshot ROOT, write B, stop, revert, start. + Verifies: ROOT marker is A; the FlexVol snapshot survives the revert. + """ + self._volume_revert_flow("ROOT") + + @attr(tags=[TAG], required_hardware=True) + def test_14_data_volume_revert(self): + """ + Take a second DATA snapshot, revert to the step 11 snapshot, then to + this later one, and delete both while the VM is still running. + Verifies: + - After revert to the older snapshot, the DATA marker is the step 11 + token and the later snapshot is still listed + - After revert to the later snapshot, the DATA marker is that token + - Both snapshots and their FlexVol snapshots are gone; the VM stays + Running and the later marker is unchanged + """ + vm = self._require_snapshot_vm() + older, token_older, ontap_older = self._adopt_kept_data_snapshot() + token_latest = new_token("data-s2") + self._write_marker(token_latest, disks=("DATA",)) + before = self._ontap_snap_names() + latest = self._take_volume_snapshot(self.__class__.volume.id) + ontap_latest = self._assert_new_ontap_snaps( + before, [latest], "volume", "latest DATA snapshot") + self._write_marker(new_token("data-after-both"), disks=("DATA",)) + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, older.id) + self._start_vm(vm.id) + self._assert_marker(token_older, disk="DATA", + msg="after revert to the older snapshot") + self.assertTrue( + self._volume_snapshot_listed(latest.id), + "The later volume snapshot should still exist") + self._assert_ontap_snaps_present( + ontap_older | ontap_latest, "after revert to the older snapshot") + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, latest.id) + self._start_vm(vm.id) + self._assert_marker(token_latest, disk="DATA", + msg="after revert to the latest snapshot") + + self._delete_volume_snapshot_while_running( + vm, latest, ontap_latest, token_latest) + self._delete_volume_snapshot_while_running( + vm, older, ontap_older, token_latest) + + # ------------------------------------------------------------------ + # Steps 13-16 - VM snapshots + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_15_vm_snapshot_create_running(self): + """ + VMS-01: VM snapshot of the running VM (ROOT + DATA on one FlexVol). + Writes before-s1, then snapshots. The snapshot is kept for step 17. + Verifies: state Ready; VM stays Running; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + self.__class__.token_s1 = new_token("before-s1") + self._write_marker(self.__class__.token_s1, disks=("ROOT", "DATA")) + before = self._ontap_snap_names() + snap = self._take_vm_snapshot(vm.id) + self.assertEqual(self._vm_state(vm.id), "Running") + new = self._assert_new_ontap_snaps(before, [snap], "vm", "VM snapshot") + self._keep_vm_snapshot(snap, new, "vm_snap_running") + + @attr(tags=[TAG], required_hardware=True) + def test_16_vm_snapshot_create_stopped(self): + """ + VMS-03: VM snapshot of the stopped VM. + Writes before-s2 while Running, then stops and snapshots. + The snapshot is kept for step 17. + Verifies: state Ready; VM stays Stopped; one new FlexVol snapshot. + """ + vm = self._require_snapshot_vm() + self.__class__.token_s2 = new_token("before-s2") + self._write_marker(self.__class__.token_s2, disks=("ROOT", "DATA")) + self._stop_vm(vm.id) + before = self._ontap_snap_names() + snap = self._take_vm_snapshot(vm.id) + self.assertEqual(self._vm_state(vm.id), "Stopped") + new = self._assert_new_ontap_snaps(before, [snap], "vm", "VM snapshot (Stopped)") + self._keep_vm_snapshot(snap, new, "vm_snap_stopped") + + @attr(tags=[TAG], required_hardware=True) + def test_17_vm_snapshot_s1_running_s2_stopped_revert(self): + """ + VMS-09 (NFS ROOT + NFS DATA on the same pool): revert to the running + snapshot from step 15, then to the stopped snapshot from step 16. + Verifies: + - After revert to S1: ROOT and DATA hold before-s1 + - After revert to S2: ROOT and DATA hold before-s2 + - Both FlexVol snapshots survive the reverts + - Both VM snapshots are then deleted while the VM is still running + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + if self.__class__.vm_ssh is None: + self.skipTest("Guest SSH not set up — test_09 must pass first") + s1, s2, token_a, token_b, ontap_names = self._adopt_kept_vm_snapshots() + vm = self.__class__.vm + + self._stop_vm(vm.id) + VmSnapshot.revertToSnapshot(self.apiClient, s1.id) + self._start_vm(vm.id) + self._assert_marker(token_a, disk="ROOT", msg="after revert to S1") + self._assert_marker(token_a, disk="DATA", msg="after revert to S1") + self.assertEqual(len(self._vm_snapshots_of(vm.id)), 2, "S2 should still be listed") + + self._stop_vm(vm.id) + VmSnapshot.revertToSnapshot(self.apiClient, s2.id) + self._start_vm(vm.id) + self._assert_marker(token_b, disk="ROOT", msg="after revert to S2") + self._assert_marker(token_b, disk="DATA", msg="after revert to S2") + self._assert_ontap_snaps_present(ontap_names, "after both reverts") + + self._delete_vm_snapshot_while_running( + vm, s1, self._ontap_names_for_snap(ontap_names, s1, "vm"), token_b) + self._delete_vm_snapshot_while_running( + vm, s2, self._ontap_names_for_snap(ontap_names, s2, "vm"), token_b) + + @attr(tags=[TAG], required_hardware=True) + def test_18_vm_snapshot_duplicate_requests(self): + """ + VMS-53: two VM snapshot requests for the same VM at the same time. + A rejected request is retried on its own; one that already succeeded + is not repeated. + Verifies: both succeed and are Ready; two new FlexVol snapshots. + """ + vm = self._require_snapshot_vm() + before = self._ontap_snap_names() + pending = [ + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_dup_a_%d" % random.randint(0, 99999)), + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_dup_b_%d" % random.randint(0, 99999)), + ] + ok = [] + last_errors = [] + attempts = 3 + for attempt in range(1, attempts + 1): + results = _run_concurrent(*pending) + last_errors = [] + still_pending = [] + retryable = True + for fn, (result, exc) in zip(pending, results): + if exc is None and result is not None: + ok.append(result) + self._vm_snapshot_ids.append(result.id) + continue + last_errors.append(exc) + if exc is None or not _is_rejected(exc): + retryable = False + still_pending.append(fn) + pending = still_pending + if not pending: + break + if not retryable or attempt == attempts: + break + logger.warning("duplicate VM snapshot rejected, retry %d/%d: %s", + attempt, attempts, last_errors) + time.sleep(15) + self.assertFalse(pending, + "Duplicate VM snapshot requests failed: %s" % last_errors) + self.assertEqual(len(ok), 2, "Expected both VM snapshots to succeed; " + "got %d" % len(ok)) + for snap in ok: + self.assertEqual(snap.state, "Ready") + self.assertEqual(snap.virtualmachineid, vm.id) + self._assert_new_ontap_snaps(before, ok, "vm", "duplicate VM snapshot requests") + + # ------------------------------------------------------------------ + # Steps 17-21 - Second VM on the same pool: isolation and concurrency + # ------------------------------------------------------------------ + + @attr(tags=[TAG], required_hardware=True) + def test_19_deploy_second_vm(self): + """ + Deploy a second VM whose ROOT disk is on the same ONTAP pool. + Verifies: VM Running; ROOT on the test pool; SSH works. + """ + self._require_snapshot_vm() + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.networkids = self.__class__.network_id + vm2 = self.apiClient.deployVirtualMachine(cmd) + self.__class__.vm2 = vm2 + self._poll_vm_state(vm2.id, "Running", timeout=900) + root = self._root_volume(vm2.id) + self.assertEqual(getattr(root, "storageid", None), self.__class__.pool.id, + "Second VM ROOT is not on the ONTAP test pool") + host, port, _ = open_ssh_port(self.apiClient, vm2.id, self.__class__.network_id) + self.__class__.vm2_ssh = (host, port) + token = new_token("second-vm-ssh-access-test") + self._write_marker(token, which="vm2") + self._assert_marker(token, which="vm2", msg="second VM access") + + @attr(tags=[TAG], required_hardware=True) + def test_20_volume_revert_leaves_other_vm_untouched(self): + """ + VS-28: two VMs on one FlexVol; revert VM-A's ROOT to its snapshot. + Verifies: VM-A is back at its snapshot; VM-B keeps data written after it. + """ + vm = self._require_snapshot_vm() + self._require_second_vm() + a1 = new_token("vm-a-before-snapshot") + a2 = new_token("vm-a-after-snapshot") + b1 = new_token("vm-b-before-snapshot") + b2 = new_token("vm-b-after-snapshot") + self._write_marker(a1) + self._write_marker(b1, which="vm2") + snap = self._take_volume_snapshot(self._root_volume(vm.id).id) + self._write_marker(a2) + self._write_marker(b2, which="vm2") + + self._stop_vm(vm.id) + Volume.revertToSnapshot(self.apiClient, snap.id) + self._start_vm(vm.id) + self._assert_marker(a1, msg="VM-A after revert") + self._assert_marker(b2, which="vm2", msg="VM-B after VM-A's revert") + + @attr(tags=[TAG], required_hardware=True) + def test_21_concurrent_vm_snapshots_two_vms(self): + """ + VMS-54: VM snapshots on two VMs of the same NFS pool at the same time. + Verifies: both Ready and owned by their VMs; two distinct new FlexVol + snapshots. + """ + vm = self._require_snapshot_vm() + vm2 = self._require_second_vm() + before = self._ontap_snap_names() + results = _run_concurrent( + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_cc_a_%d" % random.randint(0, 99999)), + lambda: VmSnapshot.create(self.apiClient, vm2.id, snapshotmemory="false", + name="ontap_cc_b_%d" % random.randint(0, 99999)), + ) + self._vm_snapshot_ids.extend(s.id for s, _ in results if s is not None) + errors = [e for _, e in results if e is not None] + self.assertFalse(errors, "Concurrent VM snapshots failed: %s" % errors) + snap_a, snap_b = results[0][0], results[1][0] + self.assertEqual(snap_a.state, "Ready") + self.assertEqual(snap_b.state, "Ready") + self.assertEqual(snap_a.virtualmachineid, vm.id) + self.assertEqual(snap_b.virtualmachineid, vm2.id) + self._assert_new_ontap_snaps(before, [snap_a, snap_b], "vm", "concurrent VM snapshots") + + @attr(tags=[TAG], required_hardware=True) + def test_22_concurrent_volume_snapshot_and_revert(self): + """ + Volume snapshot of VM-A's ROOT while VM-B's ROOT reverts, at the same time. + A rejected snapshot or revert is retried on its own. + Verifies: VM-A snapshot BackedUp with its own FlexVol snapshot; VM-B is + back at its snapshot; VM-A data unchanged. + """ + vm = self._require_snapshot_vm() + vm2 = self._require_second_vm() + a1 = new_token("vm-a-current-data") + b1 = new_token("vm-b-before-snapshot") + b2 = new_token("vm-b-after-snapshot") + self._write_marker(a1) + self._write_marker(b1, which="vm2") + snap_b = self._take_volume_snapshot(self._root_volume(vm2.id).id) + self._write_marker(b2, which="vm2") + self._stop_vm(vm2.id) + before = self._ontap_snap_names() + root_a = self._root_volume(vm.id).id + + snap_a = _snapshot_and_revert_with_retry( + lambda: Snapshot.create(self.apiClient, root_a, + account=self.account.name, domainid=self.domain.id), + lambda: Volume.revertToSnapshot(self.apiClient, snap_b.id), + ) + self._volume_snapshot_ids.append(snap_a.id) + self.assertEqual(snap_a.state, "BackedUp") + self._assert_new_ontap_snaps(before, [snap_a], "volume", "VM-A snapshot during VM-B revert") + + self._start_vm(vm2.id) + self._assert_marker(b1, which="vm2", msg="VM-B after revert") + self._assert_marker(a1, msg="VM-A during VM-B revert") + + @attr(tags=[TAG], required_hardware=True) + def test_23_concurrent_vm_snapshot_and_revert(self): + """ + VM snapshot of VM-A while VM-B reverts to its VM snapshot, at the same time. + A rejected snapshot or revert is retried on its own. + Verifies: VM-A snapshot Ready with its own FlexVol snapshot; VM-B is + back at its snapshot; VM-A data unchanged. + """ + vm = self._require_snapshot_vm() + vm2 = self._require_second_vm() + a1 = new_token("vm-a-current-data") + b1 = new_token("vm-b-before-snapshot") + b2 = new_token("vm-b-after-snapshot") + self._write_marker(a1) + self._write_marker(b1, which="vm2") + snap_b = self._take_vm_snapshot(vm2.id) + self._write_marker(b2, which="vm2") + self._stop_vm(vm2.id) + before = self._ontap_snap_names() + + snap_a = _snapshot_and_revert_with_retry( + lambda: VmSnapshot.create(self.apiClient, vm.id, snapshotmemory="false", + name="ontap_ccr_%d" % random.randint(0, 99999)), + lambda: VmSnapshot.revertToSnapshot(self.apiClient, snap_b.id), + ) + self._vm_snapshot_ids.append(snap_a.id) + self.assertEqual(snap_a.state, "Ready") + self._assert_new_ontap_snaps(before, [snap_a], "vm", "VM-A snapshot during VM-B revert") + + self._start_vm(vm2.id) + self._assert_marker(b1, which="vm2", msg="VM-B after revert") + self._assert_marker(a1, msg="VM-A during VM-B revert") + + # ------------------------------------------------------------------ + # Step 24 - Detach ONTAP data volume from the VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_24_detach_volume_from_vm(self): + """ + Detach the ONTAP data volume from the running VM. + Verifies: + - Volume state returns to 'Ready' in CloudStack + - Volume no longer lists the VM's ID + - VM remains 'Running' + - ONTAP: FlexVol remains online + - ONTAP: NFS3 volume data file persists in FlexVol after detach + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = vol.id + + max_timeout = 180 + interval = 10 + deadline = time.time() + max_timeout + last_exc = None + while True: + try: + self.apiClient.detachVolume(cmd) + last_exc = None + break + except Exception as exc: + last_exc = exc + remaining = deadline - time.time() + if remaining <= 0: + break + time.sleep(min(interval, remaining)) + interval = min(interval * 2, max_timeout) + if last_exc is not None: + raise last_exc + + # On ONTAP/NFS shared storage the volume state stays 'Ready' throughout. + # Poll until virtualmachineid is cleared instead. + deadline = time.time() + 120 + vol_vmid = "pending" + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if not vol_vmid: + break + time.sleep(5) + + self.assertIsNone( + vol_vmid, + "Volume should have no virtualmachineid after detach, got '%s'" + % vol_vmid + ) + + # VM must still be Running + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=30) + self.assertEqual(vm_obj.state, "Running", + "VM should still be 'Running' after volume detach") + + # ONTAP FlexVol must remain online + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol not found after detach") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after detach" + ) + + # ONTAP: NFS3 volume data file must still exist after detach — the file + # is only removed when deleteVolume is called, not on detach. + files = self.ontap.list_files_in_volume(pool.name) + vol_file = next((f for f in files if vol.id in f), None) + self.assertIsNotNone( + vol_file, + "Volume data file for '%s' should persist in FlexVol '%s' after " + "detach; files present: %s" % (vol.id, pool.name, files) + ) + + # ------------------------------------------------------------------ + + # ------------------------------------------------------------------ + # Step 25 - Grow detached data volume + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_25_grow_detached_volume(self): + """Grow the materialized qcow2 after it has been detached.""" + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + detached_target = self.__class__.target_volume_size + (1024 ** 3) + self._resize_volume( + self.__class__.volume.id, detached_target // (1024 ** 3)) + + self._poll_volume_size(self.__class__.volume.id, detached_target) + self.assertEqual( + self._get_db_volume_size(self.__class__.volume.id), + detached_target, + "DB size does not match detached NFS3 grow target") + self.assertEqual( + self._get_qcow2_virtual_size( + self.__class__.pool.id, self.__class__.volume_path), + detached_target, + "qcow2 virtual size does not match detached grow target") + self.__class__.target_volume_size = detached_target + + # ------------------------------------------------------------------ + # Step 26 - Destroy the second VM while it still has a VM snapshot + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_26_destroy_vm_with_vm_snapshot(self): + """ + VMS-38: destroy (expunge) the second VM while it still has a VM snapshot. + Verifies: destroy succeeds; the VM snapshot is no longer listed; its + FlexVol snapshot is removed from ONTAP. + """ + self._require_snapshot_vm() + vm2 = self._require_second_vm() + before = self._ontap_snap_names() + snap = self._take_vm_snapshot(vm2.id) + new = self._assert_new_ontap_snaps(before, [snap], "vm", "second VM snapshot") + + cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + cmd.id = vm2.id + cmd.expunge = True + self.apiClient.destroyVirtualMachine(cmd) + self.__class__.vm2 = None + self.__class__.vm2_ssh = None + self._vm_snapshot_ids.remove(snap.id) + + deadline = time.time() + 300 + while time.time() < deadline and self._vm_snapshots_of(vm2.id): + time.sleep(10) + self.assertFalse(self._vm_snapshots_of(vm2.id), + "VM snapshot still listed after destroying its VM") + self._wait_ontap_snaps_gone(new) + + # ------------------------------------------------------------------ + # Step 27 - Destroy VM, delete volume, delete pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_27_destroy_vm_and_cleanup(self): + """ + Destroy the VM, delete the ONTAP data volume, enter maintenance, + then force-delete the pool. + Verifies: + - VM is destroyed/expunged from CloudStack + - Template cache survives VM delete: template_spool_ref still Ready + and the cache file is still on the FlexVol + - Volume is deleted from CloudStack + - ONTAP: NFS3 volume data file removed from FlexVol after deleteVolume + - Pool (still holding the template cache) is removed from CloudStack + - ONTAP: FlexVol is deleted after pool removal + - ONTAP: Export policy is removed after pool removal + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + self._destroy_second_vm() + vm = self.__class__.vm + vol = self.__class__.volume + pool = self.__class__.pool + pool_name = pool.name + + # Stop VM if still running + if vm is not None: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm.id)) + current_state = vms[0].state.lower() if vms else "unknown" + if current_state not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm.id + self.apiClient.stopVirtualMachine(stop_cmd) + self._poll_vm_state(vm.id, "Stopped", timeout=300) + + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + self.apiClient.destroyVirtualMachine(dest_cmd) + self.__class__.vm = None + + spool = self._assert_single_ready_spool_ref( + self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # Delete the ONTAP data volume + if vol is not None: + vol_id = vol.id + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol_id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: NFS3 volume data file must be removed from the FlexVol + # after deleteVolume (CloudStack/libvirt deletes the file from the + # NFS mount as part of the destroy workflow). + files = self.ontap.list_files_in_volume(pool_name) + vol_file = next((f for f in files if vol_id in f), None) + self.assertIsNone( + vol_file, + "Volume data file for '%s' should be gone from FlexVol '%s' " + "after deleteVolume; files still present: %s" + % (vol_id, pool_name, files) + ) + + # Enter maintenance then delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # forced=True: the template cache (template_spool_ref) is still on + # the pool — it is only reclaimed by storage GC or pool deletion. + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, + "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: Export policy must be removed + ep_name = "cs-%s-%s" % (self.svm_name, pool_name) + ep = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + ep, + "ONTAP export policy '%s' still exists after pool deletion" + % ep_name + ) + + +# --------------------------------------------------------------------------- +# Module-level helpers (used in tearDownClass and test helpers) +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + return cmd + + +def _list_vols_cmd(vol_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + return cmd + + +def _run_concurrent(*callables): + """Start all callables at the same moment; return [(result, exception)].""" + results = [(None, None)] * len(callables) + barrier = threading.Barrier(len(callables)) + + def worker(index, fn): + try: + barrier.wait(timeout=60) + results[index] = (fn(), None) + except Exception as exc: + results[index] = (None, exc) + + threads = [threading.Thread(target=worker, args=(i, fn)) + for i, fn in enumerate(callables)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return results + + +def _is_rejected(exc): + """True when CloudStack or ONTAP refused the call because another one was in progress.""" + text = str(exc).lower() + return any(token in text for token in ( + "internal error", + "failed to create snapshot", + "failed to create instance snapshot", + "failed to revert", + "another operation", + "other active", + "try again later", + "in progress", + "busy", + )) + + +def _snapshot_and_revert_with_retry(snapshot_fn, revert_fn, attempts=3, pause=15): + """Run a snapshot and a revert together. + + A call that is rejected is retried on its own. A call that already + succeeded is not repeated. Returns the snapshot from snapshot_fn. + """ + snap = None + reverted = False + last_errors = [] + for attempt in range(1, attempts + 1): + jobs = [] + if snap is None: + jobs.append(("snapshot", snapshot_fn)) + if not reverted: + jobs.append(("revert", revert_fn)) + results = _run_concurrent(*(fn for _, fn in jobs)) + last_errors = [] + retryable = True + for (kind, _), (result, exc) in zip(jobs, results): + if exc is None: + if kind == "snapshot": + snap = result + else: + reverted = True + continue + last_errors.append("%s: %s" % (kind, exc)) + if not _is_rejected(exc): + retryable = False + if snap is not None and reverted: + return snap + if not retryable or attempt == attempts: + break + logger.warning("snapshot/revert rejected, retry %d/%d: %s", + attempt, attempts, last_errors) + time.sleep(pause) + raise AssertionError( + "Snapshot-while-revert failed after %d attempt(s): %s" + % (attempt, last_errors)) + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=120, + interval=5): + """Blocking wait for a VM to reach target_state (used in tearDownClass).""" + deadline = time.time() + timeout + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms and vms[0].state.lower() == target_state.lower(): + return + time.sleep(interval) diff --git a/test/integration/plugins/ontap/nfs3/pool/__init__.py b/test/integration/plugins/ontap/nfs3/pool/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py new file mode 100644 index 000000000000..84e8681796bb --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -0,0 +1,1858 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. + +Tests are numbered test_01 ... test_21 and must run in that order. test_01 +and test_02 are isolated create rejections. test_04 reuses the FlexVol created +by test_03. test_20 and test_21 are isolated delete cases. + +Workflow: + 01 Reject create when no online assigned aggregate has enough free space + 02 Reject create when capacitybytes has invalid values + 03 Create primary storage pool + 04 Reject create when a FlexVol of that name already exists on ONTAP + 05 Increase storage pool capacity to the maximum supported size + 06 Reject a grow beyond the maximum supported size + 07 Safely shrink storage pool capacity + 08 Disable storage pool + 09 Grow and shrink the pool while it is disabled + 10 Enable storage pool + 11 Enter maintenance mode + 12 Grow and shrink the pool while it is in maintenance mode + 13 Cancel maintenance mode + 14 Delete the storage pool + 15 Create fresh pool and allocate a CloudStack volume + 16 Deploy a VM and attach the ONTAP volume to it + 17 Reject shrink below the ONTAP used capacity of the attached volume + 18 Grow and shrink the pool while the VM holds the volume + 19 Detach/destroy the VM, delete volume, then force-delete the pool + 20 Delete an empty pool whose FlexVol was deleted directly on ONTAP + 21 Delete an empty pool whose NFS export policy was deleted on ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v + +Note: Tests 03-19 share class-level state (sequential). test_04 reuses the +FlexVol created by test_03. Running a single test with -m "test_NN" will +invoke setUpClass but the guard assertion will fail immediately if earlier +steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import ( + OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config, + _wait_for_vm_state, log_progress, run_invalid_capacitybytes_rejections, +) + +logger = logging.getLogger("TestOntapNFS3Workflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + DETAIL_VOLUME_UUID = "volumeUUID" + DETAIL_VOLUME_NAME = "volumeName" + DETAIL_DATA_LIF = "dataLIF" + DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + # Maximum FlexVol size on AFF/FAS, raised from 100 TB in ONTAP 9.12.1P2. + # ONTAP writes this as "300 TB" but means binary, hence 1024**4. The + # value has to be hardcoded: no REST field reports the platform ceiling. + # Against an ONTAP older than 9.12.1P2 the real limit is 100 TB and the + # grow step will fail. CloudStack enforces no maximum of its own (the + # plugin validates only a 20 MB floor), so it is ONTAP that refuses + # anything above this. + ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 + ONTAP_GROW_TARGET_SIZE = ONTAP_MAX_VOLUME_SIZE + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wf@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WF", + "username": "ontap_nfs3_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): + + # ---- NFS3-specific shared state ------------------------------------ + pool_ep_name = None # NFS export policy name for pool + pool2_ep_name = None # export policy for pool stashed from tests 03-14 + cluster_host_ips = None + resize_original_size = None + + # VM state (vm/template_id/network_id) lives in OntapTestBase. + _vm_network_name_prefix = "ontap-nfs3-lifecycle-net" + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + super(TestOntapNFS3PrimaryStorageWorkflow, cls).setUpClass() + testclient = super( + TestOntapNFS3PrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Resolve cluster host IPs for export policy rule assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + cls._discover_vm_deploy_resources() + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self, pool_name=None, capacitybytes=None): + """Create a pool; name and capacity default to the suite values.""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + if pool_name is None: + pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + if capacitybytes is None: + capacitybytes = ps["capacitybytes"] + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = capacitybytes + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + # Fallback: plugin typically uses cs-{svmName}-{poolName} + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert that the export policy exists and its rules include each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return # no host IPs registered; skip rule-level check + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return len(vols) > 0 + + def _assert_pool_gone_from_cs(self, pool_id, pool_name): + try: + remaining = list_storage_pools(self.apiClient, id=pool_id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after deletion" % pool_name + ) + + def _assert_ontap_pool_gone(self, pool_name, ep_name): + ontap_vol = self.ontap.get_volume(pool_name) + if ontap_vol is not None: + self.ontap.delete_volume(pool_name) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + if policy is not None: + self.ontap.delete_export_policy(ep_name) + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + def _force_delete_pool_in_maintenance(self, pool, ep_name): + """Force-delete a pool that is already in Maintenance with no volumes.""" + listed = list_storage_pools(self.apiClient, id=pool.id) + if not listed: + return + self._cleanup_kvm_storage_pool_mounts(pool.id) + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as ex: + logger.warning( + "force-delete pool '%s' failed: %s; trying ONTAP direct cleanup", + pool.name, ex + ) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self._assert_ontap_pool_gone(pool.name, ep_name) + + def _delete_volume_then_force_delete_pool(self, pool, vol, ep_name): + """Delete CS volume, enter Maintenance, unmount on KVM, force-delete pool.""" + if vol is not None and self._volume_exists_in_cs(vol.id): + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + err = str(exc).lower() + if "storage pool not found" in err or "storage pool" in err: + logger.warning( + "deleteVolume raised expected NFS3 libvirt error; " + "proceeding: %s", exc + ) + else: + raise + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool '%s' not found before delete" % pool.name) + if listed[0].state != "Maintenance": + self._assert_pool_capacity(pool, "volume-deleted") + self._enter_maintenance(pool.id) + + self._cleanup_kvm_storage_pool_mounts(pool.id) + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as ex: + logger.warning( + "force-delete pool '%s' failed: %s; trying ONTAP direct cleanup", + pool.name, ex + ) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self._assert_ontap_pool_gone(pool.name, ep_name) + + def _throwaway_pool_name(self, suffix): + return "OntapNFS3%s_%d" % (suffix, random.randint(0, 99999)) + + def _cleanup_throwaway_pool(self, pool, flexvol_name=None, ep_name=None): + """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + + Never raises, so a failed assertion in the test body is the error + that surfaces. The KVM unmount happens while the export is still + reachable, i.e. before the FlexVol goes away. + """ + backend_cleanup_safe = pool is None + if pool is not None: + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + backend_cleanup_safe = not listed + if listed: + try: + if listed[0].state != "Maintenance": + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not put pool '%s' into Maintenance: %s", + pool.name, exc, + ) + try: + self._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not safely unmount pool '%s': %s", + pool.name, exc, + ) + return + backend_cleanup_safe = True + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "cleanup: could not delete pool '%s': %s", + pool.name, exc, + ) + if flexvol_name is None: + flexvol_name = pool.name + if backend_cleanup_safe and flexvol_name: + try: + if self.ontap.get_volume(flexvol_name) is not None: + self.ontap.offline_and_delete_volume(flexvol_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete ONTAP FlexVol '%s': %s", + flexvol_name, exc, + ) + if backend_cleanup_safe and ep_name: + try: + if self.ontap.get_export_policy(ep_name) is not None: + self.ontap.delete_export_policy(ep_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete export policy '%s': %s", + ep_name, exc, + ) + if pool is None: + return + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + if not remaining: + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + def _assert_only_original_pool(self, pool): + """Assert the rejected create did not add a second pool of this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool.name) or [] + except CloudstackAPIException: + listed = [] + same_name = [ + item for item in listed if getattr(item, "name", None) == pool.name + ] + self.assertEqual( + [item.id for item in same_name], + [pool.id], + "CloudStack should still list only pool '%s' after the rejected " + "create, found: %s" % (pool.name, same_name), + ) + current = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + current, + "Original pool '%s' disappeared after the rejected create" + % pool.name, + ) + self.assertEqual( + current[0].state, "Up", + "Original pool '%s' should still be Up, got '%s'" + % (pool.name, current[0].state), + ) + + def _warn_if_duplicate_pool(self, duplicate, pool): + """Leave a stray duplicate in place. + + Deleting it would remove the FlexVol and export policy that the + rest of the suite still uses. + """ + if duplicate is None or getattr(duplicate, "id", None) == pool.id: + return + logger.warning( + "Duplicate pool '%s' (id=%s) was created against the shared " + "FlexVol; leaving it in place so cleanup does not delete the " + "pool the rest of the suite uses", + pool.name, duplicate.id, + ) + + def _assert_no_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except CloudstackAPIException: + listed = None + self.assertFalse( + listed, + "CloudStack should hold no pool named '%s' after a rejected " + "create, found: %s" % (pool_name, listed), + ) + + def _sweep_tracked_pool2(self): + """Clean a prior leftover before reusing the shared pool2 slot.""" + if self.__class__.pool2 is None: + return + self._cleanup_throwaway_pool( + self.__class__.pool2, + ep_name=self.__class__.pool2_ep_name, + ) + if self.__class__.pool2 is not None: + self.skipTest("A previously tracked pool could not be cleaned up safely") + + def _create_throwaway_pool(self, suffix): + """Create an isolated pool, park it in pool2, and return it.""" + self._sweep_tracked_pool2() + pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) + self.__class__.pool2 = pool + ep_name = self._get_export_policy_name(pool) + self.__class__.pool2_ep_name = ep_name + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (pool.name, pool.state), + ) + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ) + return pool, ep_name + + # ------------------------------------------------------------------ + # Step 01 - reject create when no aggregate space + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_01_reject_create_when_no_aggregate_space(self): + """ + Ask for 1 GiB more than the largest online aggregate assigned to the + SVM can provide, so no aggregate qualifies and the plugin refuses + before creating anything. + + Verifies: + - createStoragePool raises CloudstackAPIException + - the error names the aggregate shortage rather than some other + failure ('No suitable aggregates') + - no pool is left in CloudStack and no FlexVol on ONTAP + """ + self._sweep_tracked_pool2() + max_free = self.ontap.max_online_aggregate_available_bytes( + self.svm_name + ) + if not max_free: + self.skipTest( + "No online aggregate with reported free space is assigned to " + "SVM '%s'; cannot build an unsatisfiable request" + % self.svm_name + ) + requested = int(max_free) + 1024 ** 3 + if requested > TestData.ONTAP_MAX_VOLUME_SIZE: + self.skipTest( + "Largest aggregate free space (%d B) + 1 GiB exceeds the " + "ONTAP FlexVol maximum (%d B); the request would be refused " + "for the size limit rather than the aggregate shortage" + % (max_free, TestData.ONTAP_MAX_VOLUME_SIZE) + ) + + pool_name = self._throwaway_pool_name("NoSpace") + log_progress( + logger, "info", + "Requesting pool '%s' of %d B; largest online aggregate on SVM " + "'%s' has %d B free (expect reject)", + pool_name, requested, self.svm_name, max_free, + ) + try: + with self.assertRaises(CloudstackAPIException) as caught: + self.__class__.pool2 = self._create_pool( + pool_name=pool_name, capacitybytes=requested + ) + error_text = str(caught.exception) + log_progress( + logger, "info", + "Rejected no-space create for '%s': %s", pool_name, error_text, + ) + self.assertIn( + "No suitable aggregates", error_text, + "Expected the rejection to report 'No suitable aggregates', " + "got: %s" % error_text, + ) + self._assert_no_pool_named(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' was created despite the rejected pool " + "create" % pool_name, + ) + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool_name, + ep_name="cs-%s-%s" % (self.svm_name, pool_name), + ) + + # ------------------------------------------------------------------ + # Step 02 - reject create with invalid capacitybytes values + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_02_reject_create_with_invalid_capacitybytes_values(self): + """ + Negative case: reject pool creation when capacitybytes has invalid + values. + + Verifies, for each invalid value: + - createStoragePool raises CloudstackAPIException + - zero and negative report that capacity must be at least the + ONTAP minimum + - special characters and a decimal are rejected as an invalid + capacitybytes value + - no pool is left in CloudStack and no FlexVol on ONTAP + """ + def cleanup(pool, pool_name): + self._cleanup_throwaway_pool( + pool, + flexvol_name=pool_name, + ep_name="cs-%s-%s" % (self.svm_name, pool_name), + ) + + run_invalid_capacitybytes_rejections( + self, + lambda name, capacity: self._create_pool( + pool_name=name, capacitybytes=capacity + ), + cleanup, + logger, + ) + + # ------------------------------------------------------------------ + # Step 03 - create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_create_primary_storage_pool(self): + """ + Create an NFS3 primary storage pool and verify: + - CloudStack state is Up, type is NetworkFilesystem + - nfsmountopts contains 'vers=3' + - ONTAP: FlexVol exists and is online + - ONTAP: NFS export policy exists with cluster host IP rules + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "NetworkFilesystem", + "Pool type should be 'NetworkFilesystem', got '%s'" % pool.type + ) + + # Verify nfsmountopts via listStoragePools + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone(listed, "listStoragePools returned None for pool %s" % pool.id) + nfs_opts = getattr(listed[0], "nfsmountopts", "") + self.assertIn( + "vers=3", nfs_opts, + "nfsmountopts should contain 'vers=3', got '%s'" % nfs_opts + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with host IP rules + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "pool-created") + + + # ------------------------------------------------------------------ + # Step 04 - reject create when flexvol name exists + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_reject_create_when_flexvol_name_exists(self): + """ + The pool from test_03 already has a FlexVol of that name on ONTAP. + A second createStoragePool with the same name must be rejected, and + the existing pool, FlexVol, and export policy must be left untouched. + + Verifies: + - createStoragePool raises CloudstackAPIException + - CloudStack still lists only the pool from test_03 + - the existing ONTAP FlexVol is still online + - the existing export policy is still present + """ + pool = self.__class__.pool + self.assertIsNotNone(pool, "Pool absent — test_03 must pass first") + pool_name = pool.name + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' from test_03 is missing" % pool_name, + ) + ep_name = self.__class__.pool_ep_name + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' from test_03 is missing" % ep_name, + ) + log_progress( + logger, "info", + "Pool '%s' already owns a FlexVol; requesting another pool of " + "the same name (expect reject)", pool_name, + ) + duplicate = None + try: + with self.assertRaises(CloudstackAPIException) as caught: + duplicate = self._create_pool(pool_name=pool_name) + log_progress( + logger, "info", + "Rejected duplicate-name create for '%s': %s", + pool_name, caught.exception, + ) + self._assert_only_original_pool(pool) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "Existing ONTAP FlexVol '%s' was removed by the failed " + "pool create" % pool_name, + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "Existing ONTAP FlexVol '%s' should still be online, got '%s'" + % (pool_name, ontap_vol.get("state")), + ) + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' was removed by the rejected pool create" + % ep_name, + ) + finally: + self._warn_if_duplicate_pool(duplicate, pool) + + # ------------------------------------------------------------------ + # Step 05 - grow storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_05_grow_storage_pool(self): + """Grow the original pool to 300 TiB and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = TestData.ONTAP_GROW_TARGET_SIZE + self.assertGreater( + requested_size, original_size, + "Grow target %d B must exceed the current pool size %d B" + % (requested_size, original_size) + ) + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, requested_size, timeout=120 + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after capacity increase", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 06 - reject grow above max size + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_06_reject_grow_above_max_size(self): + """ + Request a capacity above the 300 TiB ONTAP FlexVol maximum and verify + it is refused, leaving the pool at the size test_05 grew it to. + ONTAP reports this as an autosize-maximum violation rather than an + oversized-volume error. + Verifies: + - updateStoragePool raises CloudstackAPIException + - the error text is about the size limit, not about some other + failure such as the aggregate running out of space + - CloudStack capacity and ONTAP FlexVol size are both unchanged + - Pool stays 'Up' + - ONTAP: export policy is untouched + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_05 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before over-max resize") + self.assertEqual( + int(getattr(listed[0], "capacitybytes", 0) or 0), + TestData.ONTAP_MAX_VOLUME_SIZE, + "Pool should sit at the 300 TiB maximum - test_05 must pass first" + ) + + # Overshoot by only 1 GiB: at 0.0003% of the limit this probes the + # 300 TiB boundary itself rather than asking for something absurd. + above_max = self._align_flexvol_bytes( + TestData.ONTAP_MAX_VOLUME_SIZE + 1024 ** 3 + ) + log_progress( + logger, "info", + "Requesting %d B for pool '%s' (ONTAP maximum is %d B, " + "expect reject)", + above_max, pool.name, TestData.ONTAP_MAX_VOLUME_SIZE, + ) + + # ONTAP enforces the platform cap through autosize rather than + # rejecting the size outright, reporting "Volume maximum autosize + # must be greater than or equal to the current volume size 300.0TB". + # Matching on that keeps the test from passing when the resize fails + # for an unrelated reason, such as the aggregate running out of space. + self._assert_capacity_update_rejected( + pool.id, above_max, "grow-above-max-flexvol", + pool.id, pool.name, + expected_error=( + "maximum autosize", + "maximum size", + "exceeds", + ), + ) + + after = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(after, "Pool disappeared after rejected over-max grow") + self.assertEqual( + after[0].state, "Up", + "Pool should stay 'Up' after a rejected resize, got '%s'" + % after[0].state + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after a rejected over-max grow", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 07 - shrink storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_07_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_05 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent — test_05 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, target_size, timeout=120 + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after safe shrink", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 08 - disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_08_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + + # ------------------------------------------------------------------ + # Step 09 - grow and shrink storage pool while disabled + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_09_grow_and_shrink_storage_pool_while_disabled(self): + """ + Grow and shrink the disabled pool and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - KVM hosts report each resized NFS capacity + - The pool remains Disabled throughout + - The FlexVol remains online and the export policy is unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before disabled resize") + self.assertEqual( + listed[0].state, "Disabled", + "Pool must be Disabled - test_08 must pass first, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before disabled resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s disabled NFS3 pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Disabled", + "Pool should remain Disabled after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, target, timeout=120 + ) + + if self.__class__.pool_ep_name: + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared during disabled resize", + ) + self._assert_export_policy_has_host_ips( + self.__class__.pool_ep_name + ) + + + # ------------------------------------------------------------------ + # Step 10 - enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_10_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + + # ------------------------------------------------------------------ + # Step 11 - enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_11_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online and export policy unchanged + (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") + + result = self._enter_maintenance(self.__class__.pool.id) + self.assertEqual(result.state, "Maintenance") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during maintenance" + % self.__class__.pool_ep_name + ) + + + # ------------------------------------------------------------------ + # Step 12 - grow and shrink storage pool in maintenance + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_12_grow_and_shrink_storage_pool_in_maintenance(self): + """ + Grow and shrink the pool while it is in maintenance mode and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - The pool remains in Maintenance throughout + - The FlexVol remains online and the export policy is unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before maintenance resize") + self.assertEqual( + listed[0].state, "Maintenance", + "Pool must be in Maintenance - test_11 must pass first, got '%s'" + % listed[0].state, + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before maintenance resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s NFS3 pool '%s' while in Maintenance: target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Maintenance", + "Pool should remain in Maintenance after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + + if self.__class__.pool_ep_name: + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared during maintenance resize", + ) + + + # ------------------------------------------------------------------ + # Step 13 - cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_13_cancel_maintenance_mode(self): + """ + Cancel maintenance mode and verify the pool returns to Up. + + Verifies: + - CloudStack reports pool state Up + - ONTAP: FlexVol is still online + - ONTAP: NFS export policy still present + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_03 must pass first") + + result = self._exit_maintenance(self.__class__.pool.id) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" + % result.state + ) + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after cancel maintenance" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy( + self.__class__.pool_ep_name + ) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after cancel maintenance" + % self.__class__.pool_ep_name + ) + + + # ------------------------------------------------------------------ + # Step 14 - delete pool from maintenance + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_14_delete_pool_from_maintenance(self): + """ + Enter maintenance mode then delete the storage pool. + + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol is deleted + - ONTAP: NFS export policy is deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Pool is Up after test_13 succeeded; must enter Maintenance before deletion. + self._enter_maintenance(pool.id) + + self._delete_pool(pool.id) + self.__class__.pool = None + self.__class__.pool_ep_name = None + self.__class__.resize_original_size = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + + # ------------------------------------------------------------------ + # Step 15 - create volume on pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_15_create_volume_on_pool(self): + """ + Create a new NFS3 pool and allocate a CloudStack data volume. + For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record + only — no new ONTAP object is created). + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: FlexVol is still online and export policy still present + """ + + pool = self._create_pool() + self.__class__.pool = pool + log_progress( + logger, "info", + "test_15: created storage pool name='%s' id=%s state=%s", + pool.name, pool.id, pool.state, + ) + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + log_progress( + logger, "info", + "test_15: created CloudStack volume name='%s' id=%s state=%s " + "on pool='%s' (id=%s) account='%s' domain='%s' — " + "switch to this account in the UI to see the volume", + getattr(vol, "name", "?"), getattr(vol, "id", "?"), + getattr(vol, "state", "?"), pool.name, pool.id, + self.account.name, self.domain.name, + ) + + # ONTAP: FlexVol must still be online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after volume creation" % ep_name + ) + + # Capacity reporting: FlexVol size and reported capacity unchanged after volume allocation + self._assert_pool_capacity(pool, "volume-allocated") + + + # ------------------------------------------------------------------ + # Step 16 - create vm and attach volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_16_create_vm_and_attach_volume(self): + """ + Deploy a VM and attach the ONTAP data volume from test_15 to it. + Verifies: + - VM reaches 'Running' + - attachVolume sets the volume's virtualmachineid (on NFS3 shared + storage CloudStack leaves the volume state at 'Ready', so + virtualmachineid is the reliable attach signal) + - VM is still 'Running' after the attach + - ONTAP: FlexVol stays online and the pool capacity is unchanged + - ONTAP: the volume's data file is materialised in the FlexVol + (NFS3 creates it lazily, during attach rather than createVolume) + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template in the zone - cannot deploy a VM" + ) + self.assertIsNotNone( + self.__class__.service_offering_id, + "No service offering available - check the CloudStack setup" + ) + + pool = self.__class__.pool + vol = self.__class__.volume + + log_progress( + logger, "info", + "Deploying VM for NFS3 pool '%s' (template=%s, offering=%s)", + pool.name, self.__class__.template_id, + self.__class__.service_offering_id, + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=600) + self.assertIsNotNone( + vm_obj, "VM %s never appeared in listVirtualMachines" % vm.id + ) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running' before attach, got '%s'" % vm_obj.state + ) + + log_progress( + logger, "info", + "Attaching volume '%s' to VM '%s'", vol.id, vm.id, + ) + attach_cmd = attachVolumeAPI.attachVolumeCmd() + attach_cmd.id = vol.id + attach_cmd.virtualmachineid = vm.id + self.assertIsNotNone( + self.apiClient.attachVolume(attach_cmd), + "attachVolume returned None" + ) + + vol_vmid = self._poll_volume_attached(vol.id, timeout=180) + self.assertEqual( + vol_vmid, vm.id, + "Volume %s should report virtualmachineid=%s after attach, got %s" + % (vol.id, vm.id, vol_vmid) + ) + + vm_after = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=60) + self.assertEqual( + vm_after.state, "Running", + "VM should still be 'Running' after attach, got '%s'" + % vm_after.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing after volume attach" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach, got '%s'" + % ontap_vol.get("state") + ) + + # NFS3 materialises the volume's backing file during attach, not at + # createVolume time, so it must be present now. + files = self.ontap.list_files_in_volume(pool.name) + self.assertIsNotNone( + next((f for f in files if vol.id in f), None), + "No data file matching volume UUID '%s' in FlexVol '%s' after " + "attach; files present: %s" % (vol.id, pool.name, files) + ) + + self._assert_pool_capacity(pool, "vm-attached") + + + # ------------------------------------------------------------------ + # Step 17 - reject shrink below used capacity + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_17_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the pool below ONTAP used space and verify the + request is rejected while the volume, pool capacity, and NFS export + policy remain unchanged. + + Runs after test_16, so the FlexVol holds the data file of a volume + attached to a running VM. That file is thin, though, and in practice + leaves used space under the 20 MiB FlexVol floor, so incompressible + data is written through the ONTAP files API to get above it and + removed before this test returns. The fill is skipped whenever used + space is already high enough on its own. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_16 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent — test_16 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + export_policy_name = self.__class__.pool_ep_name + self.assertIsNotNone( + export_policy_name, + "Export policy name was not recorded by test_15", + ) + export_policy_before = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_before, + "Export policy '%s' missing before rejected used-capacity shrink" + % export_policy_name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + # The attached volume is thin on both protocols, so it usually does + # not put 20 MiB on disk by itself. This writes incompressible data + # only when used space is still under the FlexVol floor, and removes + # it before returning. + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking NFS3 pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + # ONTAP's wording: "Selected volume size is too small to + # hold the current volume data. New volume size must be at + # least 34.3MB ...". Matching the specific phrase stops an + # unrelated resize failure from counting as a pass. + expected_error=( + "too small to hold the current volume data", + "too small to hold", + "cannot reduce", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + export_policy_after_used = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_after_used, + "Export policy '%s' disappeared after rejected " + "used-capacity shrink" % export_policy_name, + ) + self.assertEqual( + export_policy_after_used.get("name"), + export_policy_before.get("name"), + "Export policy changed during rejected used-capacity shrink", + ) + self._assert_export_policy_has_host_ips(export_policy_name) + finally: + self._delete_filler_file(pool.name, filler_name) + + + # ------------------------------------------------------------------ + # Step 18 - grow and shrink pool with vm attached + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_18_grow_and_shrink_pool_with_vm_attached(self): + """ + Grow the pool and shrink it straight back while the volume is + attached to the running VM from test_16, so resize is exercised + against a pool that is genuinely in use rather than an idle one. + Verifies: + - grow: CloudStack and ONTAP both reach the requested size + - shrink back: both return to the starting size + - the VM stays 'Running' and keeps the volume across both resizes + - ONTAP: FlexVol stays online + - ONTAP: export policy still covers every cluster host IP + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.vm is None: + self.skipTest( + "No VM was deployed (test_16 skipped) - nothing to resize " + "a pool underneath" + ) + + pool = self.__class__.pool + vm = self.__class__.vm + vol = self.__class__.volume + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before the in-use resize") + original_size = int(getattr(listed[0], "capacitybytes", 0) or 0) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Growing in-use pool '%s' from %d B to %d B (VM %s holds the " + "volume)", pool.name, original_size, grow_target, vm.id, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = grow_target + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, grow_target, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing under a VM, got '%s'" + % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, grow_target, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, grow_target, timeout=120 + ) + self._assert_vm_running_with_volume(vm.id, vol.id, "grow-with-vm") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + log_progress( + logger, "info", + "Shrinking in-use pool '%s' back from %d B to %d B", + pool.name, grow_target, original_size, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = original_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, original_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking under a VM, got '%s'" + % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, original_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, original_size, timeout=120 + ) + self._assert_vm_running_with_volume(vm.id, vol.id, "shrink-with-vm") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 19 - delete volume and pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_19_delete_volume_and_pool(self): + """ + Detach the volume and destroy the VM from test_16, then delete the + volume, enter maintenance, and force-delete the pool. + Verifies: + - deleteVolume completes (or expected NFS3 libvirt pool-not-found) + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_15 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_15 must pass first") + + # A volume still attached to a VM cannot be deleted, so unwind test_16 + # first. Both steps are no-ops when test_16 skipped. + self._detach_volume_if_attached(self.__class__.volume.id) + self.__class__._destroy_vm_if_present() + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist before cleanup" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' before cleanup" + ) + + self._delete_volume_then_force_delete_pool(pool, vol, ep_name) + self.__class__.pool = None + self.__class__.volume = None + self.__class__.pool_ep_name = None + + # Clean up pool from test_03-04 (left in Maintenance when test_08/06 skipped). + pool2 = self.__class__.pool2 + if pool2 is not None: + self._force_delete_pool_in_maintenance( + pool2, self.__class__.pool2_ep_name + ) + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + # ------------------------------------------------------------------ + # Step 20 - delete pool with flexvol predeleted + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_20_delete_pool_with_flexvol_predeleted(self): + """ + Delete the backing FlexVol directly on ONTAP, then delete the empty + pool through CloudStack. Deletion must tolerate the missing volume + rather than leaving an undeletable pool behind. + + Verifies: + - deleteStoragePool succeeds with the FlexVol already gone + - the pool is removed from CloudStack + - the export policy is cleaned up as well + """ + pool, ep_name = self._create_throwaway_pool("PreDelVol") + try: + self._enter_maintenance(pool.id) + # Unmount on the KVM hosts first: once the FlexVol is gone the + # export is unreachable and a stale mount can wedge the host. + self._cleanup_kvm_storage_pool_mounts(pool.id) + log_progress( + logger, "info", + "Deleting ONTAP FlexVol '%s' behind CloudStack's back", + pool.name, + ) + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still present after direct deletion" + % pool.name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' reappeared after pool deletion" + % pool.name, + ) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' still exists after pool deletion" + % ep_name, + ) + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool.name, + ep_name=ep_name, + ) + + # ------------------------------------------------------------------ + # Step 21 - delete pool with export policy predeleted + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_21_delete_pool_with_export_policy_predeleted(self): + """ + Delete the NFS export policy directly on ONTAP, then delete the + empty pool through CloudStack. Deletion must tolerate the missing + policy and still remove the FlexVol. + + Verifies: + - deleteStoragePool succeeds with the export policy already gone + - the pool is removed from CloudStack + - ONTAP: the FlexVol is deleted + """ + pool, ep_name = self._create_throwaway_pool("PreDelEp") + try: + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' missing for the fresh throwaway pool" + % ep_name, + ) + self._enter_maintenance(pool.id) + # Dropping the policy revokes the hosts' NFS access, so unmount + # before it disappears. + self._cleanup_kvm_storage_pool_mounts(pool.id) + log_progress( + logger, "info", + "Deleting NFS export policy '%s' behind CloudStack's back", + ep_name, + ) + self.ontap.reassign_volume_export_policy(pool.name) + self.ontap.delete_export_policy(ep_name) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' still present after direct deletion" + % ep_name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still exists after pool deletion" + % pool.name, + ) + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool.name, + ep_name=ep_name, + ) + + + # ------------------------------------------------------------------ diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py new file mode 100644 index 000000000000..5fa74ff843be --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py @@ -0,0 +1,1211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 pool lifecycle tests with a CloudStack volume present throughout. + +Tests are numbered test_01 ... test_07 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 pool and allocate a CloudStack data volume + 02 Disable pool — volume still exists in CloudStack; ONTAP FlexVol online + 03 Re-enable pool — volume still accessible; FlexVol online + 04 Enter maintenance with volume present — Maintenance state; FlexVol online + 05 Cancel maintenance with volume present — pool returns to Up (fix confirmed) + 06 Forced=False delete rejected — pool stays in Maintenance (negative) + 07 Cleanup — cancel maintenance, delete volume, force-delete pool + +Isolated tests (test_08 onwards) run after that workflow and share no state +with it. Each one builds its own pool plus CloudStack volume. Tests 08 and +09 remove one ONTAP object and then force-delete the pool. Tests 10-12 break +only that pool's view on the KVM host: + + 08 FlexVol already removed on ONTAP + 09 Export policy already removed on ONTAP + 10 Libvirt pool for this storage pool inactive on the KVM host + 11 NFS mount for this pool read-only on the KVM host + 12 NFS mount point for this pool missing on the KVM host + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values (protocol=NFS3) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_nfs3_pool_with_volumes.py -v + +Note: Tests 01-05 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. + +Post-run ONTAP cleanup: The suite ends with the pool in Maintenance state (from +test_04) and a CS volume present (test_05 negative test leaves both intact). The +OntapTestBase teardown exits Maintenance via cancelStorageMaintenance (which +transitions CS pool state even though KVM remount fails on NFS3), deletes the +volume, re-enters Maintenance, and force-deletes the pool. In rare cases where +CS pool state does not transition, one orphaned ONTAP FlexVol and export policy +may be left behind. Clean these up manually: + + curl -sk -u : \\ + "https:///api/storage/volumes?name=OntapNFS3WV_*&fields=name,state" + # Offline + DELETE each orphan, then DELETE the matching export policy.""" + +import base64 +import logging +import random +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config + +logger = logging.getLogger("TestOntapNFS3PoolWithVolumes") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wv@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WV", + "username": "ontap_nfs3_wv_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3WV_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PoolWithVolumes(OntapTestBase): + """ + NFS3 pool lifecycle tests with a CloudStack data volume present throughout. + All tests are sequential and share class-level state. + """ + + pool_ep_name = None # NFS export policy name extracted at pool creation + pool2_ep_name = None # export policy of the pool an isolated test builds + + _vol_name_prefix = "OntapNFS3WV" + + @classmethod + def setUpClass(cls): + super(TestOntapNFS3PoolWithVolumes, cls).setUpClass() + testclient = super( + TestOntapNFS3PoolWithVolumes, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3WV_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the NFS export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create pool and allocate a data volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create an NFS3 primary storage pool and allocate a CloudStack data + volume on it. + Verifies: + - Pool state is Up; ONTAP FlexVol is online + - NFS export policy exists + - createVolume returns a volume object (NFS3 data vols are CS records + backed by a qcow2 file inside the FlexVol) + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP after pool creation" % ep_name + ) + + # Allocate a CloudStack data volume on this pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # Capacity reporting: volume allocated on FlexVol + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 02 — Disable pool with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_02_disable_pool_volume_survives(self): + """ + Disable the pool while a CloudStack data volume exists on it: + - Pool should no longer be available for scheduling new CS volumes + - The existing CS volume should continue to exist (not deleted) + - ONTAP: FlexVol remains online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual( + result.state, "Disabled", + "Pool should be 'Disabled', got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool disable" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after pool disable, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after pool disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Re-enable pool with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_03_enable_pool_volume_intact(self): + """ + Re-enable the pool while a CloudStack data volume exists on it: + - Pool state transitions back to Up + - The existing CS volume is still accessible + - ONTAP: FlexVol remains online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after re-enable, got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool re-enable" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool re-enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after pool re-enable, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after pool re-enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_04_enter_maintenance_volume_present(self): + """ + Enter maintenance mode while a CloudStack data volume exists on the pool: + - Pool transitions to Maintenance state + - Existing CS volume remains in CloudStack + - ONTAP: FlexVol stays online (maintenance is a CS-only state) + - ONTAP: export policy is unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + result = self._enter_maintenance(self.__class__.pool.id) + self.assertEqual( + result.state, "Maintenance", + "Pool should be 'Maintenance', got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool entered Maintenance" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after entering Maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' in Maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during Maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance mode with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_05_cancel_maintenance_with_volume(self): + """ + Cancel maintenance mode while a CloudStack data volume exists on the pool: + - cancelStorageMaintenance succeeds (KVM/NFS3 fix confirmed) + - Pool returns to Up state + - Existing CS volume is still present in CloudStack + - ONTAP: FlexVol is still online + - ONTAP: NFS export policy is unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + result = self._exit_maintenance(self.__class__.pool.id) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after cancel maintenance" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after cancel maintenance" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after cancel maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 06 — forced=False delete rejected when volume present (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_06_forced_false_delete_rejected(self): + """ + Enter maintenance mode then attempt to delete the pool (forced=False) + while a CloudStack volume still exists on it. The operation must be + rejected: + - CloudstackAPIException is raised with an appropriate error + - Pool remains in Maintenance state + - CS volume still exists + - ONTAP: FlexVol and export policy are unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + # Pool is Up after test_05 (cancel maintenance); re-enter Maintenance + # before attempting the delete so it reaches the forced=False gate. + self._enter_maintenance(self.__class__.pool.id) + + with self.assertRaises(CloudstackAPIException, + msg="deleteStoragePool(forced=False) with a live " + "volume should raise CloudstackAPIException"): + self._delete_pool(self.__class__.pool.id, forced=False) + + # Pool must still be in Maintenance (not deleted) + try: + remaining = list_storage_pools( + self.apiClient, id=self.__class__.pool.id) + except CloudstackAPIException: + remaining = None + self.assertTrue( + remaining, + "Pool was deleted even though forced=False delete should have failed" + ) + + # Volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume was deleted even though pool deletion was rejected" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol should still exist after rejected pool deletion" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after rejected deletion" + ) + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_07_force_delete_pool_and_cleanup(self): + """ + Explicit cleanup after the test_06 negative test. + + The pool is in Maintenance with a CS volume still present. + Cleanup sequence: + 1. Try cancelStorageMaintenance. + 2. If still in Maintenance: try updateStoragePool(enabled=True) as a + fallback exit path (works on KVM/NFS3 even when cancel fails). + 3. Once pool exits Maintenance: delete volume, re-enter Maintenance. + 4. Force-delete the pool. + 5. Verify CS pool is gone. + 6. Verify ONTAP FlexVol and export policy are removed. + If the CS force-delete fails (volume couldn't be removed), the + ONTAP FlexVol and export policy are cleaned up directly via REST + so the storage array is never left with orphans. The CS pool + record is left for tearDownClass in that edge case only. + """ + pool = self.__class__.pool + vol = self.__class__.volume + self.assertIsNotNone(pool, "No pool from test_06 to clean up") + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Step 1: Try cancelStorageMaintenance + pool_state = "Maintenance" + try: + cm = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cm.id = pool.id + self.apiClient.cancelStorageMaintenance(cm) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state != "Maintenance": + pool_state = ps[0].state + break + time.sleep(5) + except Exception: + pass # falls through to step 2 + + # Step 2: If still in Maintenance, try updateStoragePool(enabled=True). + # On KVM/NFS3 this succeeds in moving the pool to Disabled/Up even + # when cancelStorageMaintenance fails. + if pool_state == "Maintenance": + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + self.apiClient.updateStoragePool(ec) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state != "Maintenance": + pool_state = ps[0].state + break + time.sleep(5) + except Exception: + pass + + # Step 3: If pool exited Maintenance, delete the CS volume and + # re-enter Maintenance so the pool can be force-deleted. + if pool_state != "Maintenance" and vol is not None: + if self._volume_exists_in_cs(vol.id): + try: + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + vol = None + except Exception: + pass + else: + self.__class__.volume = None + vol = None + try: + self._enter_maintenance(pool.id, timeout=60) + pool_state = "Maintenance" + except Exception: + pass + + # Step 4: Force-delete the CS pool. + cs_pool_deleted = False + if vol is None or not self._volume_exists_in_cs(vol.id): + # Volume is gone — safe to force-delete + self.__class__.volume = None + vol = None + try: + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + cs_pool_deleted = True + except CloudstackAPIException: + pass + + # Step 5: Assert CS pool is gone (only when deletion was attempted) + if cs_pool_deleted: + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' should have been deleted with forced=True" % pool_name + ) + + # Step 6: ONTAP FlexVol must be gone. + # If the CS pool could not be deleted (volume still present — an + # NFS3/KVM platform edge case), delete the ONTAP FlexVol and export + # policy directly via REST so the storage array is always clean. + # The orphaned CS pool record is left for tearDownClass. + ontap_vol = self.ontap.get_volume(pool_name) + if ontap_vol is not None: + self.ontap.delete_volume(pool_name) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' should be gone after cleanup" % pool_name + ) + + policy = self.ontap.get_export_policy(ep_name) + if policy is not None: + self.ontap.delete_export_policy(ep_name) + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "NFS export policy '%s' should be removed after cleanup" % ep_name + ) + + # ================================================================== + # Isolated tests — appended after the sequential workflow above. + # + # Each one creates its own pool and CloudStack volume in the pool2 / + # volume2 slots (which OntapTestBase.tearDownClass also sweeps), runs a + # single scenario, and cleans up in a finally block. They never reuse a + # pool destroyed by another test. + # ================================================================== + + def _create_isolated_pool_with_volume(self, label): + """Build a fresh pool plus CS volume for one isolated scenario. + + Returns ``(pool, volume, export_policy_name)``. Any pool a previous + isolated test could not clean up is swept first so its ONTAP objects + are never orphaned by the overwrite of the pool2 slot. + """ + if self.__class__.pool2 is not None: + self._cleanup_isolated_pool( + self.__class__.pool2, self.__class__.pool2_ep_name, + "leftover-from-previous-isolated-test" + ) + + pool = self._create_pool() + self.__class__.pool2 = pool + ep_name = self._get_export_policy_name(pool) + self.__class__.pool2_ep_name = ep_name + logger.info("[%s] created isolated pool '%s' (ep '%s')", + label, pool.name, ep_name) + + self.assertEqual( + pool.state, "Up", + "[%s] new pool state should be 'Up', got '%s'" % (label, pool.state) + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol not found for new pool '%s'" % (label, pool.name) + ) + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' not found for new pool '%s'" + % (label, ep_name, pool.name) + ) + + vol = self._create_volume(pool.id) + self.__class__.volume2 = vol + self.assertIsNotNone(vol, "[%s] createVolume returned None" % label) + return pool, vol, ep_name + + def _assert_pool_absent(self, pool, label, delete_error=None): + """Assert CloudStack no longer lists the pool.""" + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "[%s] pool '%s' is still listed after deleteStoragePool(forced=True)%s" + % (label, pool.name, + "; the API raised: %s" % delete_error if delete_error else "") + ) + + def _cleanup_isolated_pool(self, pool, ep_name, label): + """Best-effort teardown of one isolated pool and its ONTAP objects.""" + if pool is None: + return + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + listed = None + backend_cleanup_safe = not listed + if listed: + self._remove_cs_volume(pool, self.__class__.volume2, label) + try: + listed = list_storage_pools(self.apiClient, id=pool.id) or listed + if listed[0].state != "Maintenance": + self._enter_maintenance(pool.id) + self._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as exc: + logger.warning("[%s] could not safely unmount pool '%s': %s", + label, pool.name, exc) + return + backend_cleanup_safe = True + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning("[%s] could not force-delete pool '%s': %s", + label, pool.name, exc) + if not backend_cleanup_safe: + return + try: + self.ontap.offline_and_delete_volume(pool.name) + except Exception as exc: + logger.warning("[%s] ONTAP FlexVol cleanup for '%s' failed: %s", + label, pool.name, exc) + if ep_name: + try: + self.ontap.delete_export_policy(ep_name) + except Exception as exc: + logger.warning( + "[%s] ONTAP export policy cleanup for '%s' failed: %s", + label, ep_name, exc) + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + listed = None + if not listed: + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + # ------------------------------------------------------------------ + # Step 08 — FlexVol deleted on ONTAP before the pool delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_08_delete_pool_with_volume_flexvol_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after its + ONTAP FlexVol has been removed behind CloudStack's back. + + Uses its own pool and volume, so the pool is known to be healthy up to + the point the FlexVol is destroyed. Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing FlexVol + - the CloudStack pool record is removed + - the leftover CS volume record can still be cleaned up + """ + label = "flexvol-missing" + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool.id) + + # Unmount on every KVM host while the NFS export is still + # reachable, before the FlexVol is destroyed underneath it. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should be gone before the pool delete" + % (label, pool.name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + try: + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning("[%s] could not enter maintenance on '%s': %s", + label, pool.name, exc) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate a missing " + "FlexVol, but raised: %s" % (label, delete_error) + ) + + self._purge_cs_volume_record(vol, label) + finally: + self._cleanup_isolated_pool(pool, ep_name, label) + + # ------------------------------------------------------------------ + # Step 09 — Export policy deleted on ONTAP before the delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_09_delete_pool_with_volume_export_policy_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after its NFS + export policy has been removed behind CloudStack's back. + + Uses its own pool and volume. Unlike test_08 the FlexVol is still + present, so the plugin is expected to remove it as part of the delete. + Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing export policy + - the CloudStack pool record is removed + - the ONTAP FlexVol is deleted despite the missing policy + """ + label = "export-policy-missing" + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool.id) + + # Unmount before the export policy goes away, otherwise the KVM + # hosts are left holding a mount they can no longer reach. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self.ontap.reassign_volume_export_policy(pool.name) + self.ontap.delete_export_policy(ep_name) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' should be gone before the pool delete" + % (label, ep_name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + try: + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning("[%s] could not enter maintenance on '%s': %s", + label, pool.name, exc) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate a missing " + "export policy, but raised: %s" % (label, delete_error) + ) + + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should have been deleted with the pool" + % (label, pool.name) + ) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' reappeared during the pool delete" + % (label, ep_name) + ) + + self._purge_cs_volume_record(vol, label) + finally: + self._cleanup_isolated_pool(pool, ep_name, label) + + def _libvirt_pool_present(self, host_ip, pool_uuid): + output = self._kvm_run( + host_ip, + "virsh pool-info '%s' >/dev/null 2>&1 && echo PRESENT || true" + % pool_uuid + ) + return any(line.strip() == "PRESENT" for line in output) + + def _restore_libvirt_pool(self, host_ip, pool_uuid, xml_path, undefined): + quoted = pool_uuid.replace("'", "'\"'\"'") + quoted_xml = xml_path.replace("'", "'\"'\"'") + if undefined and xml_path: + self._kvm_run( + host_ip, + "virsh pool-define '%s' >/dev/null 2>&1 || true" % quoted_xml + ) + cleanup = "rm -f '%s'" % quoted_xml if xml_path else "true" + self._kvm_run( + host_ip, + "virsh pool-start '%s' >/dev/null 2>&1 || true; %s" + % (quoted, cleanup) + ) + + # ------------------------------------------------------------------ + # Step 10 — Test libvirt pool made inactive + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_10_libvirt_pool_inactive(self): + """ + Destroy only the isolated pool's libvirt definition so it is inactive, + then enter maintenance. The libvirt pool is started again before + CloudStack cleanup. + """ + label = "libvirt-pool-inactive" + host_ip = self._kvm_host_ip() + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + xml_path = "/tmp/cs-ontap-%s.xml" % pool.id + destroyed = False + temporarily_defined = False + try: + if not self._libvirt_pool_present(host_ip, pool.id): + self.skipTest( + "Libvirt has no storage pool named '%s' on %s" + % (pool.id, host_ip) + ) + self._kvm_run( + host_ip, + "virsh pool-dumpxml '%s' > '%s' && " + "virsh pool-destroy '%s' >/dev/null 2>&1 || true" + % (pool.id, xml_path, pool.id) + ) + destroyed = True + info = self._kvm_run( + host_ip, "virsh pool-info '%s' 2>/dev/null || true" % pool.id + ) + if not any(line.strip() for line in info): + self._kvm_run( + host_ip, + "virsh pool-define '%s' >/dev/null" % xml_path + ) + temporarily_defined = True + info = self._kvm_run( + host_ip, + "virsh pool-info '%s' 2>/dev/null || true" % pool.id + ) + self.assertTrue( + any("inactive" in line.lower() for line in info), + "[%s] libvirt pool was not inactive: %s" % (label, info) + ) + + self._enter_maintenance(pool.id) + self.assertTrue( + self._volume_exists_in_cs(vol.id), + "[%s] volume disappeared after maintenance" % label + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "[%s] FlexVol disappeared" % label) + self.assertEqual(ontap_vol.get("state"), "online") + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy disappeared" % label + ) + finally: + if destroyed: + try: + self._kvm_run( + host_ip, + "virsh pool-start '%s' >/dev/null 2>&1 || true; " + "%s; rm -f '%s'" + % ( + pool.id, + "virsh pool-undefine '%s' >/dev/null 2>&1 || true" + % pool.id if temporarily_defined else "true", + xml_path, + ) + ) + except Exception as exc: + logger.warning( + "[%s] could not restart libvirt pool: %s", label, exc + ) + self._cleanup_isolated_pool(pool, ep_name, label) + + # ------------------------------------------------------------------ + # Step 11 — Test NFS mount made read-only + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_11_nfs_mount_read_only(self): + """ + Remount only the isolated test pool read-only, verify writes fail, + then verify the pool can still enter maintenance. + """ + label = "nfs-mount-read-only" + host_ip = self._kvm_host_ip() + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + mount_path = "/mnt/%s" % pool.id + quoted_mount = mount_path.replace("'", "'\"'\"'") + remounted_read_only = False + try: + mounted = self._kvm_run( + host_ip, + "mountpoint -q '%s' && echo MOUNTED || true" % quoted_mount + ) + self.assertTrue( + any(line.strip() == "MOUNTED" for line in mounted), + "[%s] %s was not mounted before the test" + % (label, mount_path) + ) + + self._kvm_run( + host_ip, "mount -o remount,ro '%s'" % quoted_mount + ) + remounted_read_only = True + options = self._kvm_run( + host_ip, + "findmnt -n -o OPTIONS --target '%s'" % quoted_mount + ) + self.assertTrue( + any( + "ro" in [item.strip() for item in line.split(",")] + for line in options + ), + "[%s] %s was not remounted read-only: %s" + % (label, mount_path, options) + ) + write_probe = self._kvm_run( + host_ip, + "if touch '%s/.cloudstack-ro-probe' 2>/dev/null; then " + "rm -f '%s/.cloudstack-ro-probe'; echo WRITABLE; " + "else echo READ_ONLY; fi" % (quoted_mount, quoted_mount) + ) + self.assertTrue( + any(line.strip() == "READ_ONLY" for line in write_probe), + "[%s] writes unexpectedly succeeded on %s" + % (label, mount_path) + ) + + self._enter_maintenance(pool.id) + self.assertTrue(self._volume_exists_in_cs(vol.id)) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "[%s] FlexVol disappeared" % label) + self.assertEqual(ontap_vol.get("state"), "online") + self.assertIsNotNone(self.ontap.get_export_policy(ep_name)) + finally: + if remounted_read_only: + try: + self._kvm_run( + host_ip, + "if mountpoint -q '%s'; then " + "mount -o remount,rw '%s'; fi" + % (quoted_mount, quoted_mount) + ) + except Exception as exc: + logger.warning( + "[%s] could not restore read-write mount: %s", + label, exc + ) + self._cleanup_isolated_pool(pool, ep_name, label) + + # ------------------------------------------------------------------ + # Step 12 — Test NFS mount point deleted + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_12_nfs_mount_point_deleted(self): + """ + Unmount only the isolated test pool and delete its mount-point + directory, then verify the pool can still enter maintenance. + """ + label = "nfs-mount-point-deleted" + host_ip = self._kvm_host_ip() + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + mount_path = "/mnt/%s" % pool.id + quoted_mount = mount_path.replace("'", "'\"'\"'") + try: + mounted = self._kvm_run( + host_ip, + "mountpoint -q '%s' && echo MOUNTED || true" % quoted_mount + ) + self.assertTrue( + any(line.strip() == "MOUNTED" for line in mounted), + "[%s] %s was not mounted before the test" + % (label, mount_path) + ) + + removed = self._kvm_run( + host_ip, + "umount -f -l '%s' && rmdir '%s'; " + "if [ ! -e '%s' ]; then echo REMOVED; fi" + % (quoted_mount, quoted_mount, quoted_mount) + ) + self.assertTrue( + any(line.strip() == "REMOVED" for line in removed), + "[%s] could not remove mount point %s" % (label, mount_path) + ) + + self._enter_maintenance(pool.id) + self.assertTrue(self._volume_exists_in_cs(vol.id)) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "[%s] FlexVol disappeared" % label) + self.assertEqual(ontap_vol.get("state"), "online") + self.assertIsNotNone(self.ontap.get_export_policy(ep_name)) + finally: + self._cleanup_isolated_pool(pool, ep_name, label) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py new file mode 100644 index 000000000000..cc75106d9406 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py @@ -0,0 +1,502 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Zone-scoped primary storage lifecycle tests for NetApp ONTAP (NFS3). + +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). CloudStack calls +OntapPrimaryDatastoreLifecycle.attachZone(), which connects all eligible KVM +hosts in the zone to the pool and creates an NFS export policy covering their +IPs. + +Test order — sequential workflow that must run in order: + 01 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; + export policy has all cluster host IPs + 02 Grow and shrink zone-scoped pool — CloudStack, ONTAP, and KVM agree + 03 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 04 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 05 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM hosts registered in the zone + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values (protocol=NFS3) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py -v + +Note: Tests share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import ( + OntapRestClient, + OntapTestBase, + _parse_pool_details, + get_datacenter_config, + log_progress, +) + +logger = logging.getLogger("TestOntapZoneScopedPool") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-zone@test.com", + "firstname": "ONTAP", + "lastname": "Zone", + "username": "ontap_zone_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapZoneNFS3_%d" % random.randint(0, 9999), + TestData.scope: "ZONE", + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapZoneScopedPool(OntapTestBase): + + # ---- zone-pool-specific shared state -------------------------------- + pool_ep_name = None + cluster_host_ips = None + + _vol_name_prefix = "OntapZoneVol" + + @classmethod + def setUpClass(cls): + super(TestOntapZoneScopedPool, cls).setUpClass() + testclient = super( + TestOntapZoneScopedPool, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Collect host IPs for export policy assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_zone_pool(self): + """Create a zone-scoped NFS3 pool (no clusterid / podid).""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapZoneNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + # Intentionally omit clusterid and podid — zone-scoped pool + cmd.scope = "ZONE" + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert export policy exists and contains each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_01_create_zone_scoped_pool(self): + """ + Create a zone-scoped NFS3 primary storage pool (no clusterid/podid). + CloudStack calls attachZone(), which connects all eligible KVM hosts + in the zone and creates an NFS export policy. + Verifies: + - pool.state is Up + - ONTAP: FlexVol is online + - ONTAP: export policy exists and contains cluster host IPs + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_zone_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with cluster host IPs + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # ------------------------------------------------------------------ + # Step 02 — Grow and shrink zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_02_grow_and_shrink_zone_scoped_pool(self): + """Grow the zone-scoped pool and shrink it back to its original size.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Zone-scoped pool missing before resize") + self.assertEqual( + listed[0].state, "Up", + "Zone-scoped pool must be Up before resize, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before zone-scoped resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s zone-scoped NFS3 pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Up", + "Zone-scoped pool should remain Up after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, target, timeout=120 + ) + + self._assert_export_policy_has_host_ips( + self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_disable_zone_scoped_pool(self): + """ + Disable the zone-scoped pool. + Verifies: + - pool.state is Disabled + - ONTAP: FlexVol still online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_02 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable" + ) + + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_04_enable_zone_scoped_pool(self): + """ + Re-enable the zone-scoped pool. + Verifies: + - pool.state is Up + - ONTAP: FlexVol still online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable" + ) + + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 — Delete zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_05_delete_zone_scoped_pool(self): + """ + Enter maintenance then delete the zone-scoped pool. + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_04 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + self._enter_maintenance(pool.id) + + # Unmount the NFS on each KVM host BEFORE deleteStoragePool removes + # the ONTAP export. Without this, the mount becomes stale and + # KVMHAMonitor will fail its heartbeat 5 times then reboot the host + # via `echo b > /proc/sysrq-trigger`. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Class-level teardown + # ------------------------------------------------------------------ + + @classmethod + def tearDownClass(cls): + """ + Clean up any lingering zone-scoped pool NFS mounts on KVM hosts + before the base-class teardown deletes the ONTAP FlexVol. Without + this, a failed test_05 leaves a stale NFS mount that will cause + KVMHAMonitor to reboot the host. + """ + for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: + try: + cls._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as e: + logger.warning( + "tearDownClass: KVM NFS cleanup failed for pool %s: %s" + % (pool.id, e) + ) + super(TestOntapZoneScopedPool, cls).tearDownClass() diff --git a/test/integration/plugins/ontap/nfs3/template/__init__.py b/test/integration/plugins/ontap/nfs3/template/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py b/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py new file mode 100644 index 000000000000..184b8ac84def --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 primary template-cache negative / boundary suite (Marvin). + +Independent of the sequential happy-path suite. + + 01 Tag mismatch — no spool_ref / cache on ONTAP pool + 02 Undersized pool — deploy fails + 03 Out-of-band cache delete — reuse deploy fails + +Running: + bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache_negative +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_negative_workflow import ( + OntapTemplateCacheNegativeWorkflow, +) + + +class TestOntapNfs3TemplateCacheNegative(OntapTemplateCacheNegativeWorkflow): + PROTOCOL = "NFS3" + NOSE_TAG = "nfs3_template_cache_negative" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapNfsTmplNeg" + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_01_tag_mismatch_does_not_seed_cache(self): + self.step_01_tag_mismatch_does_not_seed_cache() + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_02_undersized_pool_deploy_fails(self): + self.step_02_undersized_pool_deploy_fails() + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_03_deleted_cache_blocks_reuse(self): + self.step_03_deleted_cache_blocks_reuse() diff --git a/test/integration/plugins/ontap/nfs3/volume/__init__.py b/test/integration/plugins/ontap/nfs3/volume/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/volume/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py b/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py new file mode 100644 index 000000000000..332c2fd1b68c --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py @@ -0,0 +1,471 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 data volume +lifecycle (volume create / delete / negative-delete / force-delete). + +For NFS3, a CloudStack data volume is a metadata record only — no new ONTAP +object is created per volume (the pool's single FlexVol serves all volumes). +Volume deletion likewise removes the CloudStack record while leaving the +FlexVol intact. + +Tests are numbered test_01 ... test_05 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 primary storage pool and allocate a CloudStack data volume + 02 Delete the volume — CloudStack record removed; FlexVol stays online + 03 Recreate volume — CS record back; FlexVol stays online (setup for 04-05) + 04 Put pool in Maintenance; attempt forced=False deleteStoragePool — must be + rejected because volumes exist; pool stays in Maintenance + 05 Delete volume from Maintenance; forced=True deleteStoragePool — FlexVol + and export policy are removed from ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has NFS configured + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/volume/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config + +logger = logging.getLogger("TestOntapNFS3VolumeLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-vol@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-Vol", + "username": "ontap_nfs3_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3Vol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "NFS3", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3VolumeLifecycle(OntapTestBase): + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + super(TestOntapNFS3VolumeLifecycle, cls).setUpClass() + testclient = super( + TestOntapNFS3VolumeLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3Vol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + # ------------------------------------------------------------------ + # Step 01 - Create pool (infrastructure) and allocate a volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create a new NFS3 pool and allocate a CloudStack data volume on it. + For NFS3, volume creation is a CloudStack metadata record only — no + new ONTAP object is created (the pool's FlexVol serves all volumes). + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: FlexVol remains online after volume allocation + - ONTAP: export policy still present + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must remain online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should exist after volume creation" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 02 - Delete volume; FlexVol must remain untouched + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_02_delete_volume(self): + """ + Delete the volume created in test_01. + For NFS3, volume deletion removes only the CloudStack record. + Verifies: + - deleteVolume completes without error + - ONTAP: FlexVol is still online (unaffected by volume deletion) + - ONTAP: export policy still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_01 must pass first") + + pool = self.__class__.pool + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion, " + "got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after volume deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 - Recreate volume for negative delete tests + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_03_recreate_volume_for_delete_tests(self): + """ + Recreate a volume on the existing pool (setup for tests 04-05). + Verifies: + - volume created successfully + - ONTAP: FlexVol still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume re-creation" + % self.__class__.pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after volume re-creation" + ) + + # ------------------------------------------------------------------ + # Step 04 - Forced=False delete with live volume must fail + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_04_forced_false_delete_with_volume_fails(self): + """ + Put pool in Maintenance then attempt deleteStoragePool(forced=False). + With a live volume present CloudStack must reject the request. + Verifies: + - Exception is raised (CloudStack rejects the delete) + - Pool is still listed in CloudStack (in Maintenance state) + - ONTAP: FlexVol still exists and is online + - ONTAP: export policy still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Enter maintenance mode + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise exception because volumes exist + with self.assertRaises(Exception): + self._delete_pool(pool.id, forced=False) + + # Pool must still be listed in CloudStack + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + listed, + "Pool should still exist in CloudStack after failed forced=False delete" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after failed delete" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after failed delete" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 - Delete volume then force-delete pool from Maintenance + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_05_delete_volume_and_force_delete_pool(self): + """ + Delete the live volume then force-delete the pool while it is still + in Maintenance state (pool is in Maintenance from test_04). + Verifies: + - Volume can be deleted while pool is in Maintenance + - Pool is removed from CloudStack using forced=True from Maintenance + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone( + self.__class__.pool, + "Pool absent - test_04 must not have cleaned up the pool" + ) + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + # Delete the volume first (pool is in Maintenance — volume deletion is + # allowed). For NFS3 a forced=False delete attempt in test_04 may have + # already destroyed the libvirt NFS pool representation on the host; + # if so deleteVolume raises "Storage pool not found". The CS metadata + # record will be cleaned up by the subsequent force-delete of the pool, + # so we treat that specific error as a no-op here. + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + if "Storage pool not found" in str(exc) or "storage pool" in str(exc).lower(): + logger.warning( + "deleteVolume raised expected NFS3 libvirt pool-not-found " + "error; proceeding to force-delete pool: %s", exc + ) + else: + raise + self.__class__.volume = None + + # Force-delete the pool from Maintenance (no live volumes remaining) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after force deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) diff --git a/test/integration/plugins/ontap/nose_compat.py b/test/integration/plugins/ontap/nose_compat.py new file mode 100644 index 000000000000..6d3fc1a931f3 --- /dev/null +++ b/test/integration/plugins/ontap/nose_compat.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run legacy nose on Python versions where collections.Callable was removed.""" + +import collections +import collections.abc +import runpy + + +if not hasattr(collections, "Callable"): + collections.Callable = collections.abc.Callable + +runpy.run_module("nose", run_name="__main__", alter_sys=True) diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg new file mode 100644 index 000000000000..47879ae4d340 --- /dev/null +++ b/test/integration/plugins/ontap/ontap.cfg @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{ + "zones": [ + { + "name": "Zone1", + "networktype": "Advanced", + "dns1": "8.8.8.8", + "dns2": "8.8.4.4", + "internaldns1": "10.192.0.250", + "internaldns2": "10.193.0.250", + "localstorageenabled": true, + "guestcidraddress": "10.1.1.0/24", + "guestVlanRange": "100-300", + "publicIpRange": { + "gateway": "10.193.56.1", + "netmask": "255.255.255.128", + "startip": "10.193.56.100", + "endip": "10.193.56.109", + "vlan": "untagged" + }, + "secondaryStorages": [ + { + "name": "Secondary1", + "provider": "NFS", + "url": "nfs://10.193.56.62/export/secondary" + } + ], + "pods": [ + { + "name": "Pod1", + "gateway": "10.193.56.1", + "netmask": "255.255.255.128", + "startip": "10.193.56.80", + "endip": "10.193.56.89", + "clusters": [ + { + "clustername": "Cluster1", + "clustertype": "CloudManaged", + "hypervisor": "KVM", + "primaryStorages": [ + { + "name": "Primary1", + "scope": "Cluster", + "url": "nfs://10.193.56.62/export/primary", + "provider": "DefaultPrimary", + "tags": "defaultPrim" + } + ], + "hosts": [ + { + "url": "http://10.193.56.62", + "username": "root", + "password": "<>", + "hosttags": "kvmHost" + } + ] + } + ] + } + ] + } + ], + "dbSvr": { + "dbSvr": "10.193.56.62", + "passwd": "", + "db": "cloud", + "port": 3306, + "user": "root" + }, + "logger": { + "LogFolderPath": "/tmp/" + }, + "mgtSvr": [ + { + "mgtSvrIp": "10.193.56.62", + "port": 8096, + "user": "admin", + "passwd": "password", + "hypervisor": "kvm" + } + ], + "ontap": { + "storageIP": "10.196.35.203", + "svmName": "vs0", + "username": "admin", + "password": "<>" + }, + "storagePool": { + "storagePoolScope": "CLUSTER", + "storagePoolProvider": "NetApp ONTAP", + "capacitybytes": null, + "protocols": { + "iscsi": { + "enabled": true, + "storagePoolTags": "ontap-iscsi" + }, + "nfs3": { + "enabled": true, + "storagePoolTags": "ontap-nfs3" + } + } + }, + "cloudstack": { + "zoneName": "Zone1", + "clusterName": null, + "domainName": "ROOT", + "templateName": "CentOS 5.5(64-bit) no GUI (KVM)", + "templateUsername": "root", + "templatePassword": "<>", + "systemVmTimeoutSec": 3600, + "templateReadyTimeoutSec": 3600, + "pollIntervalSec": 60 + } +} diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py new file mode 100644 index 000000000000..e528aba2b25f --- /dev/null +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -0,0 +1,2243 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared base class and helper utilities for NetApp ONTAP Marvin integration tests. + +Provides: + OntapRestClient - thin wrapper around the ONTAP REST API (NFS + iSCSI methods) + _parse_pool_details - converts a StoragePool details attribute to a plain dict + OntapTestBase - base cloudstackTestCase with common tearDownClass, + _poll_pool_state, _poll_vm_state, _create_volume, + _delete_pool and the template-cache assertions (every + ONTAP ROOT volume on KVM is cloned from the primary + template cache) +""" + +import logging +import os +import random +import re +import requests +import sys +import time +import urllib3 +from unittest import SkipTest +from urllib.parse import quote, urlparse + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createNetwork as createNetworkAPI, + createVolume as createVolumeAPI, + deleteNetwork as deleteNetworkAPI, + deleteStoragePool as deleteStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + destroyVolume as destroyVolumeAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listDiskOfferings as listDiskOfferingsAPI, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + stopVirtualMachine as stopVirtualMachineAPI, + resizeVolume as resizeVolumeAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackAPI import listHosts as listHostsAPI +from marvin.cloudstackException import CloudstackAPIException +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.jsonHelper import jsonDump +from marvin.lib.base import Account, DiskOffering +from marvin.sshClient import SshClient +from marvin.lib.common import ( + get_domain, get_zone, list_clusters, list_storage_pools, +) +from marvin.lib.utils import cleanup_resources + +from helpers import kvm_host +from helpers import template_cache_util as tcu + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logger = logging.getLogger("OntapTestBase") + + +def configure_console_logging(log, level=logging.INFO): + """Send INFO/WARNING/ERROR from *log* to stdout for live test-run visibility.""" + if any(isinstance(h, logging.StreamHandler) for h in log.handlers): + return + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter( + "%(asctime)s %(levelname)s [%(name)s] %(message)s", + datefmt="%H:%M:%S", + )) + handler.setLevel(level) + log.addHandler(handler) + if log.level == logging.NOTSET or log.level > level: + log.setLevel(level) + log.propagate = False + + +def enable_live_logging(test_cls): + """Attach stdout handlers to OntapTestBase and the test module logger.""" + configure_console_logging(logger) + if test_cls is not None: + mod = sys.modules.get(test_cls.__module__) + if mod is not None: + mod_logger = getattr(mod, "logger", None) + if mod_logger is not None: + configure_console_logging(mod_logger) + + +def log_progress(log, level, msg, *args): + """Log to Marvin files and stdout so long polls remain visible.""" + text = msg % args if args else msg + getattr(log, level)(text) + print("[%s] %s" % (level.upper(), text), flush=True) + + +def get_datacenter_config(testclient, test_cls): + """ + Return the --marvin-config file (e.g. ontap.cfg) as a plain dict. + + Marvin injects the datacenter config as ``test_cls.config``. The separate + ``getParsedTestDataConfig()`` API defaults to test_data.py and does not + contain ontap/cloudstack/zones sections from ontap.cfg. + """ + if getattr(test_cls, "config", None): + return jsonDump.dump(test_cls.config) + cfg = testclient.getParsedTestDataConfig() or {} + if cfg.get("ontap") or cfg.get("cloudstack") or cfg.get("zones"): + return cfg + return cfg + + +# --------------------------------------------------------------------------- +# Pool detail helper +# --------------------------------------------------------------------------- + +def _parse_pool_details(pool): + details_raw = getattr(pool, "details", None) + if not details_raw: + return {} + if isinstance(details_raw, dict): + return details_raw + if isinstance(details_raw, list): + return {d.name: d.value for d in details_raw} + return { + k: v for k, v in vars(details_raw).items() + if not k.startswith("_") and k != "typeInfo" + } + + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _list_vols_for_vm(vm_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.listall = True + return cmd + + +# --------------------------------------------------------------------------- +# ONTAP REST helper +# --------------------------------------------------------------------------- + +class OntapRestClient: + """Thin wrapper around the ONTAP REST API for backend validation.""" + + def __init__(self, storage_ip, username, password, port=443): + self._base = "https://%s:%d/api" % (storage_ip, port) + self._auth = (username, password) + + def _get(self, path, params=None): + url = self._base + path + resp = requests.get(url, auth=self._auth, params=params, + verify=False, timeout=30) + resp.raise_for_status() + return resp.json() + + def _delete(self, path, params=None): + url = self._base + path + resp = requests.delete(url, auth=self._auth, params=params, + verify=False, timeout=30) + self._raise_http(resp) + return self._response_data(resp) + + @staticmethod + def _response_data(resp): + """Return response JSON, including async job UUID from Location.""" + if resp.content: + try: + return resp.json() + except ValueError: + pass + location = resp.headers.get("Location", "") + marker = "/cluster/jobs/" + if marker in location: + return {"job": {"uuid": location.split(marker, 1)[1].split("?", 1)[0]}} + return None + + def _raise_http(self, resp): + if resp.ok: + return + body = "" + try: + body = resp.text + except Exception: + body = "" + raise requests.HTTPError( + "%s Client Error: %s for url: %s body: %s" + % (resp.status_code, resp.reason, resp.url, body), + response=resp, + ) + + def _patch(self, path, payload=None, params=None, json_body=None, timeout=60): + url = self._base + path + body = json_body if json_body is not None else payload + resp = requests.patch( + url, auth=self._auth, params=params, json=body, + verify=False, timeout=timeout, + ) + self._raise_http(resp) + return self._response_data(resp) + + def _post(self, path, params=None, data=None, json_body=None, timeout=60, + headers=None, files=None): + url = self._base + path + resp = requests.post( + url, auth=self._auth, params=params, data=data, json=json_body, + headers=headers, files=files, verify=False, timeout=timeout, + ) + self._raise_http(resp) + return self._response_data(resp) + + def _volume_uuid(self, vol_name): + vol = self.get_volume(vol_name) + return (vol or {}).get("uuid") + + def _files_path(self, vol_uuid, filename): + encoded = quote("/" + filename.lstrip("/"), safe="") + return "/storage/volumes/%s/files/%s" % (vol_uuid, encoded) + + def _wait_for_job(self, response, timeout=120): + """Wait for an asynchronous ONTAP response, if it contains a job.""" + job = (response or {}).get("job") or {} + job_uuid = job.get("uuid") + if not job_uuid: + return response + deadline = time.time() + timeout + while time.time() < deadline: + current = self._get("/cluster/jobs/%s" % job_uuid) + state = (current.get("state") or "").lower() + if state == "success": + return current + if state in ("failure", "failed", "error"): + message = current.get("message") or "unknown ONTAP job failure" + raise RuntimeError("ONTAP job %s failed: %s" % (job_uuid, message)) + time.sleep(2) + raise RuntimeError("Timed out waiting for ONTAP job %s" % job_uuid) + + def _svm_aggregates(self, svm_name): + """Return detailed aggregate records assigned to an SVM.""" + svms = self._get( + "/svm/svms", + params={"name": svm_name, "fields": "aggregates"}, + ).get("records", []) + if not svms: + raise RuntimeError("ONTAP SVM '%s' was not found" % svm_name) + aggregates = [] + for aggregate in svms[0].get("aggregates", []): + uuid = aggregate.get("uuid") + if not uuid: + continue + aggregates.append(self._get( + "/storage/aggregates/%s" % uuid, + params={"fields": "name,uuid,state,space.block_storage.available"}, + )) + return aggregates + + def max_online_aggregate_available_bytes(self, svm_name): + """Return the largest free-space value among assigned online aggregates.""" + available = [] + for aggregate in self._svm_aggregates(svm_name): + if (aggregate.get("state") or "").lower() != "online": + continue + free = (aggregate.get("space", {}) + .get("block_storage", {}).get("available")) + if free is not None: + available.append(int(float(free))) + if not available: + raise RuntimeError( + "SVM '%s' has no online aggregate with space data" % svm_name + ) + return max(available) + + def create_flexvol(self, svm_name, volume_name, size_bytes, nas_path=True): + """Create a thin FlexVol directly on a suitable SVM aggregate.""" + suitable = [] + for aggregate in self._svm_aggregates(svm_name): + free = (aggregate.get("space", {}) + .get("block_storage", {}).get("available")) + if ((aggregate.get("state") or "").lower() == "online" + and free is not None and int(float(free)) > int(size_bytes)): + suitable.append((int(float(free)), aggregate)) + if not suitable: + raise RuntimeError( + "No ONTAP aggregate can hold FlexVol '%s'" % volume_name + ) + aggregate = max(suitable, key=lambda item: item[0])[1] + request = { + "name": volume_name, + "svm": {"name": svm_name}, + "size": int(size_bytes), + "aggregates": [{"name": aggregate.get("name")}], + "guarantee": {"type": "none"}, + } + if nas_path: + request["nas"] = {"path": "/" + volume_name} + response = self._post( + "/storage/volumes", + params={"return_timeout": 15}, + json_body=request, + ) + self._wait_for_job(response) + deadline = time.time() + 120 + while time.time() < deadline: + volume = self.get_volume(volume_name) + if volume is not None: + return volume + time.sleep(2) + raise RuntimeError( + "FlexVol '%s' was not visible after creation" % volume_name + ) + + def offline_and_delete_volume(self, name): + """Offline and delete a FlexVol directly; no-op when already absent.""" + volume = self.get_volume(name) + if not volume: + return + uuid = volume.get("uuid") + if not uuid: + raise RuntimeError("FlexVol '%s' has no UUID" % name) + if (volume.get("nas") or {}).get("path"): + response = self._patch( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + json_body={"nas": {"path": ""}}, + ) + self._wait_for_job(response) + if (volume.get("state") or "").lower() != "offline": + response = self._patch( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + json_body={"state": "offline"}, + ) + self._wait_for_job(response) + response = self._delete( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + ) + self._wait_for_job(response) + deadline = time.time() + 120 + while time.time() < deadline: + if self.get_volume(name) is None: + return + time.sleep(2) + raise RuntimeError("FlexVol '%s' still exists after deletion" % name) + + def reassign_volume_export_policy(self, volume_name, policy_name="default"): + """Assign a FlexVol to another export policy before deleting its policy.""" + volume = self.get_volume(volume_name) + if not volume: + raise RuntimeError("FlexVol '%s' was not found" % volume_name) + uuid = volume.get("uuid") + if not uuid: + raise RuntimeError("FlexVol '%s' has no UUID" % volume_name) + response = self._patch( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + json_body={"nas": {"export_policy": {"name": policy_name}}}, + ) + self._wait_for_job(response) + + def create_igroup(self, svm_name, igroup_name, initiator_iqn): + """Create an ONTAP igroup holding a single initiator.""" + self._post( + "/protocols/san/igroups", + json_body={ + "svm": {"name": svm_name}, + "name": igroup_name, + "os_type": "linux", + "protocol": "iscsi", + "initiators": [{"name": initiator_iqn}], + }, + ) + igroup = self.get_igroup(svm_name, igroup_name) + if igroup is None: + raise RuntimeError( + "ONTAP igroup '%s' absent right after creation" % igroup_name + ) + return igroup + + def delete_igroup(self, svm_name, igroup_name): + """Delete an ONTAP igroup by name; no-op when already absent.""" + igroup = self.get_igroup(svm_name, igroup_name) + if not igroup: + return + uuid = igroup.get("uuid") + if not uuid: + raise RuntimeError("ONTAP igroup '%s' has no UUID" % igroup_name) + self._delete("/protocols/san/igroups/%s" % uuid) + + def create_lun_map(self, svm_name, lun_path, igroup_name): + """Map an existing LUN to an existing igroup.""" + response = self._post( + "/protocols/san/lun-maps", + params={"return_timeout": 15}, + json_body={ + "svm": {"name": svm_name}, + "lun": {"name": lun_path}, + "igroup": {"name": igroup_name}, + }, + ) + self._wait_for_job(response) + + def delete_lun_map(self, lun_map): + """Delete one LUN map returned by list_lun_maps_for_volume.""" + lun_uuid = lun_map.get("lun", {}).get("uuid") + igroup_uuid = lun_map.get("igroup", {}).get("uuid") + if not lun_uuid or not igroup_uuid: + raise RuntimeError("ONTAP LUN map is missing LUN or igroup UUID") + self._delete( + "/protocols/san/lun-maps/%s/%s" % (lun_uuid, igroup_uuid) + ) + def delete_volume(self, name): + """Delete the ONTAP FlexVol with the given name. No-op if not found.""" + data = self._get("/storage/volumes", params={"name": name}) + records = data.get("records", []) + if not records: + return + uuid = records[0].get("uuid") + if uuid: + self._delete("/storage/volumes/%s" % uuid) + + def delete_export_policy(self, name): + """Delete the NFS export policy with the given name. No-op if not found.""" + data = self._get("/protocols/nfs/export-policies", params={"name": name}) + records = data.get("records", []) + if not records: + return + policy_id = records[0].get("id") + if policy_id: + self._delete("/protocols/nfs/export-policies/%s" % policy_id) + + def get_volume(self, name): + """Return the ONTAP FlexVol record for the given name, or None.""" + data = self._get("/storage/volumes", params={"name": name}) + records = data.get("records", []) + if not records: + return None + uuid = records[0].get("uuid") + if uuid: + return self._get("/storage/volumes/%s" % uuid, + params={ + "fields": ( + "name,uuid,state,space.size,space.used," + "space.snapshot.reserve_percent," + "nas.path,nas.export_policy" + ) + }) + return records[0] + + # -- NFS helpers --------------------------------------------------------- + + def get_export_policy(self, name): + """Return the ONTAP NFS export policy record for the given name, or None.""" + data = self._get("/protocols/nfs/export-policies", params={"name": name}) + records = data.get("records", []) + if not records: + return None + policy_id = records[0].get("id") + if policy_id: + return self._get( + "/protocols/nfs/export-policies/%s" % policy_id, + params={"fields": "name,svm,rules"} + ) + return records[0] + + def get_data_lifs(self, svm_name): + """Return a list of NFS data LIF IP addresses for the given SVM.""" + data = self._get( + "/network/ip/interfaces", + params={"svm.name": svm_name, "services": "data-nfs", + "fields": "ip,name"} + ) + records = data.get("records", []) + return [r.get("ip", {}).get("address") + for r in records if r.get("ip", {}).get("address")] + + def get_iscsi_data_lifs(self, svm_name): + """Return iSCSI data LIF addresses for the given SVM.""" + data = self._get( + "/network/ip/interfaces", + params={"svm.name": svm_name, "services": "data-iscsi", + "fields": "ip,name"} + ) + return [r.get("ip", {}).get("address") + for r in data.get("records", []) + if r.get("ip", {}).get("address")] + + def get_iscsi_target_iqn(self, svm_name): + """Return the SVM iSCSI target IQN, or None when the service is absent.""" + data = self._get( + "/protocols/san/iscsi/services", + params={"svm.name": svm_name, "fields": "target", + "max_records": 1} + ) + records = data.get("records", []) + if not records: + return None + return (records[0].get("target") or {}).get("name") + + # -- iSCSI helpers ------------------------------------------------------- + + def get_igroup(self, svm_name, igroup_name): + """Return the ONTAP igroup record, or None if not found.""" + data = self._get("/protocols/san/igroups", + params={"svm.name": svm_name, "name": igroup_name, + "fields": "name,uuid,initiators"}) + records = data.get("records", []) + return records[0] if records else None + + def get_lun(self, svm_name, lun_path): + """Return the ONTAP LUN record for the given full path, or None.""" + data = self._get("/storage/luns", + params={"svm.name": svm_name, "name": lun_path, + "fields": "name,uuid,enabled,status,space.size"}) + records = data.get("records", []) + return records[0] if records else None + + def list_luns_in_volume(self, svm_name, vol_name): + """Return all LUN records whose path starts with /vol/{vol_name}/.""" + prefix = "/vol/%s/" % vol_name + data = self._get("/storage/luns", + params={"svm.name": svm_name, + "fields": "name,uuid,enabled,status,space.size"}) + return [r for r in data.get("records", []) + if r.get("name", "").startswith(prefix)] + + def list_lun_maps_for_volume(self, svm_name, vol_name): + """Return all LUN-map records for LUNs residing in the given FlexVol.""" + prefix = "/vol/%s/" % vol_name + data = self._get("/protocols/san/lun-maps", + params={"svm.name": svm_name, + "fields": "lun.name,lun.uuid,igroup.name," + "igroup.uuid,logical_unit_number"}) + return [r for r in data.get("records", []) + if r.get("lun", {}).get("name", "").startswith(prefix)] + + # -- NFS file helpers ---------------------------------------------------- + + def list_files_in_volume(self, vol_name, path="/"): + """Return a list of file names at ``path`` inside the named FlexVol. + + Uses the ONTAP REST file-system API: + GET /api/storage/volumes/{uuid}/files/{url_encoded_path} + + The path must appear in the URL (not as a query parameter). The root + directory is represented as ``%2F``. + + Returns an empty list if the volume does not exist, the path is empty, + or the request fails. + """ + vol = self.get_volume(vol_name) + if not vol: + return [] + vol_uuid = vol.get("uuid", "") + if not vol_uuid: + return [] + # URL-encode the path component (/ → %2F) and embed it in the URL. + encoded_path = quote(path, safe="") + try: + resp = self._get( + "/storage/volumes/%s/files/%s" % (vol_uuid, encoded_path), + params={"fields": "name,type", "max_records": "500"} + ) + except Exception: + return [] + return [r.get("name", "") for r in resp.get("records", []) + if r.get("name") not in (".", "..")] + + def _unmap_lun(self, svm_name, lun_uuid): + """Best-effort removal of all lun-maps for a LUN UUID.""" + try: + maps = self._get( + "/protocols/san/lun-maps", + params={ + "svm.name": svm_name, + "lun.uuid": lun_uuid, + "fields": "lun.uuid,igroup.uuid", + }, + ) + for lun_map in maps.get("records", []): + mapped_lun = lun_map.get("lun", {}).get("uuid") or lun_uuid + igroup_uuid = lun_map.get("igroup", {}).get("uuid") + if not igroup_uuid: + continue + try: + self._delete( + "/protocols/san/lun-maps/%s/%s" + % (mapped_lun, igroup_uuid) + ) + except Exception: + pass + except Exception: + pass + + def _offline_lun(self, svm_name, lun_path, lun_uuid): + """Take LUN offline. Lab ONTAP rejects allow_delete_online.""" + try: + self._patch( + "/storage/luns/%s" % lun_uuid, + payload={"enabled": False, "status": {"state": "offline"}}, + ) + except Exception: + try: + self._patch( + "/storage/luns/%s" % lun_uuid, + payload={"enabled": False}, + ) + except Exception: + return + deadline = time.time() + 60 + while time.time() < deadline: + cur = self.get_lun(svm_name, lun_path) + if not cur: + return + state = ( + (cur.get("status") or {}).get("state") + or ("online" if cur.get("enabled") else "offline") + ) + if str(state).lower() == "offline" or cur.get("enabled") is False: + return + time.sleep(2) + + def _delete_lun_clones(self, svm_name, parent_uuid): + """Delete FlexClone child LUNs so the parent cache LUN can be removed.""" + try: + data = self._get( + "/storage/luns", + params={ + "svm.name": svm_name, + "fields": "name,uuid,clone", + }, + ) + except Exception: + return + for child in data.get("records", []): + clone = child.get("clone") or {} + source = clone.get("source") or {} + if source.get("uuid") != parent_uuid: + continue + child_path = child.get("name") + child_uuid = child.get("uuid") + if not child_path or not child_uuid: + continue + self._unmap_lun(svm_name, child_uuid) + self._offline_lun(svm_name, child_path, child_uuid) + try: + self._delete("/storage/luns/%s" % child_uuid) + except Exception: + pass + + def delete_lun(self, svm_name, lun_path): + """Unmap + offline + delete a LUN by full path. No-op if missing.""" + lun = self.get_lun(svm_name, lun_path) + if not lun: + return False + uuid = lun.get("uuid") + if not uuid: + return False + + # Mapped LUNs / FlexClone parents cannot be deleted until dependents go. + self._unmap_lun(svm_name, uuid) + self._delete_lun_clones(svm_name, uuid) + self._offline_lun(svm_name, lun_path, uuid) + if not self.get_lun(svm_name, lun_path): + return True + # Do not pass allow_delete_online — rejected on this ONTAP build. + self._delete("/storage/luns/%s" % uuid) + return True + + def write_file_in_volume(self, vol_name, filename, size_bytes, + chunk_bytes=512 * 1024): + """Write *size_bytes* of incompressible data into a FlexVol file. + + Zeros compress to almost nothing on ONTAP, so the payload is random. + The files API requires ``multipart/form-data`` and rejects writes + larger than 1 MiB, so data is sent in chunks. No VM, NFS mount, or + CloudStack volume is required. + """ + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + raise RuntimeError("ONTAP FlexVol '%s' not found" % vol_name) + url_path = self._files_path(vol_uuid, filename) + written = 0 + size_bytes = int(size_bytes) + while written < size_bytes: + chunk = min(int(chunk_bytes), size_bytes - written) + files = { + "file": (filename, os.urandom(chunk), + "application/octet-stream"), + } + self._post( + url_path, + params={"byte_offset": written, "overwrite": "true"}, + files=files, + timeout=120, + ) + written += chunk + + def delete_file_in_volume(self, vol_name, file_path): + """Delete a file inside a FlexVol. + + ``file_path`` may be absolute (``/foo/bar``) or relative to the + volume root. Returns True when the volume exists and the delete was + issued. A missing file (404/409) is ignored so cleanup can retry. + Returns False when the volume is gone. + """ + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + return False + try: + self._delete(self._files_path(vol_uuid, file_path)) + except requests.HTTPError as exc: + status = getattr(exc.response, "status_code", None) + if status not in (404, 409): + raise + return True + + def list_snapshots(self, vol_name): + """Return FlexVol snapshot records (name, uuid) for the named volume. + + Uses ``GET /api/storage/volumes/{uuid}/snapshots``. An empty list means + the volume does not exist or has no snapshots. + """ + vol = self.get_volume(vol_name) + if not vol: + return [] + vol_uuid = vol.get("uuid", "") + if not vol_uuid: + return [] + try: + data = self._get( + "/storage/volumes/%s/snapshots" % vol_uuid, + params={"fields": "name,uuid", "max_records": "1000"}, + ) + except Exception: + return [] + return list(data.get("records", []) or []) + + +# Invalid createStoragePool capacitybytes values. +# (label, value, alternative fragment groups; any one group may match) +# The API rejects zero, a negative number, special characters, and a decimal +# as an invalid capacitybytes. A build that reaches the plugin instead reports +# that capacity must be at least the ONTAP minimum. +INVALID_CAPACITYBYTES_CASES = ( + ("zero", 0, (("invalid value", "capacitybytes"), ("must be at least",))), + ("negative", -1, (("invalid value", "capacitybytes"), ("must be at least",))), + ("special-characters", "!@#$", (("invalid value", "capacitybytes"),)), + ("decimal", "1.5", (("invalid value", "capacitybytes"),)), +) + + +def run_invalid_capacitybytes_rejections(test, create_pool, cleanup_pool, log): + """Reject createStoragePool for every invalid capacitybytes value. + + create_pool(pool_name, capacitybytes) calls createStoragePool. + cleanup_pool(pool, pool_name) removes a pool or FlexVol left behind. + Failures are collected so every value is attempted. + """ + test._sweep_tracked_pool2() + failures = [] + for label, capacity, fragments in INVALID_CAPACITYBYTES_CASES: + pool_name = test._throwaway_pool_name( + "BadCap%s" % label.replace("-", "") + ) + log_progress( + log, "info", + "Requesting pool '%s' with capacitybytes=%r (expect reject: %s)", + pool_name, capacity, label, + ) + created = None + try: + try: + created = create_pool(pool_name, capacity) + test.__class__.pool2 = created + failures.append( + "[%s] createStoragePool succeeded for capacitybytes=%r" + % (label, capacity) + ) + except CloudstackAPIException as exc: + error_text = str(exc) + log_progress( + log, "info", + "Rejected capacitybytes=%r for '%s': %s", + capacity, pool_name, error_text, + ) + if not any(all(frag in error_text for frag in group) + for group in fragments): + failures.append( + "[%s] error matched none of %s, got: %s" + % (label, fragments, error_text) + ) + if created is None: + try: + test._assert_no_pool_named(pool_name) + except AssertionError as exc: + failures.append("[%s] %s" % (label, exc)) + if test.ontap.get_volume(pool_name) is not None: + failures.append( + "[%s] ONTAP FlexVol '%s' was created despite the " + "rejected pool create" % (label, pool_name) + ) + except SkipTest: + raise + except Exception as exc: + failures.append( + "[%s] %s: %s" % (label, type(exc).__name__, exc) + ) + finally: + cleanup_pool(created, pool_name) + if failures: + test.fail( + "Invalid capacitybytes create was not rejected cleanly:\n%s" + % "\n".join(failures) + ) + + +# --------------------------------------------------------------------------- +# Base test class +# --------------------------------------------------------------------------- + +class OntapTestBase(cloudstackTestCase): + + # ONTAP refuses to shrink a FlexVol below this; distinct from the + # plugin's 1.56 GiB create-time floor (ONTAP_MIN_VOLUME_SIZE). + ONTAP_MIN_FLEXVOL_SIZE = 20 * 1024 * 1024 + FILLER_FILENAME = "ontap-filler.bin" + FILLER_SIZE = 32 * 1024 * 1024 + + # ---- shared state (set/cleared by individual tests) ---------------- + pool = None + volume = None + # Volumes a single test allocates on top of ``volume`` and deletes before + # it returns; tracked here only so a failed run still cleans them up. + extra_volumes = [] + # File written into the FlexVol by tests that must raise ONTAP used space. + filler_filename = None + filler_flexvol = None + pool2 = None + volume2 = None + # ---- VM state, for suites that deploy an instance ------------------ + vm = None + template_id = None + service_offering_id = None + network_id = None + _created_network_id = None + # Prefix for a guest network this suite creates on Advanced zones. + _vm_network_name_prefix = "ontap-vm-net" + disk_offering_id = None + svm_name = None + cluster_hosts = None + kvm_hosts_ssh_creds = [] # [{'host': '10.x.x.x', 'user': 'root', 'password': '...'}] + _host_iqn_cache = {} + igroup_baseline = {} + ontap = None + testdata = None + zone = None + cluster = None + domain = None + account = None + _cleanup = [] + + # Subclass sets this to distinguish volume names, e.g. "OntapNFS3Vol" + _vol_name_prefix = "OntapVol" + + # Template-cache suites set these: "NFS3" / "ISCSI" and the numeric + # vm_template.id used as the template_spool_ref key. + PROTOCOL = "NFS3" + template_db_id = None + + # VM states from which the VM will never reach any other target state. + _VM_TERMINAL_STATES = ("error", "destroyed", "expunging") + + # ---- zone guard ---------------------------------------------------- + + @classmethod + def _ensure_zone(cls, config, zone_name, cluster_name): + """ + Verify that the named zone and cluster already exist and return + (zone, cluster). Raises RuntimeError with a clear message if the + zone is absent — run the setup_zone step first: + + bash test/integration/plugins/ontap/run_tests.sh setup_zone + """ + zone = get_zone(cls.apiClient, zone_name=zone_name) + if not zone: + raise RuntimeError( + "Zone '%s' not found. Create it first by running:\n" + " bash test/integration/plugins/ontap/run_tests.sh setup_zone\n" + "Then re-run the tests." + % (zone_name or "") + ) + clusters = (list_clusters(cls.apiClient, name=cluster_name) + if cluster_name else list_clusters(cls.apiClient)) + if not clusters: + raise RuntimeError( + "No cluster found (filter: %r) in zone '%s'. " + "Verify the cluster was created by the setup_zone step." + % (cluster_name, zone.name) + ) + return zone, clusters[0] + + # ---- shared setup helper ------------------------------------------- + + @classmethod + def setUpClass(cls): + enable_live_logging(cls) + + @classmethod + def _setup_cloudstack_resources(cls, config, account_testdata): + """ + Resolve zone, cluster, domain, account, cluster hosts, and disk + offering from the Marvin config. Call this from subclass setUpClass + after ``cls.ontap`` and ``cls.svm_name`` have been assigned. + """ + cs_cfg = config.get("cloudstack", {}) + zone_name = cs_cfg.get("zoneName", None) + cluster_name = cs_cfg.get("clusterName", None) + domain_name = cs_cfg.get("domainName", "ROOT") + + cls.zone, cls.cluster = cls._ensure_zone(config, zone_name, cluster_name) + cls.domain = get_domain(cls.apiClient, domain_name=domain_name) + + cls.account = Account.create(cls.apiClient, account_testdata, admin=1) + cls._cleanup = [cls.account] + + list_hosts_cmd = listHostsAPI.listHostsCmd() + list_hosts_cmd.clusterid = cls.cluster.id + list_hosts_cmd.type = "Routing" + cls.cluster_hosts = cls.apiClient.listHosts(list_hosts_cmd) or [] + + # Use a deterministic, non-strict 2 GiB offering. Selecting the first + # offering in the environment can choose a fixed-size or very large + # offering that cannot exercise the resize API on the small test pool. + offering_name = "ontap-test-do-%d" % random.randint(0, 99999) + do = DiskOffering.create( + cls.apiClient, + { + "name": offering_name, + "displaytext": "ONTAP test disk offering", + "disksize": 2, + "storagetype": "shared", + }, + ) + cls._cleanup.append(do) + cls.disk_offering_id = do.id + + # Parse KVM host SSH credentials from zones/pods/clusters/hosts config. + # Used by _cleanup_kvm_storage_pool_mounts to unmount stale NFS pools. + cls.kvm_hosts_ssh_creds = [] + try: + for zone in config.get("zones", []): + for pod in zone.get("pods", []): + for cluster in pod.get("clusters", []): + for host_cfg in cluster.get("hosts", []): + host_ip = urlparse( + host_cfg.get("url", "") + ).hostname or "" + if host_ip: + cls.kvm_hosts_ssh_creds.append({ + "host": host_ip, + "user": host_cfg.get("username", "root"), + "password": host_cfg.get("password", ""), + }) + except Exception as parse_ex: + logger.warning( + "_setup_cloudstack_resources: could not parse KVM SSH creds: %s" + % parse_ex + ) + + # ---- KVM storage cleanup helper ------------------------------------ + + @classmethod + def _cleanup_kvm_storage_pool_mounts(cls, pool_uuid): + """Unmount and undefine the libvirt pool on each KVM host.""" + kvm_host.cleanup_kvm_storage_pool_mounts( + cls.kvm_hosts_ssh_creds, pool_uuid) + + # ---- shared teardown ----------------------------------------------- + + @classmethod + def _all_tracked_volumes(cls): + """Every volume the suite created, most recent first, without repeats.""" + seen = set() + ordered = [] + for vol in list(cls.extra_volumes or []) + [cls.volume2, cls.volume]: + vol_id = getattr(vol, "id", None) + if vol is None or vol_id in seen: + continue + seen.add(vol_id) + ordered.append(vol) + return ordered + + # ---- VM deploy/attach helpers -------------------------------------- + + @classmethod + def _discover_vm_deploy_resources(cls): + """Resolve the template, service offering, and network for VM deploys. + + A missing template is not fatal: ``template_id`` is left as None so + callers can skip the VM step while the rest of the suite still runs. + On Advanced zones an existing account network is reused when present, + otherwise an Isolated one is created and torn down in tearDownClass. + """ + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + cls.template_id = kvm_ready[0].id if kvm_ready else None + if cls.template_id is None: + logger.warning( + "No ready user KVM template in zone '%s' — VM steps will skip." + % cls.zone.name + ) + + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + if offerings: + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + cls.network_id = None + if getattr(cls.zone, "networktype", "Basic").lower() != "advanced": + return + + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + return + + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + no_offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in no_offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + no_offerings[0] if no_offerings else None + ) + if snat_offering is None: + return + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "%s-%d" % (cls._vm_network_name_prefix, + random.randint(0, 9999)) + cn_cmd.displaytext = "ONTAP test VM network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def _destroy_vm_if_present(cls): + """Stop and expunge the suite's VM. Safe to call when none exists.""" + if cls.vm is None: + return + vm_id = cls.vm.id + try: + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + state = vms[0].state.lower() if vms else "unknown" + if state not in ("stopped", "destroyed", "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=180) + except Exception as e: + logger.warning("Could not stop VM %s: %s" % (vm_id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("Could not destroy VM %s: %s" % (vm_id, e)) + cls.vm = None + + @classmethod + def _delete_created_network(cls): + """Delete the guest network this suite created, if any.""" + if cls._created_network_id is None: + return + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + except Exception as e: + logger.warning( + "Could not delete network %s: %s" % (cls._created_network_id, e) + ) + cls._created_network_id = None + + def _poll_volume_attached(self, vol_id, timeout=180, interval=5): + """Poll listVolumes until virtualmachineid is set; return it or None. + + ONTAP-backed volumes stay in state 'Ready' when attached, so the + virtualmachineid field is the reliable signal. + """ + deadline = time.time() + timeout + vol_vmid = None + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if vol_vmid: + return vol_vmid + time.sleep(interval) + return vol_vmid + + def _assert_vm_running_with_volume(self, vm_id, vol_id, label): + """Assert the VM is still up and still owns the volume.""" + vm_obj = _wait_for_vm_state(self.apiClient, vm_id, "Running", + timeout=60) + self.assertIsNotNone( + vm_obj, "[%s] VM %s vanished from listVirtualMachines" + % (label, vm_id), + ) + self.assertEqual( + vm_obj.state, "Running", + "[%s] VM should still be 'Running', got '%s'" + % (label, vm_obj.state), + ) + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "[%s] volume %s is no longer listed" % (label, vol_id), + ) + self.assertEqual( + getattr(vol, "virtualmachineid", None), vm_id, + "[%s] volume %s should still be attached to VM %s, " + "virtualmachineid is %s" + % (label, vol_id, vm_id, getattr(vol, "virtualmachineid", None)), + ) + + def _detach_volume_if_attached(self, vol_id): + """Detach the volume when a VM holds it. No-op otherwise.""" + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + if not vols or not getattr(vols[0], "virtualmachineid", None): + return + try: + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = vol_id + self.apiClient.detachVolume(cmd) + except Exception as e: + logger.warning("Could not detach volume %s: %s" % (vol_id, e)) + return + deadline = time.time() + 120 + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + if not vols or not getattr(vols[0], "virtualmachineid", None): + return + time.sleep(5) + logger.warning( + "Volume %s still reports a virtualmachineid after detach" % vol_id + ) + + @classmethod + def tearDownClass(cls): + """Best-effort cleanup of any resources left behind by a failed run.""" + cls._destroy_vm_if_present() + cls._delete_created_network() + if (getattr(cls, "ontap", None) is not None + and getattr(cls, "filler_filename", None) + and getattr(cls, "filler_flexvol", None)): + try: + cls.ontap.delete_file_in_volume( + cls.filler_flexvol, cls.filler_filename + ) + except Exception as fe: + logger.warning( + "tearDownClass: could not delete filler file %s in %s: %s" + % (cls.filler_filename, cls.filler_flexvol, fe) + ) + cls.filler_filename = None + cls.filler_flexvol = None + for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: + try: + # Step 1: Check current pool state + pools = list_storage_pools(cls.apiClient, id=pool.id) + if not pools: + continue # already deleted + pool_state = pools[0].state + + # Step 2: If in Maintenance, attempt to exit it + if pool_state == "Maintenance": + try: + cc = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cc.id = pool.id + cls.apiClient.cancelStorageMaintenance(cc) + time.sleep(5) + except Exception: + pass + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + cls.apiClient.updateStoragePool(ec) + time.sleep(3) + except Exception: + pass + pools = list_storage_pools(cls.apiClient, id=pool.id) + if pools: + pool_state = pools[0].state + + # Step 3: Delete volumes — always attempt regardless of pool + # state. For iSCSI this works even in Maintenance; for NFS3/KVM + # it may fail with NPE ("storagePoolInformation is null") when + # pool is in Maintenance — that exception is caught below. + for vol in cls._all_tracked_volumes(): + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as ve: + logger.warning( + "tearDownClass: could not delete volume %s: %s" + % (vol.id, ve)) + + # Re-enter Maintenance only if pool was Up/Disabled (avoid + # double-entering when cancel maintenance above already left it + # in Maintenance) + if pool_state in ("Up", "Disabled"): + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + cls.apiClient.enableStorageMaintenance(mc) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(cls.apiClient, id=pool.id) + if ps and ps[0].state == "Maintenance": + break + time.sleep(5) + except Exception: + pass + + # Step 4: Force-delete the pool + dc = deleteStoragePoolAPI.deleteStoragePoolCmd() + dc.id = pool.id + dc.forced = True + cls.apiClient.deleteStoragePool(dc) + except Exception as e: + logger.warning("tearDownClass: could not delete pool %s: %s" + % (pool.id, e)) + # Last resort: delete ONTAP FlexVol and export policy directly + # so that ONTAP is never left with orphaned volumes even when + # the CloudStack pool record cannot be removed. + if hasattr(cls, "ontap") and cls.ontap is not None: + try: + cls.ontap.delete_volume(pool.name) + logger.warning( + "tearDownClass: deleted ONTAP FlexVol '%s' directly" + % pool.name) + except Exception as oe: + logger.warning( + "tearDownClass: ONTAP direct volume delete '%s' " + "failed: %s" % (pool.name, oe)) + try: + # For NFS3 pools also remove the export policy + ep_name = getattr(cls, "pool_ep_name", None) + if ep_name is None: + ep_name = "cs-%s-%s" % ( + getattr(cls, "svm_name", ""), pool.name) + cls.ontap.delete_export_policy(ep_name) + logger.warning( + "tearDownClass: deleted export policy '%s' directly" + % ep_name) + except Exception: + pass + + # Clean up volumes that may not have been handled with pool teardown + for vol in cls._all_tracked_volumes(): + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as e: + logger.warning("tearDownClass: could not delete volume %s: %s" + % (vol.id, e)) + + try: + cleanup_resources(cls.apiClient, cls._cleanup) + except Exception as e: + logger.debug("tearDownClass cleanup_resources: %s" % e) + + # No per-test tearDown — state intentionally persists between steps. + + # ---- shared helpers ------------------------------------------------ + + def _poll_pool_state(self, pool_id, target_state, timeout=120, interval=5): + """Poll listStoragePools until the pool reaches target_state or timeout.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current_state = "unknown" + log_progress( + logger, "info", + "Waiting for pool %s to reach state '%s' " + "(timeout=%ds, poll every %ds).", + pool_id, target_state, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current_state = pools[0].state + if current_state == target_state: + log_progress( + logger, "info", + "Pool %s reached state '%s' after %ds (%d polls).", + pool_id, target_state, elapsed, attempt, + ) + return pools[0] + log_progress( + logger, "info", + "Pool poll #%d: pool %s state=%s (want %s) " + "[elapsed %ds, ~%ds left]", + attempt, pool_id, current_state, target_state, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "Pool %s did not reach state '%s' within %ds (last: '%s').", + pool_id, target_state, timeout, current_state, + ) + self.fail( + "Pool %s did not reach state '%s' within %ds (last: '%s')" + % (pool_id, target_state, timeout, current_state) + ) + + def _enter_maintenance(self, pool_id, timeout=120): + """Put a pool into Maintenance and wait for CloudStack to report it.""" + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = pool_id + self.apiClient.enableStorageMaintenance(cmd) + return self._poll_pool_state(pool_id, "Maintenance", timeout=timeout) + + def _exit_maintenance(self, pool_id, timeout=120): + """Cancel Maintenance on a pool and wait for CloudStack to report Up.""" + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = pool_id + self.apiClient.cancelStorageMaintenance(cmd) + return self._poll_pool_state(pool_id, "Up", timeout=timeout) + + def _exit_maintenance_quietly(self, pool, label): + """Bring a pool out of Maintenance without failing the test. + + Returns True when the pool is no longer in Maintenance, False when + it is not listed or the cancel did not take effect. + """ + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + return False + if not listed: + return False + if listed[0].state != "Maintenance": + return True + try: + self._exit_maintenance(pool.id) + return True + except Exception as exc: + logger.warning("[%s] could not cancel maintenance on '%s': %s", + label, pool.name, exc) + return False + + @classmethod + def host_iqn(cls, host): + """The iSCSI initiator IQN for a cluster host, or None.""" + host_ip = getattr(host, "ipaddress", None) + if not host_ip: + return None + if host_ip in cls._host_iqn_cache: + return cls._host_iqn_cache[host_ip] + iqn = None + creds = next( + (c for c in cls.kvm_hosts_ssh_creds if c["host"] == host_ip), None + ) + if creds is not None: + try: + ssh = SshClient(host_ip, 22, creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0) + out = ssh.execute( + "awk -F= '/^InitiatorName=/{print $2}' " + "/etc/iscsi/initiatorname.iscsi 2>/dev/null" + ) + for line in out or []: + line = line.strip() + if line.startswith("iqn."): + iqn = line + break + except Exception as ex: + logger.warning("host_iqn: SSH to %s failed: %s", host_ip, ex) + cls._host_iqn_cache[host_ip] = iqn + return iqn + + def _kvm_host_ip(self): + """Return one KVM host that this suite can reach over SSH.""" + if not self.kvm_hosts_ssh_creds: + self.skipTest("KVM SSH credentials are not configured") + return self.kvm_hosts_ssh_creds[0]["host"] + + def _kvm_run(self, host_ip, command): + """Run one command on a KVM host and return its output lines.""" + creds = next( + (c for c in self.kvm_hosts_ssh_creds if c["host"] == host_ip), + None + ) + if creds is None: + self.skipTest("No SSH credentials for KVM host %s" % host_ip) + ssh = SshClient( + host_ip, 22, creds["user"], creds["password"], + retries=3, delay=3, timeout=20.0 + ) + return ssh.execute(command) or [] + + @classmethod + def _igroup_name(cls, host_uuid): + """Return the igroup name used by OntapStorageUtils.""" + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", str(host_uuid)) + return ("cs_%s_%s" % (sanitized, cls.svm_name))[:96] + + @classmethod + def _iscsi_host_specs(cls): + """Return (igroup name, initiator IQN) for iSCSI cluster hosts.""" + specs = [] + for host in cls.cluster_hosts or []: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + or cls.host_iqn(host) + ) + host_uuid = getattr(host, "id", None) + if not iqn or not iqn.startswith("iqn.") or not host_uuid: + continue + specs.append((cls._igroup_name(host_uuid), iqn)) + return specs + + @staticmethod + def _igroup_initiators(igroup): + if igroup is None: + return None + return tuple(sorted( + i.get("name", "") for i in igroup.get("initiators", []) + )) + + @classmethod + def _capture_igroup_baseline(cls): + """Snapshot shared host igroups before an iSCSI suite creates a pool.""" + cls.igroup_baseline = {} + for igroup_name, _ in cls._iscsi_host_specs(): + igroup = cls.ontap.get_igroup(cls.svm_name, igroup_name) + cls.igroup_baseline[igroup_name] = cls._igroup_initiators(igroup) + logger.info( + "Captured iSCSI igroup baseline for SVM '%s': %s", + cls.svm_name, cls.igroup_baseline, + ) + + def _assert_igroup_baseline_unchanged(self, context): + """Assert an operation did not change pre-existing shared igroups.""" + for igroup_name, expected_initiators in self.igroup_baseline.items(): + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + actual_initiators = self._igroup_initiators(igroup) + self.assertEqual( + actual_initiators, expected_initiators, + "ONTAP igroup '%s' changed %s: expected initiators %s, got %s" + % (igroup_name, context, expected_initiators, + actual_initiators), + ) + + def _assert_no_lun_maps_for_volume(self, volume_name, context): + """Assert no LUN in a test FlexVol remains mapped to any igroup.""" + maps = self.ontap.list_lun_maps_for_volume( + self.svm_name, volume_name + ) + self.assertFalse( + maps, + "LUN maps for FlexVol '%s' remain %s: %s" + % (volume_name, context, maps), + ) + + def _get_cs_volume(self, vol_id): + """Return the CloudStack volume object, or None if it is gone.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return vols[0] if vols else None + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + return self._get_cs_volume(vol_id) is not None + + def _purge_cs_volume_record(self, vol, label): + """Remove a CS volume record the forced pool delete may have left. + + The backing storage is already gone at this point, so a failure here + only affects tidiness — the volume stays in the volume2 slot for + tearDownClass to retry and no exception is raised. + """ + if vol is None: + return + if self._volume_exists_in_cs(vol.id): + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + logger.info("[%s] deleted leftover CS volume record %s", + label, vol.id) + except Exception as exc: + logger.warning("[%s] could not delete leftover CS volume %s: %s", + label, vol.id, exc) + else: + logger.info("[%s] CS volume %s was removed along with the pool", + label, vol.id) + if not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + + def _remove_cs_volume(self, pool, vol, label): + """Clear the CloudStack volume so the pool can be force-deleted. + + deleteStoragePool(forced=True) refuses while any volume on the pool is + in a state other than Destroy. deleteVolume is tried first because it + also reclaims the backing storage, but it expunges through libvirt and + fails when the FlexVol is already gone. destroyVolume(expunge=False) + is the fallback: it only moves the record to Destroy, which is all the + forced pool delete requires - it expunges the leftovers itself. + + Returns True once the volume no longer blocks the forced pool delete. + """ + if vol is None or not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + return True + self._exit_maintenance_quietly(pool, label) + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + logger.warning("[%s] deleteVolume failed for %s (%s); falling back " + "to destroyVolume without expunge", + label, vol.id, exc) + try: + cmd = destroyVolumeAPI.destroyVolumeCmd() + cmd.id = vol.id + cmd.expunge = False + self.apiClient.destroyVolume(cmd) + except Exception as destroy_exc: + logger.warning("[%s] destroyVolume also failed for %s: %s", + label, vol.id, destroy_exc) + remaining = self._get_cs_volume(vol.id) + if remaining is None: + self.__class__.volume2 = None + return True + state = getattr(remaining, "state", None) + return state is None or state.lower() in ( + "destroy", "destroyed", "expunging", "expunged" + ) + + def _other_ontap_pools_on_svm(self, current_pool_id): + """Return other CloudStack ONTAP pools that use this suite's SVM.""" + try: + pools = list_storage_pools(self.apiClient) or [] + except Exception: + return ["unable to list storage pools"] + others = [] + for pool in pools: + if str(getattr(pool, "id", "")) == str(current_pool_id): + continue + details = _parse_pool_details(pool) + if details.get("svmName") == getattr(self, "svm_name", None): + others.append(pool) + continue + provider = (getattr(pool, "provider", "") or "").lower() + if not details and "netapp" in provider and "ontap" in provider: + others.append(pool) + return others + + def _poll_vm_state(self, vm_id, target_state, timeout=300, interval=10): + """ + Poll listVirtualMachines until the VM reaches target_state. + Fails immediately when the VM lands in a terminal state other than + the target instead of waiting out the full timeout. + """ + deadline = time.time() + timeout + current_state = "unknown" + target = target_state.lower() + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms: + current_state = vms[0].state + current = (current_state or "").lower() + if current == target: + return vms[0] + if current in self._VM_TERMINAL_STATES: + self.fail( + "VM %s entered terminal state '%s' while waiting for '%s'" + % (vm_id, current_state, target_state) + ) + time.sleep(interval) + self.fail( + "VM %s did not reach state '%s' within %ds (last: '%s')" + % (vm_id, target_state, timeout, current_state) + ) + + # ---- template-cache assertions ------------------------------------ + + def _is_iscsi(self): + return self.PROTOCOL.upper() == "ISCSI" + + def _root_volume_for_vm(self, vm_id): + vols = self.apiClient.listVolumes(_list_vols_for_vm(vm_id)) or [] + roots = [ + v for v in vols + if str(getattr(v, "type", "")).upper() == "ROOT" + ] + self.assertTrue(roots, "No ROOT volume for VM %s" % vm_id) + return roots[0] + + def _assert_root_on_pool(self, vm_id, pool): + root = self._root_volume_for_vm(vm_id) + self.assertEqual( + str(root.storageid), str(pool.id), + "ROOT volume storageid=%s should equal ONTAP pool id=%s " + "(check service-offering / pool storage tags)" + % (root.storageid, pool.id), + ) + return root + + def _wait_for_ready_spool_ref(self, pool_db_id, timeout=600): + spool = tcu.wait_for_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + timeout=timeout, + ) + tcu.assert_spool_ref_ready( + self, spool, expect_local_path=self._is_iscsi(), + ) + return spool + + def _assert_single_ready_spool_ref(self, pool_db_id): + count = tcu.count_template_spool_refs( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + ) + self.assertEqual( + count, 1, "Expected one template_spool_ref, got %s" % count, + ) + spool = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + ) + tcu.assert_spool_ref_ready( + self, spool, expect_local_path=self._is_iscsi(), + ) + return spool + + def _assert_cache_on_ontap(self, pool, spool_ref): + if self._is_iscsi(): + tcu.assert_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + cache_count = tcu.count_iscsi_template_cache_luns( + self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + self.assertEqual( + cache_count, 1, + "Expected exactly one cs_tmpl_%s LUN, found %s" + % (self.__class__.template_db_id, cache_count), + ) + else: + tcu.assert_nfs_template_cache_file( + self, self.ontap, pool.name, spool_ref.get("install_path") + ) + + def _count_non_cache_luns(self, pool): + return tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, pool.name + ) + + def _wait_for_non_cache_lun_count(self, pool, expected, timeout=180, + interval=10): + """Poll until the FlexVol holds ``expected`` non-cache LUNs.""" + deadline = time.time() + timeout + current = self._count_non_cache_luns(pool) + while current != expected and time.time() < deadline: + time.sleep(interval) + current = self._count_non_cache_luns(pool) + self.assertEqual( + current, expected, + "Expected %s non-cache LUNs in FlexVol '%s', found %s" + % (expected, pool.name, current), + ) + + def _poll_pool_capacity(self, pool_id, expected_bytes, timeout=120, + interval=5): + """Poll listStoragePools until capacitybytes equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for pool %s to report capacitybytes=%d " + "(timeout=%ds, poll every %ds).", + pool_id, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current = int(getattr(pools[0], "capacitybytes", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "Pool %s reported capacitybytes=%d after %ds (%d polls).", + pool_id, expected_bytes, elapsed, attempt, + ) + return pools[0] + log_progress( + logger, "info", + "Capacity poll #%d: pool %s capacitybytes=%d (want %d) " + "[elapsed %ds, ~%ds left]", + attempt, pool_id, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "Pool %s did not report capacitybytes=%d within %ds (last: %d).", + pool_id, expected_bytes, timeout, current, + ) + self.fail( + "Pool %s did not report capacitybytes=%d within %ds (last: %d)" + % (pool_id, expected_bytes, timeout, current) + ) + + def _poll_kvm_pool_capacity(self, pool_id, volume_name, expected_bytes, + timeout=120, interval=5): + """Poll the KVM NFS mount until it reports the resized capacity.""" + self.assertTrue( + self.kvm_hosts_ssh_creds, + "No KVM host SSH credentials are configured for host-side " + "capacity verification", + ) + volume = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + volume, + "ONTAP FlexVol '%s' was not found for host-side capacity " + "verification" % volume_name, + ) + reserve_percent = ( + volume.get("space", {}).get("snapshot", {}).get("reserve_percent") + ) + self.assertIsNotNone( + reserve_percent, + "ONTAP FlexVol '%s' did not report " + "space.snapshot.reserve_percent" % volume_name, + ) + reserve_percent = int(reserve_percent) + self.assertGreaterEqual(reserve_percent, 0) + self.assertLessEqual(reserve_percent, 100) + # CloudStack and ONTAP report provisioned size, while statvfs reports + # capacity after the FlexVol's configurable snapshot reserve. + expected_host_bytes = ( + expected_bytes * (100 - reserve_percent) // 100 + ) + tolerance = max(1024 * 1024 * 1024, expected_host_bytes // 10000) + deadline = time.time() + timeout + observed = {} + errors = {} + + while time.time() < deadline: + observed = {} + errors = {} + for creds in self.kvm_hosts_ssh_creds: + host_ip = creds["host"] + try: + ssh = SshClient( + host_ip, 22, + creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + output = ssh.execute( + "df -B1 --output=size '/mnt/%s' 2>/dev/null | " + "awk 'NR == 2 {print $1}'" % pool_id + ) + values = [ + line.strip() for line in output if line.strip().isdigit() + ] + if values: + observed[host_ip] = int(values[-1]) + else: + errors[host_ip] = "NFS pool is not mounted or has no capacity" + except Exception as ex: + errors[host_ip] = str(ex) + + if observed and all( + abs(size - expected_host_bytes) <= tolerance + for size in observed.values()): + logger.info( + "KVM hosts report resized pool %s usable capacity near " + "%d B (%d%% snapshot reserve on %d B provisioned): %s", + pool_id, expected_host_bytes, reserve_percent, + expected_bytes, observed, + ) + return observed + time.sleep(interval) + + self.fail( + "KVM hosts did not report pool %s usable capacity near %d B " + "(%d%% snapshot reserve on %d B provisioned) within %ds " + "(tolerance=%d B, observed=%s, errors=%s)" + % (pool_id, expected_host_bytes, reserve_percent, expected_bytes, + timeout, tolerance, observed, errors) + ) + + def _poll_ontap_volume_size(self, volume_name, expected_bytes, + timeout=120, interval=5): + """Poll ONTAP until the FlexVol space.size equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for ONTAP FlexVol '%s' to report space.size=%d " + "(timeout=%ds, poll every %ds).", + volume_name, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + volume = self.ontap.get_volume(volume_name) + if volume: + current = int(volume.get("space", {}).get("size", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "ONTAP FlexVol '%s' reported space.size=%d after " + "%ds (%d polls).", + volume_name, expected_bytes, elapsed, attempt, + ) + return volume + log_progress( + logger, "info", + "ONTAP capacity poll #%d: FlexVol '%s' space.size=%d " + "(want %d) [elapsed %ds, ~%ds left]", + attempt, volume_name, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d).", + volume_name, expected_bytes, timeout, current, + ) + self.fail( + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d)" + % (volume_name, expected_bytes, timeout, current) + ) + + def _cs_volume_snapshot(self, vol_id): + """Capture id, state, size, and pool so a later check can detect mutation.""" + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "CloudStack volume %s is not listed in listVolumes" % vol_id + ) + pool_id = ( + getattr(vol, "storageid", None) + or getattr(vol, "poolid", None) + ) + return { + "id": getattr(vol, "id", None), + "state": getattr(vol, "state", None), + "size": int(getattr(vol, "size", 0) or 0), + "poolid": pool_id, + } + + def _assert_cs_volume_untouched(self, before, label): + """Assert listVolumes still returns the same id, state, size, and pool.""" + after = self._cs_volume_snapshot(before["id"]) + self.assertEqual( + after["id"], before["id"], + "[%s] CloudStack volume id changed (%s -> %s)" + % (label, before["id"], after["id"]), + ) + self.assertEqual( + after["state"], before["state"], + "[%s] CloudStack volume state changed (%s -> %s)" + % (label, before["state"], after["state"]), + ) + self.assertEqual( + after["size"], before["size"], + "[%s] CloudStack volume size changed (%s -> %s)" + % (label, before["size"], after["size"]), + ) + self.assertEqual( + after["poolid"], before["poolid"], + "[%s] CloudStack volume poolid changed (%s -> %s)" + % (label, before["poolid"], after["poolid"]), + ) + + def _align_flexvol_bytes(self, value): + """Round *value* down to the 4 KiB boundary ONTAP uses for FlexVol size.""" + return (int(value) // 4096) * 4096 + + def _flexvol_used_bytes(self, vol_name): + """ONTAP physical used bytes on the FlexVol (used + reserved).""" + ontap_vol = self.ontap.get_volume(vol_name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol '%s' not found" % vol_name + ) + space = ontap_vol.get("space") or {} + return int(space.get("used") or 0) + + def _fill_flexvol_above_minimum(self, vol_name, timeout=90): + """Write a FlexVol file until ONTAP used space exceeds the minimum. + + Returns ``(filler_filename_or_None, used_bytes)``. The caller must + delete any returned filename. No CloudStack volume or VM is created. + """ + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + log_progress( + logger, "info", + "FlexVol '%s' already has %d B used (ONTAP FlexVol minimum " + "%d B); no filler file needed", + vol_name, used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + return None, used + log_progress( + logger, "info", + "FlexVol '%s' has %d B used; writing %d B filler file '%s'", + vol_name, used, self.FILLER_SIZE, self.FILLER_FILENAME, + ) + self.ontap.write_file_in_volume( + vol_name, self.FILLER_FILENAME, self.FILLER_SIZE + ) + deadline = time.time() + timeout + while time.time() < deadline: + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + break + time.sleep(2) + self.assertGreater( + used, self.ONTAP_MIN_FLEXVOL_SIZE, + "FlexVol '%s' still has only %d B used after writing a %d B " + "filler file; cannot exceed the ONTAP FlexVol minimum of %d B" + % (vol_name, used, self.FILLER_SIZE, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + log_progress( + logger, "info", + "FlexVol '%s' has %d B used after filler file '%s'", + vol_name, used, self.FILLER_FILENAME, + ) + return self.FILLER_FILENAME, used + + def _delete_filler_file(self, vol_name, filename): + if not filename or not vol_name: + return + try: + self.ontap.delete_file_in_volume(vol_name, filename) + except Exception as exc: + logger.warning( + "could not delete filler file '%s' in FlexVol '%s': %s", + filename, vol_name, exc, + ) + if self.__class__.filler_filename == filename: + self.__class__.filler_filename = None + self.__class__.filler_flexvol = None + + def _shrink_target_below_used(self, used_bytes): + """4 KiB-aligned size below used_bytes but above the FlexVol min.""" + used_bytes = int(used_bytes) + self.assertGreater( + used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE, + "Used capacity %d B is not above the ONTAP FlexVol " + "minimum %d B; cannot distinguish a used-capacity reject " + "from a minimum-size reject" + % (used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + target = self._align_flexvol_bytes(used_bytes - 4096) + if target <= self.ONTAP_MIN_FLEXVOL_SIZE: + target = self._align_flexvol_bytes( + (used_bytes + self.ONTAP_MIN_FLEXVOL_SIZE) // 2 + ) + self.assertGreater( + target, self.ONTAP_MIN_FLEXVOL_SIZE, + "Shrink target %d B is not above the ONTAP FlexVol minimum %d B" + % (target, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + self.assertLess( + target, used_bytes, + "Shrink target %d B must be below used capacity %d B" + % (target, used_bytes), + ) + return target + + def _assert_capacity_update_rejected( + self, cmd_pool_id, capacitybytes, label, verify_pool_id, + volume_name, expected_error=None): + """Assert updateStoragePool(capacitybytes) fails and sizes stay put. + + CloudStack and ONTAP are compared against their own pre-request values + because the two do not have to agree: a FlexVol created with a snapshot + reserve reports a larger space.size than the usable capacity + CloudStack records. + """ + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] listStoragePools returned no result for pool %s" + % (label, verify_pool_id), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol '%s' not found" % (label, volume_name), + ) + before_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + before_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + log_progress( + logger, "info", + "Negative resize %s: pool_id=%s capacitybytes=%s " + "(expect reject; CS=%d B ONTAP=%d B)", + label, cmd_pool_id, capacitybytes, before_cs, before_ontap, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = cmd_pool_id + cmd.capacitybytes = capacitybytes + with self.assertRaises(CloudstackAPIException) as caught: + self.apiClient.updateStoragePool(cmd) + error_text = str(caught.exception) + log_progress( + logger, "info", "Rejected resize %s: %s", label, error_text, + ) + if expected_error: + needles = ( + expected_error + if isinstance(expected_error, (list, tuple)) + else (expected_error,) + ) + self.assertTrue( + any(needle in error_text for needle in needles), + "[%s] expected the rejection to report one of %r, got: %s" + % (label, needles, error_text), + ) + + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] pool disappeared after rejected resize" % label, + ) + after_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + self.assertEqual( + after_cs, before_cs, + "[%s] CloudStack capacity changed after rejected resize " + "(got %d, want %d)" % (label, after_cs, before_cs), + ) + self.assertEqual( + listed[0].state, "Up", + "[%s] pool should remain Up after rejected resize, got '%s'" + % (label, listed[0].state), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol disappeared after rejected resize" % label, + ) + after_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + self.assertEqual( + after_ontap, before_ontap, + "[%s] ONTAP FlexVol size changed after rejected resize " + "(got %d, want %d)" % (label, after_ontap, before_ontap), + ) + + def _create_volume(self, pool_id): + """Create a data volume on the given pool; uses _vol_name_prefix.""" + cmd = createVolumeAPI.createVolumeCmd() + cmd.name = "%s_%d" % (self._vol_name_prefix, random.randint(0, 99999)) + cmd.diskofferingid = self.disk_offering_id + cmd.zoneid = self.zone.id + cmd.storageid = pool_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + return self.apiClient.createVolume(cmd) + + def _resize_volume(self, volume_id, size_gib, shrink_ok=False): + """Resize a volume to ``size_gib`` through the CloudStack API.""" + cmd = resizeVolumeAPI.resizeVolumeCmd() + cmd.id = volume_id + cmd.size = size_gib + if shrink_ok: + cmd.shrinkok = True + return self.apiClient.resizeVolume(cmd) + + def _poll_volume_size(self, volume_id, expected_bytes, timeout=180, + interval=5): + """Poll until a volume is Ready with the expected size in bytes.""" + deadline = time.time() + timeout + last_size = None + last_state = "not listed" + while time.time() < deadline: + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = volume_id + cmd.listall = True + volumes = self.apiClient.listVolumes(cmd) or [] + if volumes: + size = int(getattr(volumes[0], "size", 0) or 0) + state = getattr(volumes[0], "state", "") + last_size = size + last_state = state + if size == expected_bytes and state == "Ready": + return volumes[0] + time.sleep(interval) + self.fail( + "Volume %s did not reach Ready with size %s within %ds " + "(last size=%s, state=%s)" + % (volume_id, expected_bytes, timeout, last_size, last_state) + ) + + def _get_qcow2_virtual_size(self, pool_uuid, volume_path): + """Read a qcow2 virtual size from the KVM host mounting the NFS pool.""" + return kvm_host.get_qcow2_virtual_size( + self.kvm_hosts_ssh_creds, pool_uuid, volume_path) + + def _get_vm_data_disk_capacity(self, instance_name): + """Return the libvirt capacity of the VM's first data disk in bytes.""" + return kvm_host.get_vm_data_disk_capacity( + self.kvm_hosts_ssh_creds, instance_name) + + def _get_db_volume_size(self, volume_uuid): + """Query the cloud DB directly for the size (bytes) stored in the + ``volumes`` table for *volume_uuid*. + + Uses the per-suite ``dbConnection`` set in ``setUpClass`` (consistent + with the pattern in all other ONTAP test suites). + + Returns the integer size, or None if the row is not found. + This validates that the ONTAP driver's ``volumeDao.update()`` call + persisted the new size to the database — independently of the + CloudStack API layer. + """ + db = self.__class__.dbConnection + rows = db.execute( + "SELECT size FROM volumes WHERE uuid = %s AND removed IS NULL", + params=(volume_uuid,), + ) + if not rows: + return None + return int(rows[0][0]) + + def _delete_pool(self, pool_id, forced=False): + """Issue deleteStoragePool for the given pool id.""" + cmd = deleteStoragePoolAPI.deleteStoragePoolCmd() + cmd.id = pool_id + if forced: + cmd.forced = True + self.apiClient.deleteStoragePool(cmd) + + +# --------------------------------------------------------------------------- +# Module-level VM helpers +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _list_vols_cmd(vol_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=120, + interval=5): + """Poll listVirtualMachines until the VM reaches target_state. + + Returns the last VM object seen, which may not be in target_state if the + timeout expires — callers assert on the state themselves. + """ + deadline = time.time() + timeout + vm_obj = None + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) or [] + if vms: + vm_obj = vms[0] + if vm_obj.state == target_state: + return vm_obj + time.sleep(interval) + return vm_obj diff --git a/test/integration/plugins/ontap/requirements.txt b/test/integration/plugins/ontap/requirements.txt new file mode 100644 index 000000000000..50c150856aa0 --- /dev/null +++ b/test/integration/plugins/ontap/requirements.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pinned dependencies for the ONTAP Marvin venv. +# Marvin itself is installed separately, from MARVIN_ARCHIVE or tools/marvin. +# setuptools<81 keeps pkg_resources, which nose needs to find the Marvin plugin. +# pyvmomi 9.x needs Python 3.10+. +# paramiko 4.x dropped SHA-1 key exchange, which the CentOS 5.5 guest sshd requires. +setuptools>=40.3.0,<81 +pyvmomi==8.0.2.0.1 +paramiko>=3.4,<4 diff --git a/test/integration/plugins/ontap/run_tests.sh b/test/integration/plugins/ontap/run_tests.sh new file mode 100755 index 000000000000..5a73adb22553 --- /dev/null +++ b/test/integration/plugins/ontap/run_tests.sh @@ -0,0 +1,431 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Run ONTAP Marvin integration tests by tag or protocol batch. +# Each test file runs individually so sequential test state is preserved. +# +# Usage (from cloudstack repo root): +# bash test/integration/plugins/ontap/run_tests.sh # setup + iscsi + nfs3 +# bash test/integration/plugins/ontap/run_tests.sh iscsi # all iSCSI suites +# bash test/integration/plugins/ontap/run_tests.sh nfs3 # all NFS3 suites +# bash test/integration/plugins/ontap/run_tests.sh both # iscsi then nfs3 +# bash test/integration/plugins/ontap/run_tests.sh nfs3_workflow # single suite + +ONTAP_DIR=test/integration/plugins/ontap +CFG=${ONTAP_DIR}/ontap.cfg +RESULTS_BASE=${ONTAP_DIR}/results +AGGREGATE=${ONTAP_DIR}/aggregate_results.py +NOSE_RUNNER=${ONTAP_DIR}/nose_compat.py +ONTAP_PREREQS=${ONTAP_DIR}/check_ontap_prereqs.py +export PYTHONPATH=${ONTAP_DIR}:${PYTHONPATH:-} +export PYTHONUNBUFFERED=1 +FILTER="${1:-all}" + +if [[ -x ${ONTAP_DIR}/.venv/bin/python ]]; then + PYTHON=${ONTAP_DIR}/.venv/bin/python +else + PYTHON=python3 +fi + +PASS=0 +FAIL=0 +SKIP=0 +SUMMARY_FILE=$(mktemp) +BATCH_SUMMARY="" +BATCH_FAIL=0 +GLOBAL_BATCH_FAIL=0 +RUN_DIR="" +BATCH_PROTOCOL="" +BATCH_START="" +SUITE_RC_FILE=$(mktemp) + +trap 'rm -f "$SUMMARY_FILE" "$SUITE_RC_FILE"; [[ -n "$BATCH_SUMMARY" && -f "$BATCH_SUMMARY" ]] && rm -f "$BATCH_SUMMARY"' EXIT + +# --------------------------------------------------------------------------- +# Protocol suite definitions (label|tag|file) +# Order: pool lifecycle → with volumes → volume lifecycle → zone → VM last +# --------------------------------------------------------------------------- + +ISCSI_SUITES=( + "iSCSI pool lifecycle|iscsi_workflow|${ONTAP_DIR}/iscsi/pool/test_pool_lifecycle.py" + "iSCSI pool with volumes|iscsi_with_volumes|${ONTAP_DIR}/iscsi/pool/test_pool_with_volumes.py" + "iSCSI volume lifecycle|iscsi_volume|${ONTAP_DIR}/iscsi/volume/test_volume_lifecycle.py" + "iSCSI zone-scoped pool|iscsi_zone_pool|${ONTAP_DIR}/iscsi/pool/test_zone_scoped_pool.py" + "iSCSI VM volume workflow|iscsi_vm_workflow|${ONTAP_DIR}/iscsi/instance/test_vm_volume_attach.py" + "iSCSI template cache negative|iscsi_template_cache_negative|${ONTAP_DIR}/iscsi/template/test_template_cache_negative.py" +) + +NFS3_SUITES=( + "NFS3 pool lifecycle|nfs3_workflow|${ONTAP_DIR}/nfs3/pool/test_pool_lifecycle.py" + "NFS3 pool with volumes|nfs3_with_volumes|${ONTAP_DIR}/nfs3/pool/test_pool_with_volumes.py" + "NFS3 volume lifecycle|nfs3_volume|${ONTAP_DIR}/nfs3/volume/test_volume_lifecycle.py" + "NFS3 zone-scoped pool|zone_pool|${ONTAP_DIR}/nfs3/pool/test_zone_scoped_pool.py" + "NFS3 VM volume attach|vm_volume_workflow|${ONTAP_DIR}/nfs3/instance/test_vm_volume_attach.py" + "NFS3 template cache negative|nfs3_template_cache_negative|${ONTAP_DIR}/nfs3/template/test_template_cache_negative.py" +) + +record_results() { + local tag="$1" + local label="$2" + local results_file="$3" + local dest="${4:-$SUMMARY_FILE}" + $PYTHON -c " +import re, sys +tag, label, path = sys.argv[1], sys.argv[2], sys.argv[3] +with open(path, encoding='utf-8') as fh: + for line in fh: + line = line.rstrip('\n') + m = re.search(r'TestName: (\S+) \| Status : (\S+)', line) + if m: + print('%s\t%s\t%s\t%s\t' % (tag, label, m.group(1), m.group(2))) + continue + m = re.match(r'(.+?) \.\.\. SKIP: (.+)$', line) + if m: + name = m.group(1).strip() + if len(name) > 72: + name = name[:69] + '...' + detail = m.group(2).strip() + if len(detail) > 120: + detail = detail[:117] + '...' + print('%s\t%s\t%s\tSKIP\t%s' % (tag, label, name, detail)) +" "$tag" "$label" "$results_file" >> "$dest" +} + +print_final_summary() { + echo "" + echo "================================================================" + echo " TEST SUMMARY" + echo "================================================================" + $PYTHON "$AGGREGATE" --out-dir "$(mktemp -d)" --summary-tsv "$SUMMARY_FILE" --print 2>/dev/null \ + | sed -n '/TEST SUMMARY/,$p' | tail -n +2 +} + +init_batch() { + local protocol="$1" + local parent_dir="${2:-}" + + BATCH_PROTOCOL="$protocol" + BATCH_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + BATCH_FAIL=0 + local stamp + stamp=$(date +"%Y%m%d-%H%M%S") + + if [[ -n "$parent_dir" ]]; then + RUN_DIR="${parent_dir}/${protocol}" + else + RUN_DIR="${RESULTS_BASE}/${stamp}-${protocol}" + fi + + mkdir -p "${RUN_DIR}/suites" + : > "$SUITE_RC_FILE" + BATCH_SUMMARY=$(mktemp) + + if [[ -z "$parent_dir" ]]; then + ln -sfn "$(basename "$RUN_DIR")" "${RESULTS_BASE}/latest-${protocol}" + fi + + echo "" + echo "################################################################" + echo " Protocol batch: $(echo "$protocol" | tr '[:lower:]' '[:upper:]')" + echo " Results: ${RUN_DIR}" + echo "################################################################" +} + +finalize_batch() { + local batch_meta batch_summary + batch_summary=$(mktemp) + $PYTHON -c " +import json, sys +from datetime import datetime, timezone +run_dir, protocol, start, rc_file = sys.argv[1:5] +suites = [] +try: + with open(rc_file) as fh: + for line in fh: + line = line.strip() + if not line: + continue + tag, label, rc = line.split('\t', 2) + suites.append({'tag': tag, 'label': label, 'exitCode': int(rc)}) +except (IOError, OSError): + pass +meta = { + 'protocol': protocol, + 'startedAt': start, + 'finishedAt': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'), + 'resultsDir': run_dir, + 'suites': suites, +} +with open(run_dir + '/run.meta.json', 'w') as fh: + json.dump(meta, fh, indent=2) + fh.write('\n') +print(json.dumps(meta)) +" "$RUN_DIR" "$BATCH_PROTOCOL" "$BATCH_START" "$SUITE_RC_FILE" > "$batch_summary" + + $PYTHON "$AGGREGATE" \ + --out-dir "$RUN_DIR" \ + --summary-tsv "$BATCH_SUMMARY" \ + --meta-json "$batch_summary" \ + --print + + cat "$BATCH_SUMMARY" >> "$SUMMARY_FILE" + + if [[ "$BATCH_FAIL" -ne 0 ]]; then + echo " Batch ${BATCH_PROTOCOL}: FAILED (see ${RUN_DIR}/summary.txt)" + else + echo " Batch ${BATCH_PROTOCOL}: all suites passed" + fi + echo " Artifacts: ${RUN_DIR}/summary.json" + + rm -f "$batch_summary" "$BATCH_SUMMARY" + BATCH_SUMMARY="" +} + +copy_suite_logs() { + local tag="$1" + local log_folder="$2" + local stdout_file="$3" + + local dest="${RUN_DIR}/suites/${tag}" + mkdir -p "$dest" + + if [[ -f "$stdout_file" ]]; then + cp "$stdout_file" "${dest}/stdout.log" + fi + if [[ -n "$log_folder" && -d "$log_folder" ]]; then + [[ -f "${log_folder}/results.txt" ]] && cp "${log_folder}/results.txt" "${dest}/" + [[ -f "${log_folder}/runinfo.txt" ]] && cp "${log_folder}/runinfo.txt" "${dest}/" + fi +} + +should_run_tag() { + local tag="$1" + + case "$FILTER" in + all) + [[ "$tag" != "cleanup_zone" ]] + ;; + both) + [[ "$tag" != "setup_zone" && "$tag" != "cleanup_zone" ]] + ;; + iscsi) + [[ "$tag" == iscsi_* ]] + ;; + nfs3) + [[ "$tag" == nfs3_* || "$tag" == "zone_pool" || "$tag" == "vm_volume_workflow" ]] + ;; + *) + [[ "$FILTER" == "$tag" ]] + ;; + esac +} + +run_group() { + local label="$1" + local tag="$2" + local file="$3" + + if ! should_run_tag "$tag"; then + return 0 + fi + + echo "" + echo "================================================================" + echo " ${label} (tag: ${tag})" + echo "================================================================" + + local out tmpout rc log_folder record_dest + tmpout=$(mktemp) + if [[ -n "$BATCH_SUMMARY" ]]; then + record_dest="$BATCH_SUMMARY" + else + record_dest="$SUMMARY_FILE" + fi + + set +e + $PYTHON "$NOSE_RUNNER" --with-marvin --marvin-config="$CFG" "$file" -a "tags=${tag}" -v -s 2>&1 | tee "$tmpout" + rc=${PIPESTATUS[0]} + set -e + out=$(cat "$tmpout") + + log_folder=$(echo "$out" | grep "Final results are now copied to" | sed 's/.*copied to: //; s/ ===.*//' | tr -d '[:space:]') + log_folder=$($PYTHON -c "import os; print(os.path.realpath('$log_folder'))" 2>/dev/null || echo "") + + if [[ -n "$RUN_DIR" ]]; then + copy_suite_logs "$tag" "$log_folder" "$tmpout" + printf '%s\t%s\t%d\n' "$tag" "$label" "$rc" >> "$SUITE_RC_FILE" + fi + + if [[ -n "$log_folder" && -f "${log_folder}/results.txt" ]]; then + local suite_pass suite_fail suite_skip + record_results "$tag" "$label" "${log_folder}/results.txt" "$record_dest" + while IFS= read -r line; do + echo " $line" + done < <(grep "TestName.*Status" "${log_folder}/results.txt" | grep -v "^===") + suite_pass=$(grep -c "Status : SUCCESS" "${log_folder}/results.txt" 2>/dev/null | tr -d '[:space:]' || echo 0) + suite_fail=$(grep -E "Status : FAIL|Status : EXCEPTION" "${log_folder}/results.txt" 2>/dev/null | wc -l | tr -d '[:space:]' || echo 0) + suite_skip=$(grep -c "\.\.\. SKIP:" "${log_folder}/results.txt" 2>/dev/null | tr -d '[:space:]' || echo 0) + PASS=$((PASS + suite_pass)) + FAIL=$((FAIL + suite_fail)) + SKIP=$((SKIP + suite_skip)) + echo " -> ${suite_pass} passed, ${suite_fail} failed, ${suite_skip} skipped" + else + echo "$out" | grep -E "ERROR|Exception|failed" | head -5 + echo " [could not read results — log folder: ${log_folder:-not found}]" + printf '%s\t%s\t%s\tFAIL\t%s\n' "$tag" "$label" "(suite)" "results not found" >> "$record_dest" + FAIL=$((FAIL + 1)) + fi + + rm -f "$tmpout" + + if [[ "$rc" -ne 0 ]]; then + BATCH_FAIL=$((BATCH_FAIL + 1)) + GLOBAL_BATCH_FAIL=$((GLOBAL_BATCH_FAIL + 1)) + fi + return 0 +} + +run_iscsi_suites() { + local entry label tag file + for entry in "${ISCSI_SUITES[@]}"; do + IFS='|' read -r label tag file <<< "$entry" + run_group "$label" "$tag" "$file" + done +} + +run_nfs3_suites() { + local entry label tag file + for entry in "${NFS3_SUITES[@]}"; do + IFS='|' read -r label tag file <<< "$entry" + run_group "$label" "$tag" "$file" + done +} + +check_ontap_prereqs() { + local protocol="$1" + echo "==> Checking ONTAP ${protocol} prerequisites" + $PYTHON "$ONTAP_PREREQS" "$CFG" "$protocol" +} + +run_protocol_batch() { + local protocol="$1" + local parent_dir="${2:-}" + + check_ontap_prereqs "$protocol" + init_batch "$protocol" "$parent_dir" + + case "$protocol" in + iscsi) run_iscsi_suites ;; + nfs3) run_nfs3_suites ;; + *) + echo "Unknown protocol: $protocol" >&2 + return 1 + ;; + esac + + finalize_batch +} + +run_single_suite_by_tag() { + local want_tag="$1" + local entry label tag file + for entry in "${ISCSI_SUITES[@]}" "${NFS3_SUITES[@]}"; do + IFS='|' read -r label tag file <<< "$entry" + if [[ "$tag" == "$want_tag" ]]; then + if [[ "$want_tag" == iscsi_* ]]; then + check_ontap_prereqs iscsi + else + check_ontap_prereqs nfs3 + fi + run_group "$label" "$tag" "$file" + return 0 + fi + done + return 1 +} + +write_combined_both_summary() { + local both_dir="$1" + $PYTHON "$AGGREGATE" \ + --out-dir "$both_dir" \ + --summary-tsv "$SUMMARY_FILE" \ + --meta-json "{\"filter\":\"both\",\"resultsDir\":\"${both_dir}\"}" \ + --print +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +mkdir -p "$RESULTS_BASE" + +case "$FILTER" in + all) + run_group "Advanced zone setup" "setup_zone" \ + "${ONTAP_DIR}/zone_setup/test_setup_zone.py" + + BOTH_DIR="${RESULTS_BASE}/$(date +"%Y%m%d-%H%M%S")-both" + mkdir -p "$BOTH_DIR" + ln -sfn "$(basename "$BOTH_DIR")" "${RESULTS_BASE}/latest-both" + + run_protocol_batch iscsi "$BOTH_DIR" + run_protocol_batch nfs3 "$BOTH_DIR" + write_combined_both_summary "$BOTH_DIR" + ;; + both) + BOTH_DIR="${RESULTS_BASE}/$(date +"%Y%m%d-%H%M%S")-both" + mkdir -p "$BOTH_DIR" + ln -sfn "$(basename "$BOTH_DIR")" "${RESULTS_BASE}/latest-both" + + run_protocol_batch iscsi "$BOTH_DIR" + run_protocol_batch nfs3 "$BOTH_DIR" + write_combined_both_summary "$BOTH_DIR" + ;; + iscsi) + run_protocol_batch iscsi + ;; + nfs3) + run_protocol_batch nfs3 + ;; + setup_zone) + run_group "Advanced zone setup" "setup_zone" \ + "${ONTAP_DIR}/zone_setup/test_setup_zone.py" + print_final_summary + ;; + cleanup_zone) + run_group "Advanced zone cleanup" "cleanup_zone" \ + "${ONTAP_DIR}/zone_setup/test_cleanup_zone.py" + print_final_summary + ;; + *) + if run_single_suite_by_tag "$FILTER"; then + print_final_summary + else + echo "Unknown filter: $FILTER" >&2 + echo "Use: all | both | iscsi | nfs3 | setup_zone | cleanup_zone | " >&2 + exit 1 + fi + ;; +esac + +echo "" +echo "================================================================" +echo " GRAND TOTAL: ${PASS} passed, ${FAIL} failed, ${SKIP} skipped" +echo "================================================================" + +[[ "$FAIL" -eq 0 && "$GLOBAL_BATCH_FAIL" -eq 0 ]] diff --git a/test/integration/plugins/ontap/setup_env.sh b/test/integration/plugins/ontap/setup_env.sh new file mode 100755 index 000000000000..c0238a1ac1e8 --- /dev/null +++ b/test/integration/plugins/ontap/setup_env.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-time (or repeat) setup for ONTAP Marvin integration tests. +# +# Creates a local venv and installs Marvin + dependencies. CI sets +# MARVIN_ARCHIVE to the version-matched archive produced during packaging. +# A source-tree fallback remains available for standalone development. +# +# Usage (from repo root): +# bash test/integration/plugins/ontap/setup_env.sh + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ONTAP_DIR="${REPO_ROOT}/test/integration/plugins/ontap" +VENV="${ONTAP_DIR}/.venv" +COMMANDS_XML="${REPO_ROOT}/tools/apidoc/target/commands.xml" +MARVIN_API="${REPO_ROOT}/tools/marvin/marvin/cloudstackAPI" +MARVIN_ARCHIVE="${MARVIN_ARCHIVE:-}" +NOSE_RUNNER="${ONTAP_DIR}/nose_compat.py" + +echo "==> Repo root: ${REPO_ROOT}" + +if [[ ! -d "${VENV}" ]]; then + echo "==> Creating Python venv at ${VENV}" + if ! python3 -m venv "${VENV}"; then + echo "Unable to create the Marvin venv; install python3-venv on this host" >&2 + exit 1 + fi +fi + +PIP="${VENV}/bin/pip" +PYTHON="${VENV}/bin/python" + +echo "==> Upgrading pip / setuptools / wheel" +if ! "${PIP}" install --upgrade pip wheel; then + echo "Unable to install Python build tools; verify access to the configured Python package index" >&2 + exit 1 +fi + +# requirements.txt is applied after Marvin. Marvin asks for +# setuptools>=40.3.0, paramiko>=1.13.0 and pyvmomi>=5.5.0, so its install +# would otherwise take the newest of each. +REQ="${ONTAP_DIR}/requirements.txt" + +if [[ -n "${MARVIN_ARCHIVE}" ]]; then + if [[ ! -f "${MARVIN_ARCHIVE}" ]]; then + echo "Marvin archive does not exist: ${MARVIN_ARCHIVE}" >&2 + exit 1 + fi + echo "==> Installing Marvin archive ${MARVIN_ARCHIVE}" + if ! "${PIP}" install --no-compile "${MARVIN_ARCHIVE}"; then + echo "Unable to install Marvin; verify the archive and Python package-index access" >&2 + exit 1 + fi +else + if [[ ! -f "${COMMANDS_XML}" ]]; then + echo "==> Building apidoc (generates commands.xml) — first run may take ~2 min" + (cd "${REPO_ROOT}/tools" && mvn -pl apidoc -am package -DskipTests -q) + fi + + if [[ ! -d "${MARVIN_API}" ]]; then + echo "==> Generating Marvin cloudstackAPI from commands.xml" + (cd "${REPO_ROOT}/tools/marvin/marvin" && \ + "${PYTHON}" codegenerator.py -s "${COMMANDS_XML}") + fi + + echo "==> Installing Marvin from the source tree" + if ! "${PIP}" install --no-compile "${REPO_ROOT}/tools/marvin"; then + echo "Unable to install Marvin; verify the source tree and Python package-index access" >&2 + exit 1 + fi +fi + +echo "==> Installing pinned Python dependencies from requirements.txt" +if ! "${PIP}" install -r "${REQ}"; then + echo "Unable to install pinned Python dependencies; verify access to the configured Python package index" >&2 + exit 1 +fi + +echo "" +echo "==> Verifying installation" +if ! "${PYTHON}" -c "import marvin; print('Marvin OK')"; then + echo "Marvin import verification failed" >&2 + exit 1 +fi +if ! nose_plugins="$("${PYTHON}" "${NOSE_RUNNER}" -p 2>&1)"; then + printf '%s\n' "${nose_plugins}" >&2 + echo "Unable to query installed nose plugins" >&2 + exit 1 +fi +if ! grep -q "Plugin marvin" <<< "${nose_plugins}"; then + printf '%s\n' "${nose_plugins}" >&2 + echo "Marvin nose plugin verification failed" >&2 + exit 1 +fi +echo "Marvin nose plugin OK" +if ! discovery_output="$( + PYTHONPATH="${ONTAP_DIR}:${PYTHONPATH:-}" \ + "${PYTHON}" "${NOSE_RUNNER}" --collect-only \ + "${ONTAP_DIR}/zone_setup/test_setup_zone.py" \ + -a "tags=setup_zone" 2>&1 +)"; then + printf '%s\n' "${discovery_output}" >&2 + echo "Nose test discovery verification failed" >&2 + exit 1 +fi +echo "Nose test discovery compatibility OK" + +echo "" +echo "Done. Guest SSH uses paramiko>=3.4,<4 (CentOS 5.5 sshd is SHA-1 only)." +echo "Activate the venv with:" +echo " source test/integration/plugins/ontap/.venv/bin/activate" +echo "" +echo "Run zone setup tests:" +echo " bash test/integration/plugins/ontap/run_tests.sh setup_zone" +echo "" +echo "Or run manually:" +echo " test/integration/plugins/ontap/.venv/bin/python \\" +echo " test/integration/plugins/ontap/nose_compat.py --with-marvin \\" +echo " --marvin-config=test/integration/plugins/ontap/ontap.cfg \\" +echo " test/integration/plugins/ontap/zone_setup/test_setup_zone.py \\" +echo " -a \"tags=setup_zone\" -v" diff --git a/test/integration/plugins/ontap/zone_setup/__init__.py b/test/integration/plugins/ontap/zone_setup/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/zone_setup/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/zone_setup/test_cleanup_zone.py b/test/integration/plugins/ontap/zone_setup/test_cleanup_zone.py new file mode 100644 index 000000000000..e901c23685f5 --- /dev/null +++ b/test/integration/plugins/ontap/zone_setup/test_cleanup_zone.py @@ -0,0 +1,898 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Full teardown of the Advanced zone created by setup_zone. + +Destroys user VMs, all primary storage pools (NFS + ONTAP), secondary +storage, guest networks, hosts, cluster, pod, physical network, and +the zone itself. Each step is idempotent (skips when the resource is +already gone). + +Tag: cleanup_zone + +Usage (from cloudstack root): + bash test/integration/plugins/ontap/run_tests.sh cleanup_zone + +Warning: destructive — not run as part of ``run_tests.sh all``. +""" + +import logging +import time +import unittest +from urllib.parse import urlparse + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance as cancelStorageMaintenanceAPI, + deleteCluster as deleteClusterAPI, + deleteHost as deleteHostAPI, + deleteImageStore as deleteImageStoreAPI, + deleteNetwork as deleteNetworkAPI, + deletePhysicalNetwork as deletePhysicalNetworkAPI, + deletePod as deletePodAPI, + deleteStoragePool as deleteStoragePoolAPI, + deleteVlanIpRange as deleteVlanIpRangeAPI, + deleteVolume as deleteVolumeAPI, + deleteZone as deleteZoneAPI, + destroyRouter as destroyRouterAPI, + destroySystemVm as destroySystemVmAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + destroyVolume as destroyVolumeAPI, + enableStorageMaintenance as enableStorageMaintenanceAPI, + listClusters as listClustersAPI, + listHosts as listHostsAPI, + listImageStores as listImageStoresAPI, + listNetworks as listNetworksAPI, + listPhysicalNetworks as listPhysicalNetworksAPI, + listPods as listPodsAPI, + listPublicIpAddresses as listPublicIpAddressesAPI, + listRouters as listRoutersAPI, + listStoragePools as listStoragePoolsAPI, + listSystemVms as listSystemVmsAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + listVlanIpRanges as listVlanIpRangesAPI, + releaseIpAddress as releaseIpAddressAPI, + stopVirtualMachine as stopVirtualMachineAPI, + updatePhysicalNetwork as updatePhysicalNetworkAPI, + updateStoragePool as updateStoragePoolAPI, + updateZone as updateZoneAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.codes import FAILED +from marvin.jsonHelper import jsonDump +from marvin.lib.common import get_zone, list_storage_pools +from marvin.sshClient import SshClient + +from ontap_test_base import OntapRestClient, enable_live_logging + +logger = logging.getLogger("TestAdvancedZoneCleanup") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _list_all_zone_volumes(api_client, zone_id): + """List user and system VM volumes in the zone (deduped by id).""" + seen = {} + for list_system in (False, True): + cmd = listVolumesAPI.listVolumesCmd() + cmd.zoneid = zone_id + cmd.listall = True + if list_system: + cmd.listsystemvms = True + for vol in api_client.listVolumes(cmd) or []: + seen[vol.id] = vol + return list(seen.values()) + + +def _purge_volume(api_client, vol): + """Delete or expunge a single volume.""" + state = (getattr(vol, "state", "") or "").lower() + if state in ("expunged",): + return + if state in ("destroy", "destroyed", "expunging"): + dc = destroyVolumeAPI.destroyVolumeCmd() + dc.id = vol.id + dc.expunge = True + api_client.destroyVolume(dc) + else: + try: + dc = deleteVolumeAPI.deleteVolumeCmd() + dc.id = vol.id + api_client.deleteVolume(dc) + except CloudstackAPIException: + dc = destroyVolumeAPI.destroyVolumeCmd() + dc.id = vol.id + dc.expunge = True + api_client.destroyVolume(dc) + logger.info( + "Purged volume %s (%s) state=%s." + % (vol.id, getattr(vol, "name", ""), state) + ) + + +def _purge_all_zone_volumes(api_client, zone_id): + """Remove every volume CloudStack still tracks for the zone.""" + volumes = _list_all_zone_volumes(api_client, zone_id) + for vol in volumes: + try: + _purge_volume(api_client, vol) + except CloudstackAPIException as ex: + logger.warning( + "Could not purge volume %s: %s" % (vol.id, ex) + ) + remaining = _list_all_zone_volumes(api_client, zone_id) + return remaining + + +def _parse_kvm_ssh_creds(config): + """Return SSH credential dicts for KVM hosts from ontap.cfg zones block.""" + creds = [] + for zone in config.get("zones", []): + for pod in zone.get("pods", []): + for cluster in pod.get("clusters", []): + for host_cfg in cluster.get("hosts", []): + host_ip = urlparse(host_cfg.get("url", "")).hostname or "" + if host_ip: + creds.append({ + "host": host_ip, + "user": host_cfg.get("username", "root"), + "password": host_cfg.get("password", ""), + }) + return creds + + +def _cleanup_kvm_storage_pool_mounts(pool_uuid, kvm_creds): + """Unmount and undefine libvirt NFS pool on each KVM host.""" + for creds in kvm_creds: + host_ip = creds["host"] + try: + ssh = SshClient( + host_ip, 22, + creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + for cmd in [ + "umount -f -l /mnt/{u} 2>/dev/null; true".format(u=pool_uuid), + "virsh pool-destroy {u} 2>/dev/null; true".format(u=pool_uuid), + "virsh pool-undefine {u} 2>/dev/null; true".format(u=pool_uuid), + ]: + try: + ssh.execute(cmd) + except Exception as cmd_ex: + logger.warning( + "KVM cleanup cmd '%s' failed on %s: %s" + % (cmd, host_ip, cmd_ex) + ) + except Exception as ex: + logger.warning("KVM cleanup SSH to %s failed: %s" % (host_ip, ex)) + + +def _pool_provider(pool): + return (getattr(pool, "provider", "") or "").upper() + + +def _is_ontap_pool(pool): + return "ONTAP" in _pool_provider(pool) + + +def _list_pools_in_zone(api_client, zone_id): + cmd = listStoragePoolsAPI.listStoragePoolsCmd() + cmd.zoneid = zone_id + return api_client.listStoragePools(cmd) or [] + + +def _delete_volumes_on_pool(api_client, pool_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.listall = True + cmd.storagepoolid = pool_id + volumes = api_client.listVolumes(cmd) or [] + for vol in volumes: + try: + dc = deleteVolumeAPI.deleteVolumeCmd() + dc.id = vol.id + api_client.deleteVolume(dc) + logger.info("Deleted volume %s on pool %s." % (vol.id, pool_id)) + except CloudstackAPIException as ex: + logger.warning( + "Could not delete volume %s on pool %s: %s" + % (vol.id, pool_id, ex) + ) + + +def _force_delete_pool( + api_client, pool, ontap_client=None, kvm_creds=None, svm_name=None): + """ + Delete a storage pool: volumes → maintenance → forced delete. + Optional ONTAP REST cleanup on failure; optional KVM NFS unmount. + """ + pool_id = pool.id + pool_name = pool.name + pools = list_storage_pools(api_client, id=pool_id) + if not pools: + logger.info("Pool '%s' already gone." % pool_name) + return True + + pool_state = pools[0].state + + if pool_state == "Maintenance": + try: + cc = cancelStorageMaintenanceAPI.cancelStorageMaintenanceCmd() + cc.id = pool_id + api_client.cancelStorageMaintenance(cc) + time.sleep(5) + except Exception: + pass + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool_id + ec.enabled = True + api_client.updateStoragePool(ec) + time.sleep(3) + except Exception: + pass + pools = list_storage_pools(api_client, id=pool_id) + if pools: + pool_state = pools[0].state + + _delete_volumes_on_pool(api_client, pool_id) + + if pool_state in ("Up", "Disabled"): + try: + mc = enableStorageMaintenanceAPI.enableStorageMaintenanceCmd() + mc.id = pool_id + api_client.enableStorageMaintenance(mc) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(api_client, id=pool_id) + if ps and ps[0].state == "Maintenance": + break + time.sleep(5) + except Exception as ex: + logger.warning( + "Could not enter maintenance for pool '%s': %s" + % (pool_name, ex) + ) + + if kvm_creds: + _cleanup_kvm_storage_pool_mounts(pool_id, kvm_creds) + + try: + dc = deleteStoragePoolAPI.deleteStoragePoolCmd() + dc.id = pool_id + dc.forced = True + api_client.deleteStoragePool(dc) + logger.info("Deleted storage pool '%s' (id=%s)." % (pool_name, pool_id)) + return True + except CloudstackAPIException as ex: + logger.warning( + "deleteStoragePool failed for '%s': %s" % (pool_name, ex) + ) + if ontap_client is not None: + try: + ontap_client.delete_volume(pool_name) + logger.info( + "Deleted ONTAP FlexVol '%s' directly." % pool_name + ) + except Exception as oe: + logger.warning( + "ONTAP FlexVol delete '%s' failed: %s" % (pool_name, oe) + ) + try: + ep_name = "cs-%s-%s" % (svm_name or "", pool_name) + ontap_client.delete_export_policy(ep_name) + logger.info( + "Deleted export policy '%s' directly." % ep_name + ) + except Exception: + pass + pools = list_storage_pools(api_client, id=pool_id) + if not pools: + return True + return False + + +def _stop_and_destroy_vm(api_client, vm): + state = (getattr(vm, "state", "") or "").lower() + if state == "running": + try: + sc = stopVirtualMachineAPI.stopVirtualMachineCmd() + sc.id = vm.id + api_client.stopVirtualMachine(sc) + deadline = time.time() + 120 + while time.time() < deadline: + lcmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + lcmd.id = vm.id + cur = api_client.listVirtualMachines(lcmd) or [] + if cur and cur[0].state.lower() in ("stopped", "destroyed"): + break + time.sleep(5) + except CloudstackAPIException as ex: + logger.warning("Could not stop VM %s: %s" % (vm.id, ex)) + + dc = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dc.id = vm.id + dc.expunge = True + api_client.destroyVirtualMachine(dc) + logger.info("Destroyed VM '%s' (id=%s)." % (vm.name, vm.id)) + + +def _destroy_system_vms_and_routers(api_client, zone_id): + """Destroy SSVM, console proxy, and virtual routers in the zone.""" + sys_cmd = listSystemVmsAPI.listSystemVmsCmd() + sys_cmd.zoneid = zone_id + sysvms = api_client.listSystemVms(sys_cmd) or [] + for svm in sysvms: + try: + dc = destroySystemVmAPI.destroySystemVmCmd() + dc.id = svm.id + api_client.destroySystemVm(dc) + logger.info( + "Destroyed system VM %s type=%s id=%s" + % (svm.name, svm.systemvmtype, svm.id) + ) + except CloudstackAPIException as ex: + logger.warning( + "Could not destroy system VM %s: %s" % (svm.id, ex) + ) + + router_cmd = listRoutersAPI.listRoutersCmd() + router_cmd.zoneid = zone_id + router_cmd.listall = True + routers = api_client.listRouters(router_cmd) or [] + for router in routers: + try: + dc = destroyRouterAPI.destroyRouterCmd() + dc.id = router.id + api_client.destroyRouter(dc) + logger.info("Destroyed router %s id=%s." % (router.name, router.id)) + except CloudstackAPIException as ex: + logger.warning( + "Could not destroy router %s: %s" % (router.id, ex) + ) + + +def _release_zone_public_ips(api_client, zone_id): + """Release all allocated public IPs in the zone.""" + cmd = listPublicIpAddressesAPI.listPublicIpAddressesCmd() + cmd.zoneid = zone_id + cmd.listall = True + cmd.allocatedonly = True + ips = api_client.listPublicIpAddresses(cmd) or [] + for ip in ips: + try: + rc = releaseIpAddressAPI.releaseIpAddressCmd() + rc.id = ip.id + api_client.releaseIpAddress(rc) + logger.info( + "Released public IP %s (id=%s)." + % (getattr(ip, "ipaddress", ip.id), ip.id) + ) + except CloudstackAPIException as ex: + logger.warning( + "Could not release public IP %s: %s" % (ip.id, ex) + ) + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +@attr(tags=["cleanup_zone"]) +class TestAdvancedZoneCleanup(cloudstackTestCase): + """ + Tears down the full Advanced zone from ontap.cfg. + Idempotent: skips steps when resources are already removed. + """ + + _zone_id = None + _zone_name = None + _phynet_id = None + _pod_id = None + _cluster_id = None + + _zcfg = {} + _pcfg = {} + _ccfg = {} + _kvm_creds = [] + _ontap_client = None + _svm_name = None + + @classmethod + def setUpClass(cls): + enable_live_logging(cls) + testclient = super(TestAdvancedZoneCleanup, cls).getClsTestClient() + cls.apiClient = testclient.getApiClient() + + if not getattr(cls, "config", None): + raise RuntimeError( + "Marvin datacenter config not available. Run with:\n" + " --marvin-config=test/integration/plugins/ontap/ontap.cfg" + ) + config = jsonDump.dump(cls.config) + + zone_cfgs = config.get("zones", []) + if not zone_cfgs: + raise unittest.SkipTest("No zones block in ontap.cfg — nothing to clean up.") + + cls._zcfg = zone_cfgs[0] + pods = cls._zcfg.get("pods", []) + cls._pcfg = pods[0] if pods else {} + clusters = cls._pcfg.get("clusters", []) if cls._pcfg else [] + cls._ccfg = clusters[0] if clusters else {} + + cs_cfg = config.get("cloudstack", {}) + cls._zone_name = cs_cfg.get("zoneName") or cls._zcfg.get("name") + cls._kvm_creds = _parse_kvm_ssh_creds(config) + + ontap_cfg = config.get("ontap", {}) + if ontap_cfg.get("storageIP"): + cls._ontap_client = OntapRestClient( + ontap_cfg["storageIP"], + ontap_cfg.get("username", "admin"), + ontap_cfg.get("password", ""), + ) + cls._svm_name = ontap_cfg.get("svmName", "") + + existing = get_zone(cls.apiClient, zone_name=cls._zone_name) + if not existing or existing == FAILED: + raise unittest.SkipTest( + "Zone '%s' not found — nothing to clean up." % cls._zone_name + ) + + cls._zone_id = existing.id + cls._resolve_resources() + + @classmethod + def _resolve_resources(cls): + zone_id = cls._zone_id + pod_name = cls._pcfg.get("name") + cluster_name = cls._ccfg.get("clustername") + + pod_cmd = listPodsAPI.listPodsCmd() + pod_cmd.zoneid = zone_id + for pod in cls.apiClient.listPods(pod_cmd) or []: + if not pod_name or pod.name == pod_name: + cls._pod_id = pod.id + break + + cluster_cmd = listClustersAPI.listClustersCmd() + cluster_cmd.zoneid = zone_id + if cls._pod_id: + cluster_cmd.podid = cls._pod_id + for cluster in cls.apiClient.listClusters(cluster_cmd) or []: + if not cluster_name or cluster.name == cluster_name: + cls._cluster_id = cluster.id + break + + pnet_cmd = listPhysicalNetworksAPI.listPhysicalNetworksCmd() + pnet_cmd.zoneid = zone_id + pnets = cls.apiClient.listPhysicalNetworks(pnet_cmd) or [] + if pnets: + cls._phynet_id = pnets[0].id + + def setUp(self): + pass + + # ----------------------------------------------------------------------- + # Step 01 – disable zone + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_01_disable_zone(self): + """Disable the zone before removing resources.""" + zone_id = self.__class__._zone_id + cmd = updateZoneAPI.updateZoneCmd() + cmd.id = zone_id + cmd.allocationstate = "Disabled" + try: + ret = self.apiClient.updateZone(cmd) + except CloudstackAPIException as ex: + if "disabled" in str(ex).lower(): + logger.info("Zone already disabled.") + return + raise + self.assertIsNotNone(ret) + logger.info("Zone id=%s disabled." % zone_id) + + # ----------------------------------------------------------------------- + # Step 02 – destroy user VMs + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_02_destroy_user_vms(self): + """Destroy all non-system user VMs in the zone.""" + zone_id = self.__class__._zone_id + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.zoneid = zone_id + cmd.listall = True + vms = self.apiClient.listVirtualMachines(cmd) or [] + + user_vms = [ + vm for vm in vms + if (getattr(vm, "account", "") or "").lower() != "system" + and getattr(vm, "state", "").lower() + not in ("destroyed", "expunging", "error") + ] + if not user_vms: + logger.info("No user VMs to destroy in zone.") + return + + for vm in user_vms: + try: + _stop_and_destroy_vm(self.apiClient, vm) + except CloudstackAPIException as ex: + logger.warning( + "Could not destroy VM %s (%s): %s" + % (vm.id, vm.name, ex) + ) + + # ----------------------------------------------------------------------- + # Step 02b – destroy system VMs and routers + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_02_system_vms_destroy(self): + """Destroy system VMs (SSVM, console proxy) and virtual routers.""" + _destroy_system_vms_and_routers( + self.apiClient, self.__class__._zone_id + ) + + # ----------------------------------------------------------------------- + # Step 03 – delete volumes + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_03_delete_volumes(self): + """Delete remaining volumes in the zone (user + system VM volumes).""" + remaining = _purge_all_zone_volumes( + self.apiClient, self.__class__._zone_id + ) + if remaining: + logger.warning( + "%d volume(s) still present after purge." % len(remaining) + ) + + # ----------------------------------------------------------------------- + # Step 04 – delete primary storage (NFS / non-ONTAP) + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_04_delete_primary_storage(self): + """Delete NFS primary pools from config and any remaining non-ONTAP pools.""" + zone_id = self.__class__._zone_id + all_pools = _list_pools_in_zone(self.apiClient, zone_id) + targets = [p for p in all_pools if not _is_ontap_pool(p)] + + if not targets: + logger.info("No primary (non-ONTAP) storage pools to delete.") + return + + for pool in targets: + _force_delete_pool( + self.apiClient, pool, + kvm_creds=self.__class__._kvm_creds, + ) + + # ----------------------------------------------------------------------- + # Step 05 – delete ONTAP pools + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_05_delete_ontap_pools(self): + """Delete NetApp ONTAP primary pools left by integration tests.""" + zone_id = self.__class__._zone_id + ontap_pools = [ + p for p in _list_pools_in_zone(self.apiClient, zone_id) + if _is_ontap_pool(p) + ] + if not ontap_pools: + logger.info("No ONTAP storage pools to delete.") + return + + for pool in ontap_pools: + _force_delete_pool( + self.apiClient, pool, + ontap_client=self.__class__._ontap_client, + kvm_creds=self.__class__._kvm_creds, + svm_name=self.__class__._svm_name, + ) + + # ----------------------------------------------------------------------- + # Step 06 – delete secondary storage + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_06_delete_secondary_storage(self): + """Delete all secondary/image stores in the zone.""" + zone_id = self.__class__._zone_id + cmd = listImageStoresAPI.listImageStoresCmd() + cmd.zoneid = zone_id + stores = self.apiClient.listImageStores(cmd) or [] + if not stores: + logger.info("No image stores to delete in zone.") + return + + for store in stores: + store_url = getattr(store, "url", "") or "" + try: + dc = deleteImageStoreAPI.deleteImageStoreCmd() + dc.id = store.id + self.apiClient.deleteImageStore(dc) + logger.info( + "Deleted image store '%s' (id=%s)." % (store_url, store.id) + ) + except CloudstackAPIException as ex: + if "not found" in str(ex).lower(): + logger.info("Image store already gone.") + else: + logger.warning( + "Could not delete image store %s: %s" + % (store.id, ex) + ) + + # ----------------------------------------------------------------------- + # Step 07 – delete guest networks + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_07_delete_guest_networks(self): + """Delete isolated/guest networks (skip system networks).""" + zone_id = self.__class__._zone_id + cmd = listNetworksAPI.listNetworksCmd() + cmd.zoneid = zone_id + cmd.listall = True + networks = self.apiClient.listNetworks(cmd) or [] + + skip_types = frozenset({"system", "shared", "l2vlan"}) + for net in networks: + net_type = (getattr(net, "type", "") or "").lower() + if net_type in skip_types: + continue + if getattr(net, "issystem", False): + continue + try: + dc = deleteNetworkAPI.deleteNetworkCmd() + dc.id = net.id + self.apiClient.deleteNetwork(dc) + logger.info( + "Deleted network '%s' (id=%s)." % (net.name, net.id) + ) + except CloudstackAPIException as ex: + logger.warning( + "Could not delete network %s: %s" % (net.id, ex) + ) + + # ----------------------------------------------------------------------- + # Step 08 – delete hosts + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_08_delete_hosts(self): + """Remove routing hosts from the zone.""" + zone_id = self.__class__._zone_id + cmd = listHostsAPI.listHostsCmd() + cmd.zoneid = zone_id + cmd.type = "Routing" + hosts = self.apiClient.listHosts(cmd) or [] + if not hosts: + logger.info("No routing hosts to delete.") + return + + for host in hosts: + try: + dc = deleteHostAPI.deleteHostCmd() + dc.id = host.id + dc.forced = True + self.apiClient.deleteHost(dc) + logger.info("Deleted host '%s' (id=%s)." % (host.name, host.id)) + except CloudstackAPIException as ex: + logger.warning( + "Could not delete host %s: %s" % (host.id, ex) + ) + + # ----------------------------------------------------------------------- + # Step 09 – delete cluster + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_09_delete_cluster(self): + """Delete the cluster from config.""" + zone_id = self.__class__._zone_id + cluster_id = self.__class__._cluster_id + cluster_name = self.__class__._ccfg.get("clustername") + + if not cluster_id and cluster_name: + cmd = listClustersAPI.listClustersCmd() + cmd.zoneid = zone_id + for c in self.apiClient.listClusters(cmd) or []: + if c.name == cluster_name: + cluster_id = c.id + break + + if not cluster_id: + logger.info("No cluster to delete.") + return + + try: + dc = deleteClusterAPI.deleteClusterCmd() + dc.id = cluster_id + self.apiClient.deleteCluster(dc) + logger.info("Deleted cluster id=%s." % cluster_id) + except CloudstackAPIException as ex: + if "not found" in str(ex).lower(): + logger.info("Cluster already gone.") + else: + logger.warning("Could not delete cluster %s: %s" % (cluster_id, ex)) + + # ----------------------------------------------------------------------- + # Step 10 – delete pod + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_10_delete_pod(self): + """Delete the pod from config.""" + zone_id = self.__class__._zone_id + pod_id = self.__class__._pod_id + pod_name = self.__class__._pcfg.get("name") + + if not pod_id and pod_name: + cmd = listPodsAPI.listPodsCmd() + cmd.zoneid = zone_id + for p in self.apiClient.listPods(cmd) or []: + if p.name == pod_name: + pod_id = p.id + break + + if not pod_id: + logger.info("No pod to delete.") + return + + try: + dc = deletePodAPI.deletePodCmd() + dc.id = pod_id + self.apiClient.deletePod(dc) + logger.info("Deleted pod id=%s." % pod_id) + except CloudstackAPIException as ex: + if "not found" in str(ex).lower(): + logger.info("Pod already gone.") + else: + logger.warning("Could not delete pod %s: %s" % (pod_id, ex)) + + # ----------------------------------------------------------------------- + # Step 10a – release public IPs + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_10a_release_public_ips(self): + """Release allocated public IPs before deleting VLAN ranges.""" + _release_zone_public_ips(self.apiClient, self.__class__._zone_id) + + # ----------------------------------------------------------------------- + # Step 11 – delete public IP ranges + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_11_delete_public_ip_ranges(self): + """Delete VLAN IP ranges on the physical network.""" + phynet_id = self.__class__._phynet_id + if not phynet_id: + logger.info("No physical network — skipping IP range deletion.") + return + + cmd = listVlanIpRangesAPI.listVlanIpRangesCmd() + cmd.physicalnetworkid = phynet_id + ranges = self.apiClient.listVlanIpRanges(cmd) or [] + if not ranges: + logger.info("No public IP ranges to delete.") + return + + for ipr in ranges: + try: + dc = deleteVlanIpRangeAPI.deleteVlanIpRangeCmd() + dc.id = ipr.id + self.apiClient.deleteVlanIpRange(dc) + logger.info("Deleted IP range id=%s." % ipr.id) + except CloudstackAPIException as ex: + logger.warning( + "Could not delete IP range %s: %s" % (ipr.id, ex) + ) + + # ----------------------------------------------------------------------- + # Step 12 – delete physical network + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_12_delete_physical_network(self): + """Disable and delete the physical network.""" + phynet_id = self.__class__._phynet_id + if not phynet_id: + logger.info("No physical network to delete.") + return + + try: + up = updatePhysicalNetworkAPI.updatePhysicalNetworkCmd() + up.id = phynet_id + up.state = "Disabled" + self.apiClient.updatePhysicalNetwork(up) + except CloudstackAPIException as ex: + logger.warning( + "Could not disable physical network %s: %s" % (phynet_id, ex) + ) + + try: + dc = deletePhysicalNetworkAPI.deletePhysicalNetworkCmd() + dc.id = phynet_id + self.apiClient.deletePhysicalNetwork(dc) + logger.info("Deleted physical network id=%s." % phynet_id) + except CloudstackAPIException as ex: + if "not found" in str(ex).lower(): + logger.info("Physical network already gone.") + else: + raise + + # ----------------------------------------------------------------------- + # Step 12b – final volume purge before deleteZone + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_12b_purge_zone_volumes(self): + """Expunge any volumes still blocking deleteZone (incl. system VM disks).""" + remaining = _purge_all_zone_volumes( + self.apiClient, self.__class__._zone_id + ) + if remaining: + self.fail( + "%d volume(s) still in zone after final purge: %s" + % (len(remaining), [v.id for v in remaining]) + ) + + # ----------------------------------------------------------------------- + # Step 13 – delete zone + # ----------------------------------------------------------------------- + + @attr(tags=["cleanup_zone"]) + def test_13_delete_zone(self): + """Delete the zone — final step.""" + zone_id = self.__class__._zone_id + zone_name = self.__class__._zone_name + + # Safety net: purge volumes that block deleteZone (e.g. system VM ROOT disks). + _purge_all_zone_volumes(self.apiClient, zone_id) + + try: + dc = deleteZoneAPI.deleteZoneCmd() + dc.id = zone_id + self.apiClient.deleteZone(dc) + except CloudstackAPIException as ex: + self.fail( + "deleteZone failed for '%s' (id=%s): %s\n" + "Ensure all VMs, pools, hosts, and storage are removed first." + % (zone_name, zone_id, ex) + ) + + remaining = get_zone(self.apiClient, zone_name=zone_name) + self.assertTrue( + not remaining or remaining == FAILED, + "Zone '%s' still exists after deleteZone." % zone_name, + ) + logger.info("Zone '%s' deleted." % zone_name) diff --git a/test/integration/plugins/ontap/zone_setup/test_setup_zone.py b/test/integration/plugins/ontap/zone_setup/test_setup_zone.py new file mode 100644 index 000000000000..58f7b69e87ea --- /dev/null +++ b/test/integration/plugins/ontap/zone_setup/test_setup_zone.py @@ -0,0 +1,805 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Advanced zone setup prerequisite for the ONTAP integration test suite. + +Run this before any other test group to ensure the CloudStack zone, pod, +cluster, host, primary storage, and secondary storage all exist. Each +numbered test method creates one step of the zone hierarchy. After the +zone is enabled, steps 11–12 wait for both system VMs to reach Running +and for the configured KVM template to become ready. Creation steps are +idempotent (skipped when the resource already exists); wait steps always run. + +Tag: setup_zone + +Usage (from cloudstack root): + bash test/integration/plugins/ontap/run_tests.sh setup_zone +""" + +import logging +import re +import time + +from nose.plugins.attrib import attr + +from ontap_test_base import enable_live_logging, log_progress + +from marvin.cloudstackAPI import ( + addCluster as addClusterAPI, + addHost as addHostAPI, + addImageStore as addImageStoreAPI, + addTrafficType as addTrafficTypeAPI, + createPhysicalNetwork as createPhysicalNetworkAPI, + createPod as createPodAPI, + createStoragePool as createStoragePoolAPI, + createVlanIpRange as createVlanIpRangeAPI, + createZone as createZoneAPI, + listClusters as listClustersAPI, + listHosts as listHostsAPI, + listNetworkServiceProviders as listNetworkServiceProvidersAPI, + listPhysicalNetworks as listPhysicalNetworksAPI, + listPods as listPodsAPI, + listStoragePools as listStoragePoolsAPI, + listSystemVms as listSystemVmsAPI, + listTemplates as listTemplatesAPI, + listVirtualRouterElements as listVirtualRouterElementsAPI, + configureVirtualRouterElement as configureVirtualRouterElementAPI, + updateNetworkServiceProvider as updateNetworkServiceProviderAPI, + updatePhysicalNetwork as updatePhysicalNetworkAPI, + updateZone as updateZoneAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.codes import FAILED +from marvin.jsonHelper import jsonDump +from marvin.lib.common import get_zone + +logger = logging.getLogger("TestAdvancedZoneSetup") + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _normalize_template_name(name): + """Collapse naming variants so '(64 bit)' matches CloudStack's '(64-bit)'.""" + if not name: + return "" + s = name.lower().strip() + s = re.sub(r"\s+", " ", s) + s = s.replace("(64 bit)", "(64-bit)") + s = s.replace("(64bit)", "(64-bit)") + return s + + +def _list_kvm_templates(api_client, zone_id): + cmd = listTemplatesAPI.listTemplatesCmd() + cmd.templatefilter = "all" + cmd.listall = True + cmd.zoneid = zone_id + templates = api_client.listTemplates(cmd) or [] + return [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + ] + + +def _find_kvm_template(api_client, zone_id, template_name): + """Find a KVM template by normalized name (API name filter is exact-only).""" + kvm_templates = _list_kvm_templates(api_client, zone_id) + target = _normalize_template_name(template_name) + for tmpl in kvm_templates: + if _normalize_template_name(tmpl.name) == target: + return tmpl + return None + + +def _wait_for_hosts_up(api_client, zone_id, cluster_id, timeout=120): + """Poll listHosts until all routing hosts in the cluster are Up.""" + deadline = time.time() + timeout + while time.time() < deadline: + cmd = listHostsAPI.listHostsCmd() + cmd.zoneid = zone_id + cmd.clusterid = cluster_id + cmd.type = "Routing" + hosts = api_client.listHosts(cmd) or [] + if hosts and all(h.state == "Up" for h in hosts): + logger.info("All %d host(s) in cluster are Up." % len(hosts)) + return True + time.sleep(10) + logger.warning( + "_wait_for_hosts_up: hosts did not reach Up state within %ds." % timeout + ) + return False + + +def _cluster_has_up_host(api_client, zone_id, cluster_id): + cmd = listHostsAPI.listHostsCmd() + cmd.zoneid = zone_id + cmd.clusterid = cluster_id + cmd.type = "Routing" + hosts = api_client.listHosts(cmd) or [] + return any(h.state == "Up" for h in hosts) + + +def _wait_for_system_vms(api_client, zone_id, timeout=3600, interval=60): + """Poll listSystemVms until both SSVM and Console Proxy are Running.""" + start = time.time() + deadline = start + timeout + attempt = 0 + log_progress(logger, + "info", + "Waiting for system VMs (SSVM + Console Proxy) in zone %s " + "(timeout=%ds, poll every %ds).", + zone_id, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + + cmd = listSystemVmsAPI.listSystemVmsCmd() + cmd.zoneid = zone_id + all_vms = api_client.listSystemVms(cmd) or [] + running = [v for v in all_vms if v.state == "Running"] + summary = ", ".join( + "%s/%s=%s" % (v.name, v.systemvmtype, v.state) for v in all_vms + ) or "(none yet)" + + log_progress(logger, + "info", + "System VM poll #%d: %d/%d Running (%d total) " + "[elapsed %ds, ~%ds left] — %s", + attempt, len(running), 2, len(all_vms), elapsed, remaining, summary, + ) + + if len(running) >= 2: + for vm in running: + log_progress(logger, + "info", + "System VM Running: name=%s type=%s id=%s", + vm.name, vm.systemvmtype, vm.id, + ) + return True + + time.sleep(interval) + + log_progress(logger, + "error", + "System VMs did not reach Running state within %ds.", timeout, + ) + return False + + +def _wait_for_template_ready( + api_client, zone_id, template_name, timeout=3600, interval=60): + """Poll listTemplates until the named KVM template has isready=True.""" + start = time.time() + deadline = start + timeout + attempt = 0 + log_progress(logger, + "info", + "Waiting for KVM template '%s' in zone %s " + "(timeout=%ds, poll every %ds).", + template_name, zone_id, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + + tmpl = _find_kvm_template(api_client, zone_id, template_name) + if tmpl and getattr(tmpl, "isready", False): + log_progress(logger, + "info", + "Template ready: name=%s id=%s hypervisor=%s " + "(configured as '%s', after %ds, %d polls)", + tmpl.name, tmpl.id, tmpl.hypervisor, + template_name, elapsed, attempt, + ) + return True + + if tmpl: + log_progress(logger, + "info", + "Template poll #%d: matched '%s' (configured '%s') " + "but not ready (isready=%s) [elapsed %ds, ~%ds left]", + attempt, tmpl.name, template_name, + getattr(tmpl, "isready", False), + elapsed, remaining, + ) + else: + kvm_templates = _list_kvm_templates(api_client, zone_id) + kvm_names = [t.name for t in kvm_templates] + log_progress(logger, + "info", + "Template poll #%d: no match for '%s' " + "[elapsed %ds, ~%ds left]", + attempt, template_name, elapsed, remaining, + ) + if attempt == 1 or attempt % 5 == 0: + log_progress(logger, + "warning", + "Configured template '%s' not matched. " + "KVM templates in zone: %s", + template_name, + kvm_names if kvm_names else "(none listed)", + ) + + time.sleep(interval) + + log_progress(logger, + "error", + "Template '%s' not ready within %ds.", template_name, timeout, + ) + return False + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +@attr(tags=["setup_zone"]) +class TestAdvancedZoneSetup(cloudstackTestCase): + """ + Creates a full Advanced-zone infrastructure from ontap.cfg. + Creation steps are idempotent (skipped when resources already exist). + Wait steps (11–12) always run to verify system VMs and template readiness. + """ + + # Class-level state shared across numbered test methods + _zone_exists = False + _zone_id = None + _phynet_id = None + _pod_id = None + _cluster_id = None + + # Raw config dicts read from ontap.cfg + _zcfg = {} # zones[0] + _pcfg = {} # zones[0].pods[0] + _ccfg = {} # zones[0].pods[0].clusters[0] + _cs_cfg = {} # cloudstack section + _template_name = "CentOS 5.5(64-bit) no GUI (KVM)" + _system_vm_timeout = 3600 + _template_ready_timeout = 3600 + _poll_interval = 60 + + @classmethod + def setUpClass(cls): + enable_live_logging(cls) + testclient = super(TestAdvancedZoneSetup, cls).getClsTestClient() + cls.apiClient = testclient.getApiClient() + + # Marvin injects the --marvin-config file as cls.config (parsed JSON). + # getParsedTestDataConfig() defaults to test_data.py and does NOT + # contain the datacenter zones block from ontap.cfg. + if not getattr(cls, "config", None): + raise RuntimeError( + "Marvin datacenter config not available. Run with:\n" + " --marvin-config=test/integration/plugins/ontap/ontap.cfg" + ) + config = jsonDump.dump(cls.config) + + zone_cfgs = config.get("zones", []) + if not zone_cfgs: + raise RuntimeError( + "ontap.cfg is missing a 'zones' entry. " + "Add zone creation fields as described in the README." + ) + + cls._zcfg = zone_cfgs[0] + pods = cls._zcfg.get("pods", []) + cls._pcfg = pods[0] if pods else {} + clusters = cls._pcfg.get("clusters", []) if cls._pcfg else [] + cls._ccfg = clusters[0] if clusters else {} + + cls._cs_cfg = config.get("cloudstack", {}) + cls._template_name = cls._cs_cfg.get( + "templateName", cls._template_name + ) + cls._system_vm_timeout = cls._cs_cfg.get("systemVmTimeoutSec", 3600) + cls._template_ready_timeout = cls._cs_cfg.get( + "templateReadyTimeoutSec", 3600 + ) + cls._poll_interval = cls._cs_cfg.get("pollIntervalSec", 60) + + zone_name = cls._zcfg.get("name") + required = {"name", "networktype", "dns1", "internaldns1"} + missing = required - cls._zcfg.keys() + if missing: + raise RuntimeError( + "ontap.cfg zones[0] is missing required creation fields: %s" + % sorted(missing) + ) + + existing = get_zone(cls.apiClient, zone_name=zone_name) + if existing and existing != FAILED: + logger.info( + "Zone '%s' already exists (id=%s) — skipping test_01 only." + % (zone_name, existing.id) + ) + cls._zone_exists = True + cls._zone_id = existing.id + cls._resolve_existing_resources() + + @classmethod + def _resolve_existing_resources(cls): + """Populate pod/cluster/phynet ids when re-running against an existing zone.""" + zone_id = cls._zone_id + pod_name = cls._pcfg.get("name") + cluster_name = cls._ccfg.get("clustername") + + pod_cmd = listPodsAPI.listPodsCmd() + pod_cmd.zoneid = zone_id + pods = cls.apiClient.listPods(pod_cmd) or [] + for pod in pods: + if not pod_name or pod.name == pod_name: + cls._pod_id = pod.id + break + + cluster_cmd = listClustersAPI.listClustersCmd() + cluster_cmd.zoneid = zone_id + if cls._pod_id: + cluster_cmd.podid = cls._pod_id + clusters = cls.apiClient.listClusters(cluster_cmd) or [] + for cluster in clusters: + if not cluster_name or cluster.name == cluster_name: + cls._cluster_id = cluster.id + break + + pnet_cmd = listPhysicalNetworksAPI.listPhysicalNetworksCmd() + pnet_cmd.zoneid = zone_id + pnets = cls.apiClient.listPhysicalNetworks(pnet_cmd) or [] + if pnets: + cls._phynet_id = pnets[0].id + + def setUp(self): + pass + + # ----------------------------------------------------------------------- + # Step 1 – zone + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_01_create_zone(self): + """Create the Advanced zone.""" + if self.__class__._zone_exists: + self.skipTest( + "Zone '%s' already exists (id=%s)." + % (self.__class__._zcfg.get("name"), self.__class__._zone_id) + ) + + zcfg = self.__class__._zcfg + + cmd = createZoneAPI.createZoneCmd() + cmd.name = zcfg["name"] + cmd.networktype = zcfg["networktype"] + cmd.dns1 = zcfg["dns1"] + cmd.dns2 = zcfg.get("dns2", "") + cmd.internaldns1 = zcfg["internaldns1"] + cmd.internaldns2 = zcfg.get("internaldns2", "") + cmd.localstorageenabled = zcfg.get("localstorageenabled", False) + cmd.guestcidraddress = zcfg.get("guestcidraddress", "10.1.1.0/24") + + zone = self.apiClient.createZone(cmd) + self.assertIsNotNone(zone, "createZone returned None") + self.assertIsNotNone(zone.id, "Zone id is None after createZone") + + self.__class__._zone_id = zone.id + logger.info("Zone '%s' created with id=%s." % (zcfg["name"], zone.id)) + + # ----------------------------------------------------------------------- + # Step 2 – physical network + traffic types + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_02_create_physical_network(self): + """Create a single physical network with Guest, Management, and Public traffic types.""" + if self.__class__._phynet_id: + self.skipTest( + "Physical network already exists (id=%s)." % self.__class__._phynet_id + ) + + zone_id = self.__class__._zone_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + cmd = createPhysicalNetworkAPI.createPhysicalNetworkCmd() + cmd.zoneid = zone_id + cmd.name = "PhysNet1" + cmd.isolationmethods = "VLAN" + + phynet = self.apiClient.createPhysicalNetwork(cmd) + self.assertIsNotNone(phynet, "createPhysicalNetwork returned None") + self.assertIsNotNone(phynet.id, "Physical network id is None") + + pnet_id = phynet.id + self.__class__._phynet_id = pnet_id + logger.info("Physical network created with id=%s." % pnet_id) + + for traffic_type in ("Guest", "Management", "Public"): + tt_cmd = addTrafficTypeAPI.addTrafficTypeCmd() + tt_cmd.physicalnetworkid = pnet_id + tt_cmd.traffictype = traffic_type + ret = self.apiClient.addTrafficType(tt_cmd) + self.assertIsNotNone(ret, "addTrafficType returned None for %s" % traffic_type) + logger.info("Traffic type '%s' added." % traffic_type) + + # ----------------------------------------------------------------------- + # Step 3 – configure VR providers + enable physical network + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_03_configure_providers_and_enable_network(self): + """Enable VirtualRouter and VpcVirtualRouter providers; set VLAN range; enable network.""" + pnet_id = self.__class__._phynet_id + self.assertIsNotNone(pnet_id, "phynet_id not set — did test_02 pass?") + + vlan_range = self.__class__._zcfg.get("guestVlanRange", "100-300") + + for provider_name in ("VirtualRouter", "VpcVirtualRouter"): + list_cmd = listNetworkServiceProvidersAPI.listNetworkServiceProvidersCmd() + list_cmd.physicalnetworkid = pnet_id + list_cmd.name = provider_name + providers = self.apiClient.listNetworkServiceProviders(list_cmd) or [] + + if not providers: + logger.warning( + "Provider '%s' not found on physical network %s — skipping." + % (provider_name, pnet_id) + ) + continue + + provider = providers[0] + + # Configure the VirtualRouter element (enable it) + vr_cmd = listVirtualRouterElementsAPI.listVirtualRouterElementsCmd() + vr_cmd.nspid = provider.id + vr_elements = self.apiClient.listVirtualRouterElements(vr_cmd) or [] + if vr_elements: + cfg_cmd = configureVirtualRouterElementAPI.configureVirtualRouterElementCmd() + cfg_cmd.id = vr_elements[0].id + cfg_cmd.enabled = "true" + self.apiClient.configureVirtualRouterElement(cfg_cmd) + logger.info("VR element for '%s' configured." % provider_name) + + # Enable the provider + upd_cmd = updateNetworkServiceProviderAPI.updateNetworkServiceProviderCmd() + upd_cmd.id = provider.id + upd_cmd.state = "Enabled" + self.apiClient.updateNetworkServiceProvider(upd_cmd) + logger.info("Provider '%s' enabled." % provider_name) + + # Enable physical network and set guest VLAN range + upnet_cmd = updatePhysicalNetworkAPI.updatePhysicalNetworkCmd() + upnet_cmd.id = pnet_id + upnet_cmd.state = "Enabled" + upnet_cmd.vlan = vlan_range + self.apiClient.updatePhysicalNetwork(upnet_cmd) + logger.info( + "Physical network %s enabled with VLAN range %s." % (pnet_id, vlan_range) + ) + + # ----------------------------------------------------------------------- + # Step 4 – public IP range + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_04_create_public_ip_range(self): + """Create the public traffic IP range.""" + zone_id = self.__class__._zone_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + ipr = self.__class__._zcfg.get("publicIpRange", {}) + if not ipr: + self.skipTest("No publicIpRange defined in ontap.cfg — skipping.") + + cmd = createVlanIpRangeAPI.createVlanIpRangeCmd() + cmd.zoneid = zone_id + cmd.gateway = ipr["gateway"] + cmd.netmask = ipr["netmask"] + cmd.startip = ipr["startip"] + cmd.endip = ipr["endip"] + cmd.vlan = ipr.get("vlan", "untagged") + cmd.forvirtualnetwork = "true" + + try: + ret = self.apiClient.createVlanIpRange(cmd) + except CloudstackAPIException as ex: + if "overlap" in str(ex).lower() or "already" in str(ex).lower(): + self.skipTest("Public IP range already exists: %s" % ex) + raise + self.assertIsNotNone(ret, "createVlanIpRange returned None") + logger.info( + "Public IP range %s–%s created." % (ipr["startip"], ipr["endip"]) + ) + + # ----------------------------------------------------------------------- + # Step 5 – pod + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_05_create_pod(self): + """Create the management pod with reserved system IPs.""" + if self.__class__._pod_id: + self.skipTest("Pod already exists (id=%s)." % self.__class__._pod_id) + + zone_id = self.__class__._zone_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + pcfg = self.__class__._pcfg + if not pcfg: + self.skipTest("No pod config found in ontap.cfg — skipping.") + + cmd = createPodAPI.createPodCmd() + cmd.zoneid = zone_id + cmd.name = pcfg["name"] + cmd.gateway = pcfg["gateway"] + cmd.netmask = pcfg["netmask"] + cmd.startip = pcfg["startip"] + cmd.endip = pcfg["endip"] + + pod = self.apiClient.createPod(cmd) + self.assertIsNotNone(pod, "createPod returned None") + self.assertIsNotNone(pod.id, "Pod id is None") + + self.__class__._pod_id = pod.id + logger.info("Pod '%s' created with id=%s." % (pcfg["name"], pod.id)) + + # ----------------------------------------------------------------------- + # Step 6 – cluster + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_06_add_cluster(self): + """Add the KVM cluster.""" + if self.__class__._cluster_id: + self.skipTest( + "Cluster already exists (id=%s)." % self.__class__._cluster_id + ) + + zone_id = self.__class__._zone_id + pod_id = self.__class__._pod_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + self.assertIsNotNone(pod_id, "pod_id not set — did test_05 pass?") + + ccfg = self.__class__._ccfg + if not ccfg: + self.skipTest("No cluster config found in ontap.cfg — skipping.") + + cmd = addClusterAPI.addClusterCmd() + cmd.zoneid = zone_id + cmd.podid = pod_id + cmd.clustername = ccfg["clustername"] + cmd.clustertype = ccfg.get("clustertype", "CloudManaged") + cmd.hypervisor = ccfg.get("hypervisor", "KVM") + + clusters = self.apiClient.addCluster(cmd) + self.assertTrue( + clusters and len(clusters) > 0, "addCluster returned empty response" + ) + cluster_id = clusters[0].id + self.__class__._cluster_id = cluster_id + logger.info( + "Cluster '%s' added with id=%s." % (ccfg["clustername"], cluster_id) + ) + + # ----------------------------------------------------------------------- + # Step 7 – host(s) + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_07_add_host(self): + """Add KVM host(s) to the cluster and wait for them to come Up.""" + zone_id = self.__class__._zone_id + pod_id = self.__class__._pod_id + cluster_id = self.__class__._cluster_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + self.assertIsNotNone(pod_id, "pod_id not set — did test_05 pass?") + self.assertIsNotNone(cluster_id, "cluster_id not set — did test_06 pass?") + + hosts_cfg = self.__class__._ccfg.get("hosts", []) + hypervisor = self.__class__._ccfg.get("hypervisor", "KVM") + + if not hosts_cfg: + self.skipTest("No host config found in ontap.cfg — skipping.") + + if _cluster_has_up_host(self.apiClient, zone_id, cluster_id): + self.skipTest("Cluster already has at least one Up routing host.") + + for hcfg in hosts_cfg: + cmd = addHostAPI.addHostCmd() + cmd.zoneid = zone_id + cmd.podid = pod_id + cmd.clusterid = cluster_id + cmd.hypervisor = hypervisor + cmd.url = hcfg["url"] + cmd.username = hcfg["username"] + cmd.password = hcfg["password"] + if hcfg.get("hosttags"): + cmd.hosttags = hcfg["hosttags"] + + try: + ret = self.apiClient.addHost(cmd) + except CloudstackAPIException as ex: + self.skipTest( + "addHost failed for %s — verify SSH from the management " + "server and host credentials in ontap.cfg: %s" + % (hcfg["url"], ex) + ) + self.assertTrue( + ret and len(ret) > 0, + "addHost returned empty response for %s" % hcfg["url"], + ) + logger.info("Host '%s' added." % hcfg["url"]) + + if not _wait_for_hosts_up(self.apiClient, zone_id, cluster_id, timeout=120): + self.skipTest( + "Host(s) were added but did not reach Up state within 120s." + ) + + # ----------------------------------------------------------------------- + # Step 8 – primary storage + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_08_create_primary_storage(self): + """Create NFS primary storage pool (cluster-scoped).""" + zone_id = self.__class__._zone_id + pod_id = self.__class__._pod_id + cluster_id = self.__class__._cluster_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + self.assertIsNotNone(pod_id, "pod_id not set — did test_05 pass?") + self.assertIsNotNone(cluster_id, "cluster_id not set — did test_06 pass?") + + primary_storages = self.__class__._ccfg.get("primaryStorages", []) + if not primary_storages: + self.skipTest("No primaryStorages defined in cluster config — skipping.") + + if not _cluster_has_up_host(self.apiClient, zone_id, cluster_id): + self.skipTest( + "No Up routing host in cluster — primary storage requires a " + "connected KVM host (see test_07_add_host)." + ) + + for pscfg in primary_storages: + pool_cmd = listStoragePoolsAPI.listStoragePoolsCmd() + pool_cmd.zoneid = zone_id + pool_cmd.name = pscfg["name"] + existing = self.apiClient.listStoragePools(pool_cmd) or [] + if existing: + logger.info( + "Primary storage '%s' already exists (id=%s) — skipping." + % (pscfg["name"], existing[0].id) + ) + continue + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.zoneid = zone_id + cmd.name = pscfg["name"] + cmd.url = pscfg["url"] + cmd.scope = pscfg.get("scope", "Cluster") + if cmd.scope.lower() == "cluster": + cmd.podid = pod_id + cmd.clusterid = cluster_id + if pscfg.get("tags"): + cmd.tags = pscfg["tags"] + + ret = self.apiClient.createStoragePool(cmd) + self.assertIsNotNone(ret, "createStoragePool returned None for '%s'" % pscfg["name"]) + logger.info("Primary storage '%s' created with id=%s." % (pscfg["name"], ret.id)) + + # ----------------------------------------------------------------------- + # Step 9 – secondary storage + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_09_add_secondary_storage(self): + """Add NFS secondary storage (image store).""" + zone_id = self.__class__._zone_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + secondary_storages = self.__class__._zcfg.get("secondaryStorages", []) + if not secondary_storages: + self.skipTest("No secondaryStorages defined in zones[0] config — skipping.") + + for sscfg in secondary_storages: + cmd = addImageStoreAPI.addImageStoreCmd() + cmd.provider = sscfg.get("provider", "NFS") + cmd.url = sscfg["url"] + cmd.zoneid = zone_id + + try: + ret = self.apiClient.addImageStore(cmd) + except CloudstackAPIException as ex: + if "already exists" in str(ex).lower(): + logger.info( + "Secondary storage '%s' already exists — skipping." + % sscfg.get("url") + ) + continue + raise + self.assertIsNotNone(ret, "addImageStore returned None for '%s'" % sscfg.get("name")) + logger.info( + "Secondary storage '%s' added with id=%s." + % (sscfg.get("name", ret.id), ret.id) + ) + + # ----------------------------------------------------------------------- + # Step 10 – enable zone + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_10_enable_zone(self): + """Enable the zone (allocationstate=Enabled).""" + zone_id = self.__class__._zone_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + cmd = updateZoneAPI.updateZoneCmd() + cmd.id = zone_id + cmd.allocationstate = "Enabled" + ret = self.apiClient.updateZone(cmd) + + self.assertIsNotNone(ret, "updateZone returned None") + logger.info( + "Zone id=%s enabled (allocationstate=Enabled)." % zone_id + ) + + # ----------------------------------------------------------------------- + # Step 11 – wait for system VMs + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_11_wait_for_system_vms(self): + """Wait until both system VMs (SSVM + Console Proxy) are Running.""" + zone_id = self.__class__._zone_id + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + ok = _wait_for_system_vms( + self.apiClient, + zone_id, + timeout=self.__class__._system_vm_timeout, + interval=self.__class__._poll_interval, + ) + self.assertTrue( + ok, + "Both system VMs did not reach Running within %ds." + % self.__class__._system_vm_timeout, + ) + + # ----------------------------------------------------------------------- + # Step 12 – wait for KVM template + # ----------------------------------------------------------------------- + + @attr(tags=["setup_zone"]) + def test_12_wait_for_kvm_template(self): + """Wait until the configured KVM template is ready (isready=True).""" + zone_id = self.__class__._zone_id + template_name = self.__class__._template_name + self.assertIsNotNone(zone_id, "zone_id not set — did test_01 pass?") + + ok = _wait_for_template_ready( + self.apiClient, + zone_id, + template_name, + timeout=self.__class__._template_ready_timeout, + interval=self.__class__._poll_interval, + ) + self.assertTrue( + ok, + "Template '%s' not ready within %ds." + % (template_name, self.__class__._template_ready_timeout), + ) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 7c2c9f09eae4..df95b01438fb 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -216,7 +216,7 @@ export default { show: (record) => { return record.hypervisor !== 'External' && (((['Running'].includes(record.state) && record.hypervisor !== 'LXC') || (['Stopped'].includes(record.state) && (!['KVM', 'LXC'].includes(record.hypervisor) || - (record.hypervisor === 'KVM' && ['PowerFlex', 'Filesystem', 'NetworkFilesystem', 'SharedMountPoint'].includes(record.pooltype))))) && record.vmtype !== 'sharedfsvm') + (record.hypervisor === 'KVM' && ['PowerFlex', 'Filesystem', 'NetworkFilesystem', 'SharedMountPoint', 'OntapiSCSI'].includes(record.pooltype))))) && record.vmtype !== 'sharedfsvm') }, disabled: (record) => { return record.hostcontrolstate === 'Offline' && record.hypervisor === 'KVM' }, mapping: { diff --git a/ui/src/views/infra/AddPrimaryStorage.vue b/ui/src/views/infra/AddPrimaryStorage.vue index 7d189032f098..e5d9e6aa81af 100644 --- a/ui/src/views/infra/AddPrimaryStorage.vue +++ b/ui/src/views/infra/AddPrimaryStorage.vue @@ -301,6 +301,12 @@ + + + +
diff --git a/ui/src/views/infra/zone/ZoneWizardAddResources.vue b/ui/src/views/infra/zone/ZoneWizardAddResources.vue index 9bd9c6d37aef..faf9a10679f6 100644 --- a/ui/src/views/infra/zone/ZoneWizardAddResources.vue +++ b/ui/src/views/infra/zone/ZoneWizardAddResources.vue @@ -611,7 +611,7 @@ export default { title: 'label.capacityiops', key: 'capacityIops', hidden: { - provider: ['DefaultPrimary', 'PowerFlex', 'Linstor', 'NetApp ONTAP'] + provider: ['DefaultPrimary', 'PowerFlex', 'Linstor'] } }, { diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index f21201572dff..9402fe99a9c9 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -1619,6 +1619,9 @@ export default { if (this.prefillContent.capacityBytes && this.prefillContent.capacityBytes.length > 0) { params.capacityBytes = this.prefillContent.capacityBytes.split(',').join('') } + if (this.prefillContent.capacityIops && this.prefillContent.capacityIops.length > 0) { + params.capacityIops = this.prefillContent.capacityIops.split(',').join('') + } } params.tags = this.prefillContent?.primaryStorageTags || '' diff --git a/ui/src/views/storage/ChangeOfferingForVolume.vue b/ui/src/views/storage/ChangeOfferingForVolume.vue index 66a3cf1442e4..29d00dfe512e 100644 --- a/ui/src/views/storage/ChangeOfferingForVolume.vue +++ b/ui/src/views/storage/ChangeOfferingForVolume.vue @@ -158,7 +158,7 @@ export default { miniops: [{ type: 'number', validator: async (rule, value) => { - if (value && (isNaN(value) || value <= 0)) { + if (value && (isNaN(value) || value < 0)) { return Promise.reject(this.$t('message.error.number')) } return Promise.resolve() diff --git a/ui/src/views/storage/CreateVolume.vue b/ui/src/views/storage/CreateVolume.vue index ae203bdfa9b8..073ff204bba1 100644 --- a/ui/src/views/storage/CreateVolume.vue +++ b/ui/src/views/storage/CreateVolume.vue @@ -311,7 +311,7 @@ export default { size: [{ required: true, message: this.$t('message.error.custom.disk.size') }], miniops: [{ validator: async (rule, value) => { - if (value && (isNaN(value) || value <= 0)) { + if (value && (isNaN(value) || value < 0)) { return Promise.reject(this.$t('message.error.number')) } return Promise.resolve() diff --git a/ui/src/views/storage/ResizeVolume.vue b/ui/src/views/storage/ResizeVolume.vue index 5f9efd6ed506..c017816f1004 100644 --- a/ui/src/views/storage/ResizeVolume.vue +++ b/ui/src/views/storage/ResizeVolume.vue @@ -65,7 +65,7 @@