diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index f2af46d4cc8a..835a79770a49 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -120,7 +120,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); final Connect conn = libvirtUtilitiesHelper.getConnection(); - final StorageVol v = conn.storageVolLookupByPath(path); + final StorageVol v = libvirtUtilitiesHelper.getStorageVolumeByPath(conn, pool, path); int flags = 0; if (conn.getLibVirVersion() > 1001000 && vol.getFormat() == PhysicalDiskFormat.RAW && pool.getType() != StoragePoolType.RBD) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java index 0a37c971a571..10dcf744f656 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java @@ -30,11 +30,14 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.libvirt.Connect; +import org.libvirt.Error.ErrorNumber; import org.libvirt.LibvirtException; +import org.libvirt.StorageVol; import com.cloud.agent.api.VMSnapshotTO; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; import com.cloud.storage.template.Processor; import com.cloud.storage.template.QCOW2Processor; @@ -156,4 +159,31 @@ public KVMStoragePool getPrimaryPoolFromDataTo(DataTO dataTO, KVMStoragePoolMana return storageManager.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); } + /** + * Looks a volume up through libvirt by its path. + * + * An RBD volume is created through librbd, and KVMStoragePool.getPhysicalDisk() looks RBD + * volumes up through librbd too, so nothing may have made libvirt refresh the pool since the + * volume was created, and the volume may be missing from libvirt's cache of the pool on this + * host. For an RBD pool, a lookup that fails because the volume is not found refreshes the + * pool and looks the volume up once more. Any other failure, and a failure on any other pool + * type, is thrown to the caller. + */ + public StorageVol getStorageVolumeByPath(final Connect conn, final KVMStoragePool pool, final String path) throws LibvirtException { + try { + return conn.storageVolLookupByPath(path); + } catch (final LibvirtException e) { + if (pool.getType() != StoragePoolType.RBD || e.getError() == null + || e.getError().getCode() != ErrorNumber.VIR_ERR_NO_STORAGE_VOL) { + throw e; + } + LOGGER.info("Volume " + path + " is not in libvirt's cache of RBD pool " + pool.getUuid() + " (" + e.getMessage() + + "). Refreshing the pool and looking the volume up again."); + if (!pool.refresh()) { + LOGGER.warn("Failed to refresh RBD pool " + pool.getUuid() + ". Looking volume " + path + " up again anyway."); + } + return conn.storageVolLookupByPath(path); + } + } + } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index fa204f806261..8664e1a55ca2 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -92,6 +92,7 @@ import org.libvirt.DomainInfo; import org.libvirt.DomainInfo.DomainState; import org.libvirt.DomainInterfaceStats; +import org.libvirt.Error.ErrorNumber; import org.libvirt.LibvirtException; import org.libvirt.MemoryStatistic; import org.libvirt.NodeInfo; @@ -101,6 +102,7 @@ import org.libvirt.jna.virDomainMemoryStats; import org.mockito.ArgumentCaptor; import org.mockito.BDDMockito; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; @@ -175,6 +177,7 @@ import com.cloud.agent.api.storage.CreateCommand; import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; +import com.cloud.agent.api.storage.ResizeVolumeAnswer; import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DiskTO; @@ -4803,7 +4806,7 @@ public void testResizeVolumeCommand() { when(libvirtComputingResourceMock.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); try { when(libvirtUtilitiesHelper.getConnection()).thenReturn(conn); - when(conn.storageVolLookupByPath(path)).thenReturn(v); + when(libvirtUtilitiesHelper.getStorageVolumeByPath(conn, storagePool, path)).thenReturn(v); when(libvirtUtilitiesHelper.getConnectionByVmName(vmInstance)).thenReturn(conn); when(conn.domainLookupByName(vmInstance)).thenReturn(vm); when(vm.getInfo()).thenReturn(info); @@ -4830,6 +4833,103 @@ public void testResizeVolumeCommand() { } } + private static final String RBD_RESIZE_PATH = "cloudstack/0d3c2bd6-cb39-4c8b-a1d9-0e7fbd3b8a5b"; + + /** + * Sets up a resize of a RAW volume at {@link #RBD_RESIZE_PATH} on an RBD pool, up to the point + * where the wrapper looks the volume up through {@code libvirtUtilitiesHelper} on {@code conn}. + */ + private ResizeVolumeCommand prepareRbdResizeLookup(final KVMStoragePool storagePool, final KVMPhysicalDisk vol, + final LibvirtUtilitiesHelper libvirtUtilitiesHelper, final Connect conn) throws LibvirtException { + final StorageFilerTO pool = Mockito.mock(StorageFilerTO.class); + final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); + + when(pool.getType()).thenReturn(StoragePoolType.RBD); + when(libvirtComputingResourceMock.getStoragePoolMgr()).thenReturn(storagePoolMgr); + when(storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid())).thenReturn(storagePool); + when(storagePool.getPhysicalDisk(RBD_RESIZE_PATH)).thenReturn(vol); + when(storagePool.getType()).thenReturn(StoragePoolType.RBD); + when(vol.getPath()).thenReturn(RBD_RESIZE_PATH); + when(vol.getFormat()).thenReturn(PhysicalDiskFormat.RAW); + when(libvirtComputingResourceMock.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + Mockito.doReturn(conn).when(libvirtUtilitiesHelper).getConnection(); + // Only reached when the resize gets as far as telling the VM about the new size. + Mockito.lenient().doReturn(conn).when(libvirtUtilitiesHelper).getConnectionByVmName("Test"); + + return new ResizeVolumeCommand(RBD_RESIZE_PATH, pool, 100L, 200L, false, "Test", null); + } + + /** + * An RBD volume created through librbd is not in libvirt's cache of the pool until the pool is + * refreshed, so the first lookup by path misses. The resize looks the volume up through + * LibvirtUtilitiesHelper.getStorageVolumeByPath(), which refreshes the pool and finds it. The + * resize must then resize the volume and tell the running VM about the new size. + */ + @Test + public void testResizeVolumeCommandRbdVolumeNotInLibvirtPoolCache() throws LibvirtException { + final KVMStoragePool storagePool = Mockito.mock(KVMStoragePool.class); + final KVMPhysicalDisk vol = Mockito.mock(KVMPhysicalDisk.class); + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.spy(LibvirtUtilitiesHelper.class); + final Connect conn = Mockito.mock(Connect.class); + final StorageVol v = Mockito.mock(StorageVol.class); + final Domain vm = Mockito.mock(Domain.class); + final DomainInfo info = Mockito.mock(DomainInfo.class); + info.state = DomainInfo.DomainState.VIR_DOMAIN_RUNNING; + + final ResizeVolumeCommand command = prepareRbdResizeLookup(storagePool, vol, libvirtUtilitiesHelper, conn); + final org.libvirt.Error error = Mockito.mock(org.libvirt.Error.class); + when(error.getCode()).thenReturn(ErrorNumber.VIR_ERR_NO_STORAGE_VOL); + final LibvirtException notFound = Mockito.mock(LibvirtException.class); + when(notFound.getError()).thenReturn(error); + when(conn.storageVolLookupByPath(RBD_RESIZE_PATH)).thenThrow(notFound).thenReturn(v); + when(storagePool.refresh()).thenReturn(true); + when(conn.getLibVirVersion()).thenReturn(10010L); + when(conn.domainLookupByName("Test")).thenReturn(vm); + when(vm.getInfo()).thenReturn(info); + when(vol.getVirtualSize()).thenReturn(200L); + + // The resize script succeeds (a mocked Script.execute() returns null), so the answer comes from the size read back after the resize. + try (MockedConstruction