FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

kvm: fill the template's holes before taking its LUKS clone snapshot by calvix · Pull Request #14656 · apache/cloudstack · GitHub

Repository navigation

Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .java  (6) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
          • QemuImg.java
          • RbdEncryption.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -1401,7 +1401,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
&& srcPool.getSourceHost().equals(destPool.getSourceHost())
&& srcPool.getSourceDir().equals(destPool.getSourceDir());
if (sameClusterRbd) {
return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase);
return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase, timeout);
}
return createEncryptedRootFullCopy(srcPool, template, destPool, newUuid, disk, passphrase);
}
Expand Down Expand Up @@ -1562,6 +1562,64 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
return disk;
}

/**
* Create and protect a snapshot of a template image. Fine if another host created it at the same time.
*/
private void createProtectedSnapshot(RbdImage templateImage, String snapshotName) throws RbdException {
try {
templateImage.snapCreate(snapshotName);
templateImage.snapProtect(snapshotName);
} catch (RbdException e) {
if (!templateImage.snapIsProtected(snapshotName)) {
throw e;
}
}
}

/**
* Prepare the template snapshot encrypted roots are cloned from, once per template per pool: grow the
* template by {@link #LUKS2_HEADER_RESERVE_BYTES}, rewrite it without holes and take the snapshot.
* Nothing to do if the snapshot exists. If this fails before the snapshot is taken, the next deploy
* from the template starts over.
*
* @param timeout limit for the rewrite of the template, which writes its whole virtual size
*/
void prepareLuksTemplateSnapshot(RbdImage templateImage, KVMPhysicalDisk template, KVMStoragePool destPool,
String luksSnapshotName, long timeout) throws RbdException {
boolean luksSnapshotExists = false;
boolean baseSnapshotExists = false;
for (RbdSnapInfo snapshotInfo : templateImage.snapList()) {
if (luksSnapshotName.equals(snapshotInfo.name)) {
luksSnapshotExists = true;
} else if (rbdTemplateSnapName.equals(snapshotInfo.name)) {
baseSnapshotExists = true;
}
}
if (luksSnapshotExists) {
return;
}
// the rewrite below reads the template from its base snapshot, which only the plaintext clone path
// creates: take it here when the first deploy from this template is encrypted
if (!baseSnapshotExists) {
createProtectedSnapshot(templateImage, rbdTemplateSnapName);
}
templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES);
// fill the template's holes first: an encrypted clone reads a hole in its parent as
// garbage once a guest write copies up the object around it
createRbdEncryption().copyTemplateDense(destPool.getSourceHost(), destPool.getSourcePort(),
destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(),
template.getName(), rbdTemplateSnapName, timeout);
createProtectedSnapshot(templateImage, luksSnapshotName);
logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksSnapshotName);
}

/**
* Seam for unit tests.
*/
protected RbdEncryption createRbdEncryption() {
return new RbdEncryption();
}

/**
* Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster
* as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to
Expand All @@ -1572,8 +1630,9 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
* @return the encrypted CoW clone, or {@code null} if the Ceph operations failed
*/
private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KVMStoragePool destPool,
String newUuid, KVMPhysicalDisk disk, byte[] passphrase) {
String luksReservedSnapshotName = rbdTemplateSnapName + "-luks";
String newUuid, KVMPhysicalDisk disk, byte[] passphrase, long timeout) {
// not "-luks": templates prepared before carry a snapshot of that name that still has holes
String luksReservedSnapshotName = rbdTemplateSnapName + "-luks2";
Rados radosConnection = null;
IoCTX ioContext = null;
Rbd rbdClient = null;
Expand All @@ -1587,19 +1646,7 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV
ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir());
rbdClient = new Rbd(ioContext);
templateImage = rbdClient.open(template.getName());
boolean luksSnapshotExists = false;
for (RbdSnapInfo snapshotInfo : templateImage.snapList()) {
if (luksReservedSnapshotName.equals(snapshotInfo.name)) {
luksSnapshotExists = true;
break;
}
}
if (!luksSnapshotExists) {
templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES);
templateImage.snapCreate(luksReservedSnapshotName);
templateImage.snapProtect(luksReservedSnapshotName);
logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName);
}
prepareLuksTemplateSnapshot(templateImage, template, destPool, luksReservedSnapshotName, timeout);
rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder);
} catch (RadosException | RbdException e) {
logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage());
Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public class QemuImg {
private String cloudQemuImgPath = "cloud-qemu-img";
private long timeout;
private boolean skipZero = false;
private boolean writeZeroRanges = false;
private boolean skipTargetVolumeCreation = false;
private boolean noCache = false;
private long version;
Expand Down Expand Up @@ -522,6 +523,11 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI
script.add("-n");
}

if (writeZeroRanges) {
script.add("-S");
script.add("0");
}

if (destImageOpts == null) {
script.add("-O");
script.add(destFile.getFormat().toString());
Expand Down Expand Up @@ -1025,6 +1031,14 @@ public void setSkipZero(boolean skipZero) {
this.skipZero = skipZero;
}

/**
* Make convert write the source's zero ranges to the destination ({@code -S 0}) instead of
* leaving them unallocated.
*/
public void setWriteZeroRanges(boolean writeZeroRanges) {
this.writeZeroRanges = writeZeroRanges;
}

public void setSkipTargetVolumeCreation(boolean skipTargetVolumeCreation) {
this.skipTargetVolumeCreation = skipTargetVolumeCreation;
}
Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ public void importTemplate(String srcRbdPool, String srcRbdImage,
secretParams.put(QemuObject.ObjectParameter.FILE, passFile.toString());
QemuObject luksSecret = new QemuObject(QemuObject.ObjectType.SECRET, secretParams);

QemuImg qemu = createQemuImg();
QemuImg qemu = createQemuImg(0);
qemu.convertIntoExistingTarget(srcQemuFile, null, List.of(luksSecret), srcImageOpts, destImageOpts, forceSourceFormat);
logger.debug("Imported template into encrypted RBD image {}", imageSpec);
} catch (IOException | QemuImgException | LibvirtException ex) {
Expand All @@ -250,11 +250,49 @@ public void importTemplate(String srcRbdPool, String srcRbdImage,
}
}

/**
* Rewrite an RBD image from one of its snapshots, writing out the zero ranges too ({@code -S 0}),
* so the image has no holes. Used on a template before taking the snapshot encrypted roots are
* cloned from: a hole in that parent reads as garbage through the clone's encryption once a
* guest write copies up the object around it.
*
* @param timeout qemu-img timeout in milliseconds; the copy writes the whole virtual size of the image
*/
public void copyTemplateDense(String monHost, int monPort, String authUser, String authSecret,
String cephPool, String image, String srcSnapshot, long timeout) {
Path conf = null;
Path keyring = null;
try {
final FileAttribute<?> ownerOnly = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------"));
keyring = Files.createTempFile("cs-ceph-", ".keyring", ownerOnly);
Files.writeString(keyring, "[client." + authUser + "]\n\tkey = " + authSecret + "\n");
conf = Files.createTempFile("cs-ceph-", ".conf", ownerOnly);
Files.writeString(conf, "[global]\nmon_host = " + monSpec(monHost, monPort) + "\nkeyring = " + keyring + "\n");

QemuImgFile srcQemuFile = new QemuImgFile(cephPool + "/" + image + "@" + srcSnapshot, QemuImg.PhysicalDiskFormat.RAW);
Map<String, String> srcParams = rbdImageOptions(cephPool, image, conf.toString(), authUser);
srcParams.put("snapshot", srcSnapshot);
QemuImageOptions srcImageOpts = new QemuImageOptions(srcParams);
srcImageOpts.setImageOptsFlag(true);
QemuImageOptions destImageOpts = new QemuImageOptions(rbdImageOptions(cephPool, image, conf.toString(), authUser));

QemuImg qemu = createQemuImg(timeout);
qemu.setWriteZeroRanges(true);
qemu.convertIntoExistingTarget(srcQemuFile, null, null, srcImageOpts, destImageOpts, false);
logger.debug("Rewrote RBD image {}/{} from @{} without holes", cephPool, image, srcSnapshot);
} catch (IOException | QemuImgException | LibvirtException ex) {
throw new CloudRuntimeException(String.format("Failed to rewrite RBD image %s/%s from @%s", cephPool, image, srcSnapshot), ex);
} finally {
deleteQuietly(conf);
deleteQuietly(keyring);
}
}

/**
* Seam for unit tests; {@link QemuImg} probes the qemu version through libvirt on construction.
*/
protected QemuImg createQemuImg() throws QemuImgException, LibvirtException {
return new QemuImg(0);
protected QemuImg createQemuImg(long timeout) throws QemuImgException, LibvirtException {
return new QemuImg(timeout);
}

/** qemu image options addressing an RBD image (as used with --image-opts / --target-image-opts). */
Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,21 @@
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.apache.cloudstack.utils.rbd.RbdEncryption;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.libvirt.Connect;
import org.libvirt.StoragePool;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
Expand All @@ -40,6 +45,9 @@
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;

import com.ceph.rbd.RbdException;
import com.ceph.rbd.RbdImage;
import com.ceph.rbd.jna.RbdSnapInfo;
import com.cloud.hypervisor.kvm.resource.LibvirtConnection;
import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef;
import com.cloud.storage.Storage;
Expand Down Expand Up @@ -347,4 +355,131 @@ public void testCLVMPoolDetailsSecureZeroFillDefault() {
assert !secureZeroEnabled;
}

private static final String BASE_SNAP = "cloudstack-base-snap";
private static final String LUKS_SNAP = "cloudstack-base-snap-luks2";
private static final long TEMPLATE_SIZE = 4L << 30;
private static final long TIMEOUT = 10800000L;

private RbdImage templateImageWithSnapshots(String... names) throws RbdException {
RbdImage image = Mockito.mock(RbdImage.class);
List<RbdSnapInfo> snaps = new ArrayList<>();
for (String name : names) {
RbdSnapInfo info = new RbdSnapInfo();
info.name = name;
snaps.add(info);
}
Mockito.when(image.snapList()).thenReturn(snaps);
return image;
}

private KVMPhysicalDisk rbdTemplate() {
KVMPhysicalDisk template = Mockito.mock(KVMPhysicalDisk.class);
Mockito.lenient().when(template.getName()).thenReturn("tmpl");
Mockito.lenient().when(template.getVirtualSize()).thenReturn(TEMPLATE_SIZE);
return template;
}

private KVMStoragePool rbdPool() {
KVMStoragePool pool = Mockito.mock(KVMStoragePool.class);
Mockito.lenient().when(pool.getSourceHost()).thenReturn("1.2.3.4");
Mockito.lenient().when(pool.getSourcePort()).thenReturn(6789);
Mockito.lenient().when(pool.getAuthUserName()).thenReturn("cloudstack");
Mockito.lenient().when(pool.getAuthSecret()).thenReturn("secret");
Mockito.lenient().when(pool.getSourceDir()).thenReturn("cloudstack");
return pool;
}

private LibvirtStorageAdaptor adaptorWith(RbdEncryption rbdEncryption) {
LibvirtStorageAdaptor adaptor = Mockito.spy(new LibvirtStorageAdaptor(null));
Mockito.doReturn(rbdEncryption).when(adaptor).createRbdEncryption();
return adaptor;
}

@Test
public void prepareLuksTemplateSnapshotCreatesBaseSnapshotWhenFirstDeployIsEncrypted() throws Exception {
RbdImage image = templateImageWithSnapshots();
RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class);

adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT);

// the base snapshot must exist, and still hold the template's original size and data, before the rewrite reads it
InOrder order = Mockito.inOrder(image, rbdEncryption);
order.verify(image).snapCreate(BASE_SNAP);
order.verify(image).snapProtect(BASE_SNAP);
order.verify(image).resize(TEMPLATE_SIZE + (16L << 20));
order.verify(rbdEncryption).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT);
order.verify(image).snapCreate(LUKS_SNAP);
order.verify(image).snapProtect(LUKS_SNAP);
}

@Test
public void prepareLuksTemplateSnapshotUpgradesOldLuksSnapshotUsingExistingBase() throws Exception {
RbdImage image = templateImageWithSnapshots(BASE_SNAP, "cloudstack-base-snap-luks");
RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class);

adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT);

Mockito.verify(image, never()).snapCreate(BASE_SNAP);
Mockito.verify(rbdEncryption).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT);
Mockito.verify(image).snapCreate(LUKS_SNAP);
}

@Test
public void prepareLuksTemplateSnapshotDoesNothingWhenAlreadyPrepared() throws Exception {
RbdImage image = templateImageWithSnapshots(BASE_SNAP, LUKS_SNAP);
RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class);

adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT);

Mockito.verify(image, never()).resize(anyLong());
Mockito.verify(image, never()).snapCreate(anyString());
Mockito.verifyNoInteractions(rbdEncryption);
}

@Test
public void prepareLuksTemplateSnapshotTakesNoSnapshotWhenRewriteFailsAndRetryStartsOver() throws Exception {
RbdImage image = templateImageWithSnapshots();
RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class);
Mockito.doThrow(new CloudRuntimeException("qemu-img failed")).doNothing().when(rbdEncryption)
.copyTemplateDense(anyString(), Mockito.anyInt(), anyString(), anyString(), anyString(), anyString(), anyString(), anyLong());
LibvirtStorageAdaptor adaptor = adaptorWith(rbdEncryption);

Assert.assertThrows(CloudRuntimeException.class,
() -> adaptor.prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT));
// a snapshot of a half-rewritten template would be cloned from forever
Mockito.verify(image, never()).snapCreate(LUKS_SNAP);

// next deploy: the base snapshot is there now, the rewrite runs again and the snapshot is taken
RbdSnapInfo baseSnapshot = new RbdSnapInfo();
baseSnapshot.name = BASE_SNAP;
Mockito.when(image.snapList()).thenReturn(List.of(baseSnapshot));
adaptor.prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT);
Mockito.verify(image, Mockito.times(1)).snapCreate(BASE_SNAP);
Mockito.verify(rbdEncryption, Mockito.times(2)).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT);
Mockito.verify(image).snapCreate(LUKS_SNAP);
}

@Test
public void prepareLuksTemplateSnapshotAcceptsAlreadyProtectedDuplicateSnapshots() throws Exception {
RbdImage image = templateImageWithSnapshots();
RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class);
Mockito.doThrow(new RbdException("File exists", -17)).when(image).snapCreate(anyString());
Mockito.when(image.snapIsProtected(anyString())).thenReturn(true);

adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT);

Mockito.verify(rbdEncryption).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT);
}

@Test
public void prepareLuksTemplateSnapshotFailsWhenSnapshotCannotBeTaken() throws Exception {
RbdImage image = templateImageWithSnapshots(BASE_SNAP);
RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class);
Mockito.doThrow(new RbdException("No space left", -28)).when(image).snapCreate(LUKS_SNAP);
Mockito.when(image.snapIsProtected(LUKS_SNAP)).thenReturn(false);
LibvirtStorageAdaptor adaptor = adaptorWith(rbdEncryption);

Assert.assertThrows(RbdException.class,
() -> adaptor.prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT));
}
}
Loading

Back | FazBrowse Home | New Git URL