Skip to content

Commit

Permalink
linstor: add support for encryption
Browse files Browse the repository at this point in the history
This introduces a new encryption mode, instead of a simple bool.
Now also storage driver can just provide encrypted volumes to CloudStack.
  • Loading branch information
rp- committed Dec 19, 2024
1 parent 4fee43a commit 02eacd0
Show file tree
Hide file tree
Showing 5 changed files with 199 additions and 27 deletions.
65 changes: 42 additions & 23 deletions api/src/main/java/com/cloud/storage/Storage.java
Original file line number Diff line number Diff line change
Expand Up @@ -135,34 +135,49 @@ public static enum TemplateType {
ISODISK /* Template corresponding to a iso (non root disk) present in an OVA */
}

public enum EncryptionSupport {
/**
* Encryption not supported.
*/
UnSupported,
/**
* Will use hypervisor encryption driver (qemu -> luks)
*/
Hypervisor,
/**
* Storage pool handles encryption and just provides an encrypted volume
*/
Storage
}

public static enum StoragePoolType {
Filesystem(false, true, true), // local directory
NetworkFilesystem(true, true, true), // NFS
IscsiLUN(true, false, false), // shared LUN, with a clusterfs overlay
Iscsi(true, false, false), // for e.g., ZFS Comstar
ISO(false, false, false), // for iso image
LVM(false, false, false), // XenServer local LVM SR
CLVM(true, false, false),
RBD(true, true, false), // http://libvirt.org/storage.html#StorageBackendRBD
SharedMountPoint(true, true, true),
VMFS(true, true, false), // VMware VMFS storage
PreSetup(true, true, false), // for XenServer, Storage Pool is set up by customers.
EXT(false, true, false), // XenServer local EXT SR
OCFS2(true, false, false),
SMB(true, false, false),
Gluster(true, false, false),
PowerFlex(true, true, true), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS)
ManagedNFS(true, false, false),
Linstor(true, true, false),
DatastoreCluster(true, true, false), // for VMware, to abstract pool of clusters
StorPool(true, true, true),
FiberChannel(true, true, false); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
Filesystem(false, true, EncryptionSupport.Hypervisor), // local directory
NetworkFilesystem(true, true, EncryptionSupport.Hypervisor), // NFS
IscsiLUN(true, false, EncryptionSupport.UnSupported), // shared LUN, with a clusterfs overlay
Iscsi(true, false, EncryptionSupport.UnSupported), // for e.g., ZFS Comstar
ISO(false, false, EncryptionSupport.UnSupported), // for iso image
LVM(false, false, EncryptionSupport.UnSupported), // XenServer local LVM SR
CLVM(true, false, EncryptionSupport.UnSupported),
RBD(true, true, EncryptionSupport.UnSupported), // http://libvirt.org/storage.html#StorageBackendRBD
SharedMountPoint(true, true, EncryptionSupport.Hypervisor),
VMFS(true, true, EncryptionSupport.UnSupported), // VMware VMFS storage
PreSetup(true, true, EncryptionSupport.UnSupported), // for XenServer, Storage Pool is set up by customers.
EXT(false, true, EncryptionSupport.UnSupported), // XenServer local EXT SR
OCFS2(true, false, EncryptionSupport.UnSupported),
SMB(true, false, EncryptionSupport.UnSupported),
Gluster(true, false, EncryptionSupport.UnSupported),
PowerFlex(true, true, EncryptionSupport.Hypervisor), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS)
ManagedNFS(true, false, EncryptionSupport.UnSupported),
Linstor(true, true, EncryptionSupport.Storage),
DatastoreCluster(true, true, EncryptionSupport.UnSupported), // for VMware, to abstract pool of clusters
StorPool(true, true, EncryptionSupport.Hypervisor),
FiberChannel(true, true, EncryptionSupport.UnSupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)

private final boolean shared;
private final boolean overProvisioning;
private final boolean encryption;
private final EncryptionSupport encryption;

StoragePoolType(boolean shared, boolean overProvisioning, boolean encryption) {
StoragePoolType(boolean shared, boolean overProvisioning, EncryptionSupport encryption) {
this.shared = shared;
this.overProvisioning = overProvisioning;
this.encryption = encryption;
Expand All @@ -177,6 +192,10 @@ public boolean supportsOverProvisioning() {
}

public boolean supportsEncryption() {
return encryption == EncryptionSupport.Hypervisor || encryption == EncryptionSupport.Storage;
}

public EncryptionSupport encryptionSupportMode() {
return encryption;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3169,7 +3169,8 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
disk.setCacheMode(DiskDef.DiskCacheMode.valueOf(volumeObjectTO.getCacheMode().toString().toUpperCase()));
}

if (volumeObjectTO.requiresEncryption()) {
if (volumeObjectTO.requiresEncryption() &&
pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) {
String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase());
DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()));
disk.setLibvirtDiskEncryptDetails(encryptDetails);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import com.cloud.storage.JavaStorageLayer;
import com.cloud.storage.MigrationOptions;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageLayer;
Expand Down Expand Up @@ -1452,7 +1453,8 @@ protected synchronized void attachOrDetachDisk(final Connect conn, final boolean
}
}

if (encryptDetails != null) {
if (encryptDetails != null &&
attachingPool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor) {
diskdef.setLibvirtDiskEncryptDetails(encryptDetails);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,14 @@
import com.linbit.linstor.api.DevelopersApi;
import com.linbit.linstor.api.model.ApiCallRc;
import com.linbit.linstor.api.model.ApiCallRcList;
import com.linbit.linstor.api.model.AutoSelectFilter;
import com.linbit.linstor.api.model.LayerType;
import com.linbit.linstor.api.model.Properties;
import com.linbit.linstor.api.model.ResourceDefinition;
import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest;
import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted;
import com.linbit.linstor.api.model.ResourceDefinitionCreate;
import com.linbit.linstor.api.model.ResourceGroup;
import com.linbit.linstor.api.model.ResourceGroupSpawn;
import com.linbit.linstor.api.model.ResourceMakeAvailable;
import com.linbit.linstor.api.model.Snapshot;
Expand All @@ -34,6 +37,7 @@
import com.linbit.linstor.api.model.VolumeDefinitionModify;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;

import java.util.Arrays;
Expand All @@ -43,6 +47,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.ResizeVolumeAnswer;
Expand Down Expand Up @@ -105,6 +110,8 @@
import org.apache.cloudstack.storage.volume.VolumeObject;
import org.apache.log4j.Logger;

import java.nio.charset.StandardCharsets;

public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver {
private static final Logger s_logger = Logger.getLogger(LinstorPrimaryDataStoreDriverImpl.class);
@Inject private PrimaryDataStoreDao _storagePoolDao;
Expand Down Expand Up @@ -393,11 +400,56 @@ private String getRscGrp(StoragePoolVO storagePoolVO) {
storagePoolVO.getUserInfo() : "DfltRscGrp";
}

/**
* Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE.
* If the resourceGroup layer list already contains LUKS this layer list will be returned.
* @param api Linstor developers API
* @param resourceGroup Resource group to get the encryption layer list
* @return layer list with LUKS added
*/
public List<LayerType> getEncryptedLayerList(DevelopersApi api, String resourceGroup) {
try {
List<ResourceGroup> rscGrps = api.resourceGroupList(
Collections.singletonList(resourceGroup), Collections.emptyList(), null, null);

if (rscGrps == null || rscGrps.isEmpty()) {
throw new CloudRuntimeException(
String.format("Resource Group %s not found on Linstor cluster.", resourceGroup));
}

final ResourceGroup rscGrp = rscGrps.get(0);
List<LayerType> layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE);
List<String> curLayerStack = rscGrp.getSelectFilter() != null ?
rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList();
if (!(curLayerStack == null || curLayerStack.isEmpty())) {
layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList());
if (!layers.contains(LayerType.LUKS)) {
layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE
}
}
return layers;
} catch (ApiException e) {
throw new CloudRuntimeException(
String.format("Resource Group %s not found on Linstor cluster.", resourceGroup));
}
}

private String createResourceBase(
String rscName, long sizeInBytes, String volName, String vmName, DevelopersApi api, String rscGrp) {
String rscName, long sizeInBytes, String volName, String vmName,
@Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, String rscGrp) {
ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn();
rscGrpSpawn.setResourceDefinitionName(rscName);
rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024);
if (passPhraseId != null) {
AutoSelectFilter asf = new AutoSelectFilter();
List<LayerType> luksLayers = getEncryptedLayerList(api, rscGrp);
asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList()));
rscGrpSpawn.setSelectFilter(asf);
if (passPhrase != null) {
String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8);
rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase));
}
}

try
{
Expand All @@ -422,7 +474,8 @@ private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) {

final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid();
String deviceName = createResourceBase(
rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), linstorApi, rscGrp);
rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(),
linstorApi, rscGrp);

try
{
Expand Down Expand Up @@ -463,6 +516,14 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV
s_logger.info("Clone resource definition " + cloneRes + " to " + rscName);
ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest();
cloneRequest.setName(rscName);
if (volumeInfo.getPassphraseId() != null) {
List<LayerType> encryptionLayer = getEncryptedLayerList(linstorApi, getRscGrp(storagePoolVO));
cloneRequest.setLayerList(encryptionLayer);
if (volumeInfo.getPassphrase() != null) {
String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8);
cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase));
}
}
ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone(
cloneRes, cloneRequest);

Expand Down Expand Up @@ -915,6 +976,8 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) {
tInfo.getSize(),
tInfo.getName(),
"",
null,
null,
api,
getRscGrp(pool));

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.datastore.driver;

import com.linbit.linstor.api.ApiException;
import com.linbit.linstor.api.DevelopersApi;
import com.linbit.linstor.api.model.AutoSelectFilter;
import com.linbit.linstor.api.model.LayerType;
import com.linbit.linstor.api.model.ResourceGroup;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.junit.MockitoJUnitRunner;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

@RunWith(MockitoJUnitRunner.class)
public class LinstorPrimaryDataStoreDriverImplTest {

private DevelopersApi api;

@InjectMocks
private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver;

@Before
public void setUp() {
api = mock(DevelopersApi.class);
}

@Test
public void testGetEncryptedLayerList() throws ApiException {
ResourceGroup dfltRscGrp = new ResourceGroup();
dfltRscGrp.setName("DfltRscGrp");

ResourceGroup bCacheRscGrp = new ResourceGroup();
bCacheRscGrp.setName("BcacheGrp");
AutoSelectFilter asf = new AutoSelectFilter();
asf.setLayerStack(Arrays.asList(LayerType.DRBD.name(), LayerType.BCACHE.name(), LayerType.STORAGE.name()));
asf.setStoragePool("nvmePool");
bCacheRscGrp.setSelectFilter(asf);

ResourceGroup encryptedGrp = new ResourceGroup();
encryptedGrp.setName("EncryptedGrp");
AutoSelectFilter asf2 = new AutoSelectFilter();
asf2.setLayerStack(Arrays.asList(LayerType.DRBD.name(), LayerType.LUKS.name(), LayerType.STORAGE.name()));
asf2.setStoragePool("ssdPool");
encryptedGrp.setSelectFilter(asf2);

when(api.resourceGroupList(Collections.singletonList("DfltRscGrp"), Collections.emptyList(), null, null))
.thenReturn(Collections.singletonList(dfltRscGrp));
when(api.resourceGroupList(Collections.singletonList("BcacheGrp"), Collections.emptyList(), null, null))
.thenReturn(Collections.singletonList(bCacheRscGrp));
when(api.resourceGroupList(Collections.singletonList("EncryptedGrp"), Collections.emptyList(), null, null))
.thenReturn(Collections.singletonList(encryptedGrp));

List<LayerType> layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "DfltRscGrp");
Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers);

layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "BcacheGrp");
Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.BCACHE, LayerType.LUKS, LayerType.STORAGE), layers);

layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "EncryptedGrp");
Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers);
}
}

0 comments on commit 02eacd0

Please sign in to comment.