From 8f00d308a5a266614f9ef361838a3d914caa02b3 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 22 Dec 2023 13:50:09 +0100 Subject: [PATCH] linstor: implement CloudStack HA support --- .../kvm/storage/LinstorStoragePool.java | 116 ++++++++++++++++-- .../LinstorPrimaryDataStoreDriverImpl.java | 2 +- .../provider/LinstorHostListener.java | 32 +++++ .../LinstorPrimaryDatastoreProviderImpl.java | 2 +- 4 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java index 45cf7e3c6f91..dc269535c49a 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java @@ -19,16 +19,23 @@ import java.util.List; import java.util.Map; -import org.apache.cloudstack.utils.qemu.QemuImg; -import org.joda.time.Duration; - import com.cloud.agent.api.to.HostTO; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.storage.Storage; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonIOException; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.log4j.Logger; +import org.joda.time.Duration; public class LinstorStoragePool implements KVMStoragePool { private static final Logger s_logger = Logger.getLogger(LinstorStoragePool.class); @@ -38,6 +45,7 @@ public class LinstorStoragePool implements KVMStoragePool { private final Storage.StoragePoolType _storagePoolType; private final StorageAdaptor _storageAdaptor; private final String _resourceGroup; + private final String localNodeName; public LinstorStoragePool(String uuid, String host, int port, String resourceGroup, Storage.StoragePoolType storagePoolType, StorageAdaptor storageAdaptor) { @@ -47,6 +55,7 @@ public LinstorStoragePool(String uuid, String host, int port, String resourceGro _storagePoolType = storagePoolType; _storageAdaptor = storageAdaptor; _resourceGroup = resourceGroup; + localNodeName = getHostname(); } @Override @@ -205,22 +214,34 @@ public String getResourceGroup() { @Override public boolean isPoolSupportHA() { - return false; + return true; } @Override public String getHearthBeatPath() { - return null; + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + return Script.findScript(kvmScriptsDir, "kvmspheartbeat.sh"); } @Override - public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, + public String createHeartBeatCommand(HAStoragePool pool, String hostPrivateIp, boolean hostValidation) { - return null; + s_logger.trace(String.format("Linstor.createHeartBeatCommand: %s, %s, %b", pool.getPoolIp(), hostPrivateIp, hostValidation)); + boolean isStorageNodeUp = checkingHeartBeat(pool, null); + if (!isStorageNodeUp && !hostValidation) { + //restart the host + s_logger.debug(String.format("The host [%s] will be restarted because the health check failed for the storage pool [%s]", hostPrivateIp, pool.getPool().getType())); + Script cmd = new Script(pool.getPool().getHearthBeatPath(), Duration.millis(HeartBeatUpdateTimeout), s_logger); + cmd.add("-c"); + cmd.execute(); + return "Down"; + } + return isStorageNodeUp ? null : "Down"; } @Override public String getStorageNodeId() { + // only called by storpool return null; } @@ -237,11 +258,88 @@ static String getHostname() { @Override public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { - return null; + String hostName; + if (host == null) { + hostName = localNodeName; + } else { + hostName = host.getParent(); + if (hostName == null) { + s_logger.error("No hostname set in host.getParent()"); + return false; + } + } + + return checkHostUpToDateAndConnected(hostName); + } + + private String executeDrbdSetupStatus(OutputInterpreter.AllLinesParser parser) { + Script sc = new Script("drbdsetup", Duration.millis(HeartBeatUpdateTimeout), s_logger); + sc.add("status"); + sc.add("--json"); + return sc.execute(parser); + } + + private boolean checkDrbdSetupStatusOutput(String output, String otherNodeName) { + JsonParser jsonParser = new JsonParser(); + JsonArray jResources = (JsonArray) jsonParser.parse(output); + for (JsonElement jElem : jResources) { + JsonObject jRes = (JsonObject) jElem; + JsonArray jConnections = jRes.getAsJsonArray("connections"); + for (JsonElement jConElem : jConnections) { + JsonObject jConn = (JsonObject) jConElem; + if (jConn.getAsJsonPrimitive("name").getAsString().equals(otherNodeName) + && jConn.getAsJsonPrimitive("connection-state").getAsString().equalsIgnoreCase("Connected")) { + return true; + } + } + } + s_logger.warn(String.format("checkDrbdSetupStatusOutput: no resource connected to %s.", otherNodeName)); + return false; + } + + private String executeDrbdEventsNow(OutputInterpreter.AllLinesParser parser) { + Script sc = new Script("drbdsetup", Duration.millis(HeartBeatUpdateTimeout), s_logger); + sc.add("events2"); + sc.add("--now"); + return sc.execute(parser); + } + + private boolean checkDrbdEventsNowOutput(String output) { + boolean healthy = output.lines().noneMatch(line -> line.matches(".*role:Primary .* promotion_score:0.*")); + if (!healthy) { + s_logger.warn("checkDrbdEventsNowOutput: primary resource with promotion score==0; HA false"); + } + return healthy; + } + + private boolean checkHostUpToDateAndConnected(String hostName) { + s_logger.trace(String.format("checkHostUpToDateAndConnected: %s/%s", localNodeName, hostName)); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + if (localNodeName.equalsIgnoreCase(hostName)) { + String res = executeDrbdEventsNow(parser); + if (res != null) { + return false; + } + return checkDrbdEventsNowOutput(parser.getLines()); + } else { + // check drbd connections + String res = executeDrbdSetupStatus(parser); + if (res != null) { + return false; + } + try { + return checkDrbdSetupStatusOutput(parser.getLines(), hostName); + } catch (JsonIOException | JsonSyntaxException e) { + s_logger.error("Error parsing drbdsetup status --json", e); + } + } + return false; } @Override public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { - return null; + s_logger.trace(String.format("Linstor.vmActivityCheck: %s, %s", pool.getPoolIp(), host.getPrivateNetwork().getIp())); + return checkingHeartBeat(pool, host); } } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 9b493ff01b9e..ca98dc3dd460 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -1241,7 +1241,7 @@ public void provideVmTags(long vmId, long volumeId, String tagValue) { @Override public boolean isStorageSupportHA(StoragePoolType type) { - return false; + return true; } @Override diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java new file mode 100644 index 000000000000..534431ed681b --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.exception.StorageConflictException; +import com.cloud.host.HostVO; + +public class LinstorHostListener extends DefaultHostListener { + @Override + public boolean hostConnect(long hostId, long poolId) throws StorageConflictException { + HostVO host = hostDao.findById(hostId); + if (host.getParent() == null) { + host.setParent(host.getName()); + hostDao.update(host.getId(), host); + } + return super.hostConnect(hostId, poolId); + } +} diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java index 563f542db37b..962c84fffb19 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java @@ -48,7 +48,7 @@ public DataStoreLifeCycle getDataStoreLifeCycle() { public boolean configure(Map params) { lifecycle = ComponentContext.inject(LinstorPrimaryDataStoreLifeCycleImpl.class); driver = ComponentContext.inject(LinstorPrimaryDataStoreDriverImpl.class); - listener = ComponentContext.inject(DefaultHostListener.class); + listener = ComponentContext.inject(LinstorHostListener.class); return true; }