From 65306f27b37e5e383a4ede2ecbfd15ebb5e18783 Mon Sep 17 00:00:00 2001 From: "Michael N. Lipp" Date: Tue, 8 Aug 2023 19:00:15 +0200 Subject: [PATCH] Adjust grace period to powerdown timeout. --- .../vmoperator/manager/CmReconciler.java | 2 +- .../vmoperator/manager/DataReconciler.java | 2 +- .../vmoperator/manager/DisksReconciler.java | 14 ++++---- .../jdrupes/vmoperator/manager/GsonPtr.java | 22 +++++++++++++ .../org/jdrupes/vmoperator/manager/K8s.java | 20 ++++++++++-- .../vmoperator/manager/PodReconciler.java | 12 ++++--- .../vmoperator/manager/Reconciler.java | 32 +++++++++---------- .../{WatchChannel.java => VmChannel.java} | 32 ++++++++++++++++--- .../vmoperator/manager/VmDefChanged.java | 4 +-- .../jdrupes/vmoperator/manager/VmWatcher.java | 20 ++++++------ 10 files changed, 112 insertions(+), 48 deletions(-) rename org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/{WatchChannel.java => VmChannel.java} (68%) diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/CmReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/CmReconciler.java index 0bcfb2f..841eac4 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/CmReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/CmReconciler.java @@ -58,7 +58,7 @@ import org.jdrupes.vmoperator.manager.VmDefChanged.Type; * @throws ApiException the api exception */ public DynamicKubernetesObject reconcile(VmDefChanged event, - Map model, WatchChannel channel) + Map model, VmChannel channel) throws IOException, TemplateException, ApiException { // Get API and check if exists DynamicKubernetesApi cmApi = new DynamicKubernetesApi("", "v1", diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DataReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DataReconciler.java index 5aeef3e..75f4486 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DataReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DataReconciler.java @@ -55,7 +55,7 @@ import java.util.Map; * @throws IOException Signals that an I/O exception has occurred. */ @SuppressWarnings("PMD.ConfusingTernary") - public void reconcile(Map model, WatchChannel channel) + public void reconcile(Map model, VmChannel channel) throws TemplateException, ApiException, IOException { // Combine template and data and parse result var fmTemplate = fmConfig.getTemplate("runnerDataPvc.ftl.yaml"); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisksReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisksReconciler.java index 6c4bdbe..d2a3956 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisksReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisksReconciler.java @@ -42,10 +42,10 @@ import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME; * @param channel the channel * @throws ApiException the api exception */ - public void reconcile(DynamicKubernetesObject vmDef, - WatchChannel channel) throws ApiException { + public void reconcile(JsonObject vmDef, + VmChannel channel) throws ApiException { @SuppressWarnings("PMD.AvoidDuplicateLiterals") - var disks = GsonPtr.to(vmDef.getRaw()) + var disks = GsonPtr.to(vmDef) .get(JsonArray.class, "spec", "vm", "disks") .map(JsonArray::asList).orElse(Collections.emptyList()); int index = 0; @@ -55,15 +55,15 @@ import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME; } @SuppressWarnings({ "PMD.AvoidDuplicateLiterals", "PMD.ConfusingTernary" }) - private void reconcileDisk(DynamicKubernetesObject vmDefinition, - int index, JsonObject diskDef, WatchChannel channel) + private void reconcileDisk(JsonObject vmDefinition, + int index, JsonObject diskDef, VmChannel channel) throws ApiException { if (!diskDef.has("volumeClaimTemplate")) { return; } var pvcObject = new DynamicKubernetesObject(); var pvcRaw = GsonPtr.to(pvcObject.getRaw()); - var vmRaw = GsonPtr.to(vmDefinition.getRaw()); + var vmRaw = GsonPtr.to(vmDefinition); var pvcTpl = GsonPtr.to(diskDef).to("volumeClaimTemplate"); // Copy base and metadata from template and add missing/additional data. @@ -110,7 +110,7 @@ import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME; * @param channel the channel * @throws ApiException the api exception */ - public void deleteDisks(VmDefChanged event, WatchChannel channel) + public void deleteDisks(VmDefChanged event, VmChannel channel) throws ApiException { // Get API and check and list related var pvcApi = K8s.pvcApi(channel.client()); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/GsonPtr.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/GsonPtr.java index 27fa8fa..6d9070c 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/GsonPtr.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/GsonPtr.java @@ -164,6 +164,28 @@ public class GsonPtr { .map(JsonPrimitive::getAsString); } + /** + * Returns the Integer value of the selected {@link JsonPrimitive}. + * + * @param selectors the selectors + * @return the as string + */ + public Optional getAsInt(Object... selectors) { + return get(JsonPrimitive.class, selectors) + .map(JsonPrimitive::getAsInt); + } + + /** + * Returns the Long value of the selected {@link JsonPrimitive}. + * + * @param selectors the selectors + * @return the as string + */ + public Optional getAsLong(Object... selectors) { + return get(JsonPrimitive.class, selectors) + .map(JsonPrimitive::getAsLong); + } + /** * Sets the selected value. This pointer must point to a * {@link JsonObject} or {@link JsonArray}. The selector must diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java index 03b3545..3a54de3 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java @@ -31,6 +31,7 @@ import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodList; import io.kubernetes.client.util.generic.GenericKubernetesApi; +import io.kubernetes.client.util.generic.options.DeleteOptions; import io.kubernetes.client.util.generic.options.PatchOptions; import java.util.Optional; @@ -97,8 +98,8 @@ public class K8s { * @return the object */ public static - Optional get(GenericKubernetesApi api, V1ObjectMeta meta) - throws ApiException { + Optional + get(GenericKubernetesApi api, V1ObjectMeta meta) { var response = api.get(meta.getNamespace(), meta.getName()); if (response.isSuccess()) { return Optional.of(response.getObject()); @@ -121,6 +122,21 @@ public class K8s { object.getMetadata().getName()).throwsApiException(); } + /** + * Delete an object. + * + * @param the generic type + * @param the generic type + * @param api the api + * @param object the object + */ + public static + void delete(GenericKubernetesApi api, T object, + DeleteOptions options) throws ApiException { + api.delete(object.getMetadata().getNamespace(), + object.getMetadata().getName(), options).throwsApiException(); + } + /** * Apply the given patch data. * diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java index 7918942..281697f 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java @@ -25,6 +25,7 @@ import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi; import io.kubernetes.client.util.generic.dynamic.Dynamics; +import io.kubernetes.client.util.generic.options.DeleteOptions; import java.io.IOException; import java.io.StringWriter; import java.util.Map; @@ -59,15 +60,14 @@ import org.jdrupes.vmoperator.manager.VmDefChanged.Type; * @throws ApiException the api exception */ public void reconcile(VmDefChanged event, Map model, - WatchChannel channel) + VmChannel channel) throws IOException, TemplateException, ApiException { // Check if exists DynamicKubernetesApi podApi = new DynamicKubernetesApi("", "v1", "pods", channel.client()); var existing = K8s.get(podApi, event.object().getMetadata()); - // Get state. Note that model is only available if event type - // is not DELETED. + // Get desired state. var delete = event.type() == Type.DELETED || GsonPtr.to((JsonObject) model.get("cr")).to("spec", "vm") .getAsString("state").orElse("").equals(STATE_STOPPED); @@ -75,7 +75,11 @@ import org.jdrupes.vmoperator.manager.VmDefChanged.Type; // If deleted or stopped, delete if (delete) { if (existing.isPresent()) { - K8s.delete(podApi, existing.get()); + var opts = new DeleteOptions(); + opts.setGracePeriodSeconds( + GsonPtr.to((JsonObject) model.get("cr")).to("spec", "vm") + .getAsLong("powerdownTimeout").get() + 1); + K8s.delete(podApi, existing.get(), opts); } return; } diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java index b9e57b7..5a436da 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java @@ -30,7 +30,6 @@ import freemarker.template.TemplateHashModel; import freemarker.template.TemplateNotFoundException; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi; -import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -94,7 +93,7 @@ public class Reconciler extends Component { */ @Handler @SuppressWarnings("PMD.ConfusingTernary") - public void onVmDefChanged(VmDefChanged event, WatchChannel channel) + public void onVmDefChanged(VmDefChanged event, VmChannel channel) throws ApiException, TemplateException, IOException { // Get complete VM (CR) definition var apiVersion = K8s.version(event.object().getApiVersion()); @@ -102,22 +101,23 @@ public class Reconciler extends Component { apiVersion, event.crd().getName(), channel.client()); var defMeta = event.object().getMetadata(); - // Get common data for all reconciles - DynamicKubernetesObject vmDef = null; - Map model = null; + // Update state if (event.type() != Type.DELETED) { - vmDef = K8s.get(vmCrApi, defMeta).get(); - - // Prepare Freemarker model - model = new HashMap<>(); - model.put("cr", patchCr(vmDef.getRaw().deepCopy())); - model.put("constants", - (TemplateHashModel) new DefaultObjectWrapperBuilder( - Configuration.VERSION_2_3_32) - .build().getStaticModels() - .get(Constants.class.getName())); + channel.setState( + patchCr(K8s.get(vmCrApi, defMeta).get().getRaw().deepCopy())); } + // Get common data for all reconciles + JsonObject vmDef = channel.state(); + @SuppressWarnings("PMD.UseConcurrentHashMap") + Map model = new HashMap<>(); + model.put("cr", vmDef); + model.put("constants", + (TemplateHashModel) new DefaultObjectWrapperBuilder( + Configuration.VERSION_2_3_32) + .build().getStaticModels() + .get(Constants.class.getName())); + // Reconcile if (event.type() != Type.DELETED) { dataReconciler.reconcile(model, channel); @@ -132,7 +132,7 @@ public class Reconciler extends Component { } } - private Object patchCr(JsonObject vmDef) { + private JsonObject patchCr(JsonObject vmDef) { // Adjust cdromImage path var disks = GsonPtr.to(vmDef).to("spec", "vm", "disks").get(JsonArray.class); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/WatchChannel.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmChannel.java similarity index 68% rename from org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/WatchChannel.java rename to org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmChannel.java index fbc22a4..90fa122 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/WatchChannel.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmChannel.java @@ -18,34 +18,56 @@ package org.jdrupes.vmoperator.manager; +import com.google.gson.JsonObject; import io.kubernetes.client.openapi.ApiClient; import org.jgrapes.core.Channel; import org.jgrapes.core.EventPipeline; import org.jgrapes.core.Subchannel.DefaultSubchannel; /** - * A subchannel used to send the events related to a specific - * VM. + * A subchannel used to send the events related to a specific VM. */ -public class WatchChannel extends DefaultSubchannel { +public class VmChannel extends DefaultSubchannel { private final EventPipeline pipeline; private final ApiClient client; + private JsonObject state; /** * Instantiates a new watch channel. * * @param mainChannel the main channel * @param pipeline the pipeline - * @param client + * @param client the client */ - public WatchChannel(Channel mainChannel, EventPipeline pipeline, + public VmChannel(Channel mainChannel, EventPipeline pipeline, ApiClient client) { super(mainChannel); this.pipeline = pipeline; this.client = client; } + /** + * Sets the last known state of the resource. + * + * @param state the state + * @return the watch channel + */ + @SuppressWarnings("PMD.LinguisticNaming") + public VmChannel setState(JsonObject state) { + this.state = state; + return this; + } + + /** + * Returns the last known state of the resource. + * + * @return the json object + */ + public JsonObject state() { + return state; + } + /** * Returns the pipeline. * diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmDefChanged.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmDefChanged.java index 3719128..dd6f7db 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmDefChanged.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmDefChanged.java @@ -83,8 +83,8 @@ public class VmDefChanged extends Event { @Override public String toString() { StringBuilder builder = new StringBuilder(); - builder.append(Components.objectName(this)).append(" [").append(type) - .append(' ').append(object.getMetadata().getName()); + builder.append(Components.objectName(this)).append(" [") + .append(object.getMetadata().getName()).append(' ').append(type); if (channels() != null) { builder.append(", channels="); builder.append(Channel.toString(channels())); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmWatcher.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmWatcher.java index b616cbb..5a372bb 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmWatcher.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmWatcher.java @@ -39,7 +39,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; -import okhttp3.Call; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_GROUP; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_KIND_VM; import org.jdrupes.vmoperator.manager.VmDefChanged.Type; @@ -57,7 +56,7 @@ public class VmWatcher extends Component { private ApiClient client; private String managedNamespace = "qemu-vms"; - private final Map channels + private final Map channels = new ConcurrentHashMap<>(); /** @@ -90,13 +89,14 @@ public class VmWatcher extends Component { var coa = new CustomObjectsApi(client); purge(coa, vmOpApiVersions); - // Start a watcher for each existing CRD version. + // Start a watcher thread for each existing CRD version. + // The watcher will send us an "ADDED" for each existing VM. for (var version : vmOpApiVersions) { coa.getAPIResources(VM_OP_GROUP, version) .getResources().stream() .filter(r -> Constants.VM_OP_KIND_VM.equals(r.getKind())) .findFirst() - .ifPresent(crd -> serveCrVersion(coa, crd, version)); + .ifPresent(crd -> watchVmDefs(coa, crd, version)); } } @@ -150,7 +150,7 @@ public class VmWatcher extends Component { return result; } - private void serveCrVersion(CustomObjectsApi coa, V1APIResource crd, + private void watchVmDefs(CustomObjectsApi coa, V1APIResource crd, String version) { @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") var watcher = new Thread(() -> { @@ -165,7 +165,7 @@ public class VmWatcher extends Component { new TypeToken>() { }.getType())) { for (Watch.Response item : watch) { - handleVmDefinitionEvent(crd, item); + handleVmDefinitionChange(crd, item); } } catch (IllegalStateException e) { logger.log(Level.FINE, e, () -> "Probem watching: " @@ -182,11 +182,11 @@ public class VmWatcher extends Component { watcher.start(); } - private void handleVmDefinitionEvent(V1APIResource vmsCrd, + private void handleVmDefinitionChange(V1APIResource vmsCrd, Watch.Response item) { V1ObjectMeta metadata = item.object.getMetadata(); - WatchChannel channel = channels.computeIfAbsent(metadata.getName(), - k -> new WatchChannel(channel(), newEventPipeline(), client)); + VmChannel channel = channels.computeIfAbsent(metadata.getName(), + k -> new VmChannel(channel(), newEventPipeline(), client)); channel.pipeline().fire(new VmDefChanged(VmDefChanged.Type .valueOf(item.type), vmsCrd, item.object), channel); } @@ -198,7 +198,7 @@ public class VmWatcher extends Component { * @param channel the channel */ @Handler(priority = -10_000) - public void onVmDefChanged(VmDefChanged event, WatchChannel channel) { + public void onVmDefChanged(VmDefChanged event, VmChannel channel) { if (event.type() == Type.DELETED) { channels.remove(event.object().getMetadata().getName()); }