From e8097d87d9194c2d23a1b9fe91f90d2d7a59f50a Mon Sep 17 00:00:00 2001 From: "Michael N. Lipp" Date: Fri, 28 Mar 2025 18:03:09 +0100 Subject: [PATCH] Let the operator manage pod restarts. --- .../vmoperator/manager/runnerPod.ftl.yaml | 1 + .../manager/ConfigMapReconciler.java | 16 ++++-- .../manager/DisplaySecretReconciler.java | 9 ++-- .../manager/LoadBalancerReconciler.java | 6 +-- .../vmoperator/manager/PodReconciler.java | 6 +-- .../vmoperator/manager/PvcReconciler.java | 19 +++---- .../vmoperator/manager/Reconciler.java | 51 ++++++++++++++++--- .../manager/StatefulSetReconciler.java | 10 ++-- 8 files changed, 77 insertions(+), 41 deletions(-) diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml index 7518ad3..2f082ab 100644 --- a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml @@ -133,3 +133,4 @@ spec: affinity: ${ toJson(spec.affinity) } serviceAccountName: vm-runner + restartPolicy: Never diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ConfigMapReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ConfigMapReconciler.java index f133542..161678a 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ConfigMapReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ConfigMapReconciler.java @@ -86,16 +86,20 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; boolean modelChanged) throws IOException, TemplateException, ApiException { // Check if an update is needed - Object prevInputs - = channel.associated(PrevData.class, Object.class).orElse(null); + var prevData = channel.associated(PrevData.class) + .orElseGet(() -> new PrevData(null, new HashMap<>())); Object newInputs = model.get("loginRequestedFor"); - if (!modelChanged && Objects.equals(prevInputs, newInputs)) { + if (!modelChanged && Objects.equals(prevData.inputs, newInputs)) { + // Make added data available in new model + model.putAll(prevData.added); return; } - channel.setAssociated(PrevData.class, newInputs); + prevData = new PrevData(newInputs, prevData.added); + channel.setAssociated(PrevData.class, prevData); // Combine template and data and parse result model.put("adjustCloudInitMeta", adjustCloudInitMetaModel); + prevData.added.put("adjustCloudInitMeta", adjustCloudInitMetaModel); var fmTemplate = fmConfig.getTemplate("runnerConfig.ftl.yaml"); StringWriter out = new StringWriter(); fmTemplate.process(model, out); @@ -127,12 +131,14 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; maybeForceUpdate(channel.client(), updatedCm); model.put("configMapResourceVersion", updatedCm.getMetadata().getResourceVersion()); + prevData.added.put("configMapResourceVersion", + updatedCm.getMetadata().getResourceVersion()); } /** * Key for association. */ - private final class PrevData { + private record PrevData(Object inputs, Map added) { } /** diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java index dabffb6..2770347 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java @@ -120,25 +120,24 @@ public class DisplaySecretReconciler extends Component { * secret with a random password and immediate expiration, thus * preventing access to the display. * - * @param event the event + * @param vmDef the VM definition * @param model the model * @param channel the channel * @throws IOException Signals that an I/O exception has occurred. * @throws TemplateException the template exception * @throws ApiException the api exception */ - public void reconcile(VmDefChanged event, - Map model, VmChannel channel) + public void reconcile(VmDefinition vmDef, Map model, + VmChannel channel) throws IOException, TemplateException, ApiException { // Secret needed at all? - var display = event.vmDefinition().fromVm("display").get(); + var display = vmDef.fromVm("display").get(); if (!DataPath. get(display, "spice", "generateSecret") .orElse(true)) { return; } // Check if exists - var vmDef = event.vmDefinition(); ListOptions options = new ListOptions(); options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + "," + "app.kubernetes.io/component=" + DisplaySecret.NAME + "," diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/LoadBalancerReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/LoadBalancerReconciler.java index 2d632b9..16af2c8 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/LoadBalancerReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/LoadBalancerReconciler.java @@ -36,7 +36,6 @@ import java.util.logging.Logger; import org.jdrupes.vmoperator.common.K8sV1ServiceStub; import org.jdrupes.vmoperator.common.VmDefinition; import org.jdrupes.vmoperator.manager.events.VmChannel; -import org.jdrupes.vmoperator.manager.events.VmDefChanged; import org.jdrupes.vmoperator.util.GsonPtr; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; @@ -69,14 +68,14 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; /** * Reconcile. * - * @param event the event + * @param vmDef the VM definition * @param model the model * @param channel the channel * @throws IOException Signals that an I/O exception has occurred. * @throws TemplateException the template exception * @throws ApiException the api exception */ - public void reconcile(VmDefChanged event, + public void reconcile(VmDefinition vmDef, Map model, VmChannel channel) throws IOException, TemplateException, ApiException { // Check if to be generated @@ -95,7 +94,6 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; } // Load balancer can also be turned off for VM - var vmDef = event.vmDefinition(); if (vmDef .>> fromSpec(LOAD_BALANCER_SERVICE) .map(m -> m.isEmpty()).orElse(false)) { diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java index 5e9b409..acda24e 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java @@ -36,7 +36,6 @@ import org.jdrupes.vmoperator.common.K8sV1SecretStub; import org.jdrupes.vmoperator.common.VmDefinition; import org.jdrupes.vmoperator.common.VmDefinition.RequestedVmState; import org.jdrupes.vmoperator.manager.events.VmChannel; -import org.jdrupes.vmoperator.manager.events.VmDefChanged; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; @@ -62,14 +61,14 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; /** * Reconcile the pod. * - * @param event the event + * @param vmDef the vm def * @param model the model * @param channel the channel * @throws IOException Signals that an I/O exception has occurred. * @throws TemplateException the template exception * @throws ApiException the api exception */ - public void reconcile(VmDefChanged event, Map model, + public void reconcile(VmDefinition vmDef, Map model, VmChannel channel) throws IOException, TemplateException, ApiException { // Don't do anything if stateful set is still in use (pre v3.4) @@ -78,7 +77,6 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; } // Get pod stub. - var vmDef = event.vmDefinition(); var podStub = K8sV1PodStub.get(channel.client(), vmDef.namespace(), vmDef.name()); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java index 34085f0..107dca7 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java @@ -38,8 +38,8 @@ import java.util.stream.Collectors; import static org.jdrupes.vmoperator.common.Constants.APP_NAME; import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME; import org.jdrupes.vmoperator.common.K8sV1PvcStub; +import org.jdrupes.vmoperator.common.VmDefinition; import org.jdrupes.vmoperator.manager.events.VmChannel; -import org.jdrupes.vmoperator.manager.events.VmDefChanged; import org.jdrupes.vmoperator.util.DataPath; import org.jdrupes.vmoperator.util.GsonPtr; import org.yaml.snakeyaml.LoaderOptions; @@ -67,7 +67,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; /** * Reconcile the PVCs. * - * @param event the event + * @param vmDef the vm def * @param model the model * @param channel the channel * @throws IOException Signals that an I/O exception has occurred. @@ -75,11 +75,9 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; * @throws ApiException the api exception */ @SuppressWarnings("PMD.AvoidDuplicateLiterals") - public void reconcile(VmDefChanged event, Map model, + public void reconcile(VmDefinition vmDef, Map model, VmChannel channel) throws IOException, TemplateException, ApiException { - var vmDef = event.vmDefinition(); - // Existing disks ListOptions listOpts = new ListOptions(); listOpts.setLabelSelector( @@ -92,7 +90,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; .collect(Collectors.toSet()); // Reconcile runner data pvc - reconcileRunnerDataPvc(event, model, channel, knownPvcs); + reconcileRunnerDataPvc(vmDef, model, channel, knownPvcs); // Reconcile pvcs for defined disks var diskDefs = vmDef.>> fromVm("disks") @@ -117,17 +115,16 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; // Update PVC model.put("disk", diskDef); - reconcileRunnerDiskPvc(event, model, channel); + reconcileRunnerDiskPvc(vmDef, model, channel); } model.remove("disk"); } - private void reconcileRunnerDataPvc(VmDefChanged event, + private void reconcileRunnerDataPvc(VmDefinition vmDef, Map model, VmChannel channel, Set knownPvcs) throws TemplateNotFoundException, MalformedTemplateNameException, ParseException, IOException, TemplateException, ApiException { - var vmDef = event.vmDefinition(); // Look for old (sts generated) name. var stsRunnerDataPvcName @@ -161,12 +158,10 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; } } - private void reconcileRunnerDiskPvc(VmDefChanged event, + private void reconcileRunnerDiskPvc(VmDefinition vmDef, Map model, VmChannel channel) throws TemplateNotFoundException, MalformedTemplateNameException, ParseException, IOException, TemplateException, ApiException { - var vmDef = event.vmDefinition(); - // Generate PVC @SuppressWarnings("unchecked") var diskDef = (Map) model.get("disk"); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java index 70f684a..170263e 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java @@ -44,10 +44,12 @@ import java.util.Optional; import java.util.logging.Level; import org.jdrupes.vmoperator.common.Convertions; import org.jdrupes.vmoperator.common.K8sObserver; +import org.jdrupes.vmoperator.common.K8sObserver.ResponseType; import org.jdrupes.vmoperator.common.VmDefinition; import org.jdrupes.vmoperator.common.VmDefinition.Assignment; import org.jdrupes.vmoperator.common.VmPool; import org.jdrupes.vmoperator.manager.events.GetPools; +import org.jdrupes.vmoperator.manager.events.PodChanged; import org.jdrupes.vmoperator.manager.events.ResetVm; import org.jdrupes.vmoperator.manager.events.VmChannel; import org.jdrupes.vmoperator.manager.events.VmDefChanged; @@ -213,20 +215,57 @@ public class Reconciler extends Component { return; } + // Reconcile + reconcile(event, channel); + } + + private void reconcile(VmDefChanged event, VmChannel channel) + throws TemplateModelException, ApiException, IOException, + TemplateException { // Create model for processing templates - Map model = prepareModel(event.vmDefinition()); + var vmDef = event.vmDefinition(); + Map model = prepareModel(vmDef); cmReconciler.reconcile(model, channel, event.specChanged()); // The remaining reconcilers depend only on changes of the spec part. if (!event.specChanged()) { return; } - dsReconciler.reconcile(event, model, channel); + dsReconciler.reconcile(vmDef, model, channel); // Manage (eventual) removal of stateful set. - stsReconciler.reconcile(event, model, channel); - pvcReconciler.reconcile(event, model, channel); - podReconciler.reconcile(event, model, channel); - lbReconciler.reconcile(event, model, channel); + stsReconciler.reconcile(vmDef, model, channel); + pvcReconciler.reconcile(vmDef, model, channel); + podReconciler.reconcile(vmDef, model, channel); + lbReconciler.reconcile(vmDef, model, channel); + } + + /** + * On pod changed. + * + * @param event the event + * @param channel the channel + * @throws ApiException the api exception + * @throws IOException Signals that an I/O exception has occurred. + * @throws TemplateException the template exception + */ + @Handler + public void onPodChanged(PodChanged event, VmChannel channel) + throws ApiException, IOException, TemplateException { + if (event.type() != ResponseType.DELETED) { + // Nothing to reconcile + return; + } + + // If the pod was deleted, it may be necessary to recreate it + var vmDef = channel.vmDefinition(); + Map model = prepareModel(vmDef); + + // Call all steps because they may augment the model + cmReconciler.reconcile(model, channel, false); + dsReconciler.reconcile(vmDef, model, channel); + stsReconciler.reconcile(vmDef, model, channel); + pvcReconciler.reconcile(vmDef, model, channel); + podReconciler.reconcile(vmDef, model, channel); } /** diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java index 8803e61..854f630 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java @@ -27,9 +27,9 @@ import java.io.IOException; import java.util.Map; import java.util.logging.Logger; import org.jdrupes.vmoperator.common.K8sV1StatefulSetStub; +import org.jdrupes.vmoperator.common.VmDefinition; import org.jdrupes.vmoperator.common.VmDefinition.RequestedVmState; import org.jdrupes.vmoperator.manager.events.VmChannel; -import org.jdrupes.vmoperator.manager.events.VmDefChanged; /** * Before version 3.4, the pod running the VM was created by a stateful set. @@ -54,7 +54,7 @@ import org.jdrupes.vmoperator.manager.events.VmDefChanged; /** * Reconcile stateful set. * - * @param event the event + * @param vmDef the VM definition * @param model the model * @param channel the channel * @throws IOException Signals that an I/O exception has occurred. @@ -62,14 +62,14 @@ import org.jdrupes.vmoperator.manager.events.VmDefChanged; * @throws ApiException the api exception */ @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") - public void reconcile(VmDefChanged event, Map model, + public void reconcile(VmDefinition vmDef, Map model, VmChannel channel) throws IOException, TemplateException, ApiException { model.put("usingSts", false); // If exists, delete when not running or supposed to be not running. var stsStub = K8sV1StatefulSetStub.get(channel.client(), - event.vmDefinition().namespace(), event.vmDefinition().name()); + vmDef.namespace(), vmDef.name()); if (stsStub.model().isEmpty()) { return; } @@ -88,7 +88,7 @@ import org.jdrupes.vmoperator.manager.events.VmDefChanged; // Check if VM is supposed to be stopped. If so, // set replicas to 0. This is the first step of the transition, // the stateful set will be deleted when the VM is restarted. - if (event.vmDefinition().vmState() == RequestedVmState.RUNNING) { + if (vmDef.vmState() == RequestedVmState.RUNNING) { return; }